mirror of https://github.com/CIRCL/AIL-framework
chg: [objects + retro hunt] refactor retro hunt + objects retro hunts + get objects generator by filters (date_from, sources, mimetypes, ...)
Terrtia
parent
ee828a04bc
commit
37c71b8438
File diff suppressed because it is too large
Load Diff
|
|
@ -43,6 +43,9 @@ def get_object_all_subtypes(obj_type):
|
|||
def get_objects_tracked():
|
||||
return ['decoded', 'item', 'pgp']
|
||||
|
||||
def get_objects_retro_hunted():
|
||||
return ['decoded', 'item']
|
||||
|
||||
def get_all_objects_with_subtypes_tuple():
|
||||
str_objs = []
|
||||
for obj_type in get_all_objects():
|
||||
|
|
@ -56,6 +59,21 @@ def get_all_objects_with_subtypes_tuple():
|
|||
|
||||
##-- AIL OBJECTS --##
|
||||
|
||||
#### Redis ####
|
||||
|
||||
def _parse_zscan(response):
|
||||
cursor, r = response
|
||||
it = iter(r)
|
||||
return str(cursor), list(it)
|
||||
|
||||
def zscan_iter(r_redis, name): # count ???
|
||||
cursor = 0
|
||||
while cursor != "0":
|
||||
cursor, data = _parse_zscan(r_redis.zscan(name, cursor=cursor))
|
||||
yield from data
|
||||
|
||||
## -- Redis -- ##
|
||||
|
||||
def paginate_iterator(iter_elems, nb_obj=50, page=1):
|
||||
dict_page = {'nb_all_elem': len(iter_elems)}
|
||||
nb_pages = dict_page['nb_all_elem'] / nb_obj
|
||||
|
|
|
|||
|
|
@ -372,6 +372,67 @@ def search_decodeds_by_name(name_to_search, r_pos=False):
|
|||
decodeds[decoded_name]['hl-end'] = res.end()
|
||||
return decodeds
|
||||
|
||||
|
||||
############################################################################
|
||||
|
||||
def get_decodeds_dir():
|
||||
decodeds_dir = os.path.join(os.environ['AIL_HOME'], HASH_DIR)
|
||||
if not decodeds_dir.endswith("/"):
|
||||
decodeds_dir = f"{decodeds_dir}/"
|
||||
return decodeds_dir
|
||||
|
||||
# Generator
|
||||
|
||||
def get_nb_decodeds_objects(filters={}):
|
||||
nb = 0
|
||||
if 'mimetypes' in filters:
|
||||
mimetypes = filters['mimetypes']
|
||||
else:
|
||||
mimetypes = get_all_mimetypes()
|
||||
d_dir = get_decodeds_dir()
|
||||
for mimetype in mimetypes:
|
||||
for root, dirs, files in os.walk(os.path.join(d_dir, mimetype)):
|
||||
nb += len(files)
|
||||
return nb
|
||||
|
||||
def get_all_decodeds_objects(filters={}):
|
||||
if 'mimetypes' in filters:
|
||||
# TODO sanityze mimetype
|
||||
mimetypes = filters['mimetypes']
|
||||
else:
|
||||
mimetypes = get_all_mimetypes()
|
||||
mimetypes = sorted(mimetypes)
|
||||
|
||||
if filters.get('start'):
|
||||
_, start_id = filters['start'].split(':', 1)
|
||||
decoded = Decoded(start_id)
|
||||
# remove sources
|
||||
start_mimetype = decoded.get_mimetype()
|
||||
i = 0
|
||||
while start_mimetype and len(mimetypes) > i:
|
||||
if mimetypes[i] == start_mimetype:
|
||||
mimetypes = mimetypes[i:]
|
||||
start_mimetype = None
|
||||
i += 1
|
||||
else:
|
||||
start_id = None
|
||||
|
||||
d_dir = get_decodeds_dir()
|
||||
for mimetype in mimetypes:
|
||||
for root, dirs, files in os.walk(os.path.join(d_dir, mimetype)):
|
||||
if start_id:
|
||||
i = 0
|
||||
while start_id and len(files) > i:
|
||||
if files[i] == start_id:
|
||||
files = files[i:]
|
||||
start_id = None
|
||||
i += 1
|
||||
if i >= len(files):
|
||||
files = []
|
||||
for file in files:
|
||||
yield Decoded(file).id
|
||||
|
||||
|
||||
############################################################################
|
||||
|
||||
def sanityze_decoder_names(decoder_name):
|
||||
|
|
@ -538,6 +599,12 @@ def get_all_decodeds_files():
|
|||
return decodeds
|
||||
|
||||
|
||||
# if __name__ == '__main__':
|
||||
if __name__ == '__main__':
|
||||
# name_to_search = '4d36'
|
||||
# print(search_decodeds_by_name(name_to_search))
|
||||
# print(search_decodeds_by_name(name_to_search))
|
||||
# filters = {'mimetypes': ['text/html']}
|
||||
filters = {'start': ':1a005f82a4ae0940205c8fd81fd14838845696be'}
|
||||
# filters = {}
|
||||
gen = get_all_decodeds_objects(filters=filters)
|
||||
for f in gen:
|
||||
print(f)
|
||||
|
|
|
|||
|
|
@ -22,7 +22,8 @@ from lib.ail_core import get_ail_uuid
|
|||
from lib.objects.abstract_object import AbstractObject
|
||||
from lib.ConfigLoader import ConfigLoader
|
||||
from lib import item_basic
|
||||
from lib.data_retention_engine import update_obj_date
|
||||
from lib.data_retention_engine import update_obj_date, get_obj_date_first
|
||||
from packages import Date
|
||||
|
||||
|
||||
from flask import url_for
|
||||
|
|
@ -406,6 +407,115 @@ def _manual_set_items_date_first_last():
|
|||
if last != 0:
|
||||
update_obj_date(last, 'item')
|
||||
|
||||
################################################################################
|
||||
################################################################################
|
||||
################################################################################
|
||||
|
||||
def get_nb_items_objects(filters={}):
|
||||
nb = 0
|
||||
date_from = filters.get('date_from')
|
||||
date_to = filters.get('date_to')
|
||||
if 'sources' in filters:
|
||||
sources = filters['sources']
|
||||
else:
|
||||
sources = get_all_sources()
|
||||
sources = sorted(sources)
|
||||
|
||||
# date
|
||||
if date_from and date_to:
|
||||
daterange = Date.get_daterange(date_from, date_to)
|
||||
elif date_from:
|
||||
daterange = Date.get_daterange(date_from, Date.get_today_date_str())
|
||||
elif date_to:
|
||||
date_from = get_obj_date_first('item')
|
||||
daterange = Date.get_daterange(date_from, date_to)
|
||||
else:
|
||||
date_from = get_obj_date_first('item')
|
||||
daterange = Date.get_daterange(date_from, Date.get_today_date_str())
|
||||
|
||||
for source in sources:
|
||||
for date in daterange:
|
||||
date = f'{date[0:4]}/{date[4:6]}/{date[6:8]}'
|
||||
full_dir = os.path.join(ITEMS_FOLDER, source, date)
|
||||
if not os.path.isdir(full_dir):
|
||||
continue
|
||||
nb += len(os.listdir(full_dir))
|
||||
return nb
|
||||
|
||||
def get_all_items_objects(filters={}):
|
||||
date_from = filters.get('date_from')
|
||||
date_to = filters.get('date_to')
|
||||
if 'sources' in filters:
|
||||
sources = filters['sources']
|
||||
else:
|
||||
sources = get_all_sources()
|
||||
sources = sorted(sources)
|
||||
if filters.get('start'):
|
||||
_, start_id = filters['start'].split(':', 1)
|
||||
item = Item(start_id)
|
||||
# remove sources
|
||||
start_source = item.get_source()
|
||||
i = 0
|
||||
while start_source and len(sources) > i:
|
||||
if sources[i] == start_source:
|
||||
sources = sources[i:]
|
||||
start_source = None
|
||||
i += 1
|
||||
start_date = item.get_date()
|
||||
else:
|
||||
start_id = None
|
||||
start_date = None
|
||||
|
||||
# date
|
||||
if date_from and date_to:
|
||||
daterange = Date.get_daterange(date_from, date_to)
|
||||
elif date_from:
|
||||
daterange = Date.get_daterange(date_from, Date.get_today_date_str())
|
||||
elif date_to:
|
||||
date_from = get_obj_date_first('item')
|
||||
daterange = Date.get_daterange(date_from, date_to)
|
||||
else:
|
||||
date_from = get_obj_date_first('item')
|
||||
daterange = Date.get_daterange(date_from, Date.get_today_date_str())
|
||||
if start_date:
|
||||
if int(start_date) > int(date_from):
|
||||
i = 0
|
||||
while start_date and len(daterange) > i:
|
||||
if daterange[i] == start_date:
|
||||
daterange = daterange[i:]
|
||||
start_date = None
|
||||
i += 1
|
||||
|
||||
for source in sources:
|
||||
for date in daterange:
|
||||
date = f'{date[0:4]}/{date[4:6]}/{date[6:8]}'
|
||||
full_dir = os.path.join(ITEMS_FOLDER, source, date)
|
||||
s_dir = os.path.join(source, date)
|
||||
if not os.path.isdir(full_dir):
|
||||
continue
|
||||
|
||||
# TODO replace by os.scandir() ????
|
||||
all_items = sorted([os.path.join(s_dir, f)
|
||||
for f in os.listdir(full_dir)
|
||||
if os.path.isfile(os.path.join(full_dir, f))])
|
||||
# start obj id
|
||||
if start_id:
|
||||
i = 0
|
||||
while start_id and len(all_items) > i:
|
||||
if all_items[i] == start_id:
|
||||
if i == len(all_items):
|
||||
all_items = []
|
||||
else:
|
||||
all_items = all_items[i+1:]
|
||||
start_id = None
|
||||
i += 1
|
||||
for obj_id in all_items:
|
||||
yield Item(obj_id)
|
||||
|
||||
################################################################################
|
||||
################################################################################
|
||||
################################################################################
|
||||
|
||||
#### API ####
|
||||
|
||||
def api_get_item(data):
|
||||
|
|
@ -810,9 +920,13 @@ def create_item(obj_id, obj_metadata, io_content):
|
|||
# delete_item(child_id)
|
||||
|
||||
|
||||
# if __name__ == '__main__':
|
||||
if __name__ == '__main__':
|
||||
# content = 'test file content'
|
||||
# duplicates = {'tests/2020/01/02/test.gz': [{'algo':'ssdeep', 'similarity':75}, {'algo':'tlsh', 'similarity':45}]}
|
||||
#
|
||||
# item = Item('tests/2020/01/02/test_save.gz')
|
||||
# item.create(content, _save=False)
|
||||
filters = {'date_from': '20230101', 'date_to': '20230501', 'sources': ['crawled', 'submitted'], 'start': ':submitted/2023/04/28/submitted_2b3dd861-a75d-48e4-8cec-6108d41450da.gz'}
|
||||
gen = get_all_items_objects(filters=filters)
|
||||
for obj_id in gen:
|
||||
print(obj_id.id)
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ sys.path.append(os.environ['AIL_BIN'])
|
|||
# Import Project packages
|
||||
##################################
|
||||
from lib.ConfigLoader import ConfigLoader
|
||||
from lib.objects.abstract_subtype_object import AbstractSubtypeObject, get_all_id
|
||||
from lib.objects.abstract_subtype_object import AbstractSubtypeObject, get_all_id, get_all_id_iterator
|
||||
|
||||
config_loader = ConfigLoader()
|
||||
baseurl = config_loader.get_config_str("Notifications", "ail_domain")
|
||||
|
|
@ -128,8 +128,22 @@ def search_pgps_by_name(name_to_search, subtype, r_pos=False):
|
|||
pgps[pgp_name]['hl-end'] = res.end()
|
||||
return pgps
|
||||
|
||||
def get_all_pgps_objects(filters={}):
|
||||
if 'subtypes' in filters:
|
||||
subtypes = filters['subtypes']
|
||||
else:
|
||||
subtypes = get_all_subtypes()
|
||||
for subtype in subtypes:
|
||||
for z_tuple in get_all_id_iterator('pgp', subtype):
|
||||
obj_id, _ = z_tuple
|
||||
yield Pgp(obj_id, subtype)
|
||||
|
||||
# if __name__ == '__main__':
|
||||
|
||||
if __name__ == '__main__':
|
||||
# name_to_search = 'ex'
|
||||
# subtype = 'name'
|
||||
# print(search_pgps_by_name(name_to_search, subtype))
|
||||
gen = get_all_pgps_objects(filters={'subtypes': ['key']})
|
||||
for f in gen:
|
||||
print(f)
|
||||
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ sys.path.append(os.environ['AIL_BIN'])
|
|||
# Import Project packages
|
||||
##################################
|
||||
from lib.objects.abstract_object import AbstractObject
|
||||
from lib.ail_core import get_object_all_subtypes
|
||||
from lib.ail_core import get_object_all_subtypes, zscan_iter
|
||||
from lib.ConfigLoader import ConfigLoader
|
||||
from lib.item_basic import is_crawled, get_item_domain
|
||||
from lib.data_retention_engine import update_obj_date
|
||||
|
|
@ -176,6 +176,9 @@ class AbstractSubtypeObject(AbstractObject, ABC):
|
|||
def get_all_id(obj_type, subtype):
|
||||
return r_object.zrange(f'{obj_type}_all:{subtype}', 0, -1)
|
||||
|
||||
def get_all_id_iterator(obj_type, subtype):
|
||||
return zscan_iter(r_object, f'{obj_type}_all:{subtype}')
|
||||
|
||||
def get_subtypes_objs_by_date(obj_type, subtype, date):
|
||||
return r_object.hkeys(f'{obj_type}:{subtype}:{date}')
|
||||
|
||||
|
|
|
|||
|
|
@ -9,16 +9,16 @@ sys.path.append(os.environ['AIL_BIN'])
|
|||
# Import Project packages
|
||||
##################################
|
||||
from lib.ConfigLoader import ConfigLoader
|
||||
from lib.ail_core import get_all_objects, get_object_all_subtypes, get_all_objects_with_subtypes_tuple
|
||||
from lib.ail_core import get_all_objects, get_object_all_subtypes
|
||||
from lib import correlations_engine
|
||||
from lib import btc_ail
|
||||
from lib import Tag
|
||||
|
||||
from lib.objects import CryptoCurrencies
|
||||
from lib.objects.Cves import Cve
|
||||
from lib.objects.Decodeds import Decoded
|
||||
from lib.objects.Decodeds import Decoded, get_all_decodeds_objects, get_nb_decodeds_objects
|
||||
from lib.objects.Domains import Domain
|
||||
from lib.objects.Items import Item
|
||||
from lib.objects.Items import Item, get_all_items_objects, get_nb_items_objects
|
||||
from lib.objects import Pgps
|
||||
from lib.objects.Screenshots import Screenshot
|
||||
from lib.objects import Usernames
|
||||
|
|
@ -114,6 +114,9 @@ def get_obj_tags(obj_type, subtype, id):
|
|||
obj = get_object(obj_type, subtype, id)
|
||||
return obj.get_tags()
|
||||
|
||||
def is_obj_tags_safe(obj_type, subtype, id):
|
||||
obj = get_object(obj_type, subtype, id)
|
||||
return obj.is_tags_safe()
|
||||
|
||||
def add_obj_tag(obj_type, subtype, id, tag):
|
||||
obj = get_object(obj_type, subtype, id)
|
||||
|
|
@ -143,6 +146,10 @@ def get_objects_meta(objs, options=set(), flask_context=False):
|
|||
obj_type = obj['type']
|
||||
subtype = obj['subtype']
|
||||
obj_id = obj['id']
|
||||
elif isinstance(obj, tuple):
|
||||
obj_type = obj[0]
|
||||
subtype = obj[1]
|
||||
obj_id = obj[2]
|
||||
else:
|
||||
obj_type, subtype, obj_id = obj.split(':', 2)
|
||||
metas.append(get_object_meta(obj_type, subtype, obj_id, options=options, flask_context=flask_context))
|
||||
|
|
@ -184,6 +191,27 @@ def is_filtered(obj, filters):
|
|||
return True
|
||||
return False
|
||||
|
||||
def obj_iterator(obj_type, filters):
|
||||
if obj_type == 'decoded':
|
||||
return get_all_decodeds_objects(filters=filters)
|
||||
elif obj_type == 'item':
|
||||
return get_all_items_objects(filters=filters)
|
||||
elif obj_type == 'pgp':
|
||||
return Pgps.get_all_pgps_objects(filters=filters)
|
||||
|
||||
def card_objs_iterators(filters):
|
||||
nb = 0
|
||||
for obj_type in filters:
|
||||
nb += int(card_obj_iterator(obj_type, filters.get(obj_type, {})))
|
||||
return nb
|
||||
|
||||
def card_obj_iterator(obj_type, filters):
|
||||
if obj_type == 'decoded':
|
||||
return get_nb_decodeds_objects(filters=filters)
|
||||
elif obj_type == 'item':
|
||||
return get_nb_items_objects(filters=filters)
|
||||
elif obj_type == 'pgp':
|
||||
return Pgps.nb_all_pgps_objects(filters=filters)
|
||||
|
||||
def get_ui_obj_tag_table_keys(obj_type): # TODO REMOVE ME
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -79,8 +79,11 @@ class Date(object):
|
|||
comp_day = str(computed_date.day).zfill(2)
|
||||
return comp_year + comp_month + comp_day
|
||||
|
||||
def get_today_date_str():
|
||||
return datetime.date.today().strftime("%Y%m%d")
|
||||
def get_today_date_str(separator=False):
|
||||
if separator:
|
||||
datetime.date.today().strftime("%Y/%m/%d")
|
||||
else:
|
||||
return datetime.date.today().strftime("%Y%m%d")
|
||||
|
||||
def date_add_day(date, num_day=1):
|
||||
new_date = datetime.date(int(date[0:4]), int(date[4:6]), int(date[6:8])) + datetime.timedelta(num_day)
|
||||
|
|
|
|||
|
|
@ -19,149 +19,122 @@ sys.path.append(os.environ['AIL_BIN'])
|
|||
# Import Project packages
|
||||
##################################
|
||||
from modules.abstract_module import AbstractModule
|
||||
from lib.ail_core import get_objects_retro_hunted
|
||||
from lib.ConfigLoader import ConfigLoader
|
||||
from lib.objects.Items import Item
|
||||
from packages import Date
|
||||
from lib.objects import ail_objects
|
||||
from lib import Tracker
|
||||
|
||||
import NotificationHelper # # TODO: refractor
|
||||
|
||||
class Retro_Hunt(AbstractModule):
|
||||
|
||||
# mail_body_template = "AIL Framework,\nNew YARA match: {}\nitem id: {}\nurl: {}{}"
|
||||
class Retro_Hunt_Module(AbstractModule):
|
||||
|
||||
"""
|
||||
Retro_Hunt module for AIL framework
|
||||
"""
|
||||
def __init__(self):
|
||||
super(Retro_Hunt, self).__init__()
|
||||
super(Retro_Hunt_Module, self).__init__()
|
||||
config_loader = ConfigLoader()
|
||||
self.pending_seconds = 5
|
||||
|
||||
self.full_item_url = config_loader.get_config_str("Notifications", "ail_domain") + "/object/item?id="
|
||||
|
||||
# reset on each loop
|
||||
self.task_uuid = None
|
||||
self.date_from = 0
|
||||
self.date_to = 0
|
||||
self.nb_src_done = 0
|
||||
self.retro_hunt = None
|
||||
self.nb_objs = 0
|
||||
self.nb_done = 0
|
||||
self.progress = 0
|
||||
self.item = None
|
||||
self.obj = None
|
||||
self.tags = []
|
||||
|
||||
self.redis_logger.info(f"Module: {self.module_name} Launched")
|
||||
|
||||
# # TODO: send mails
|
||||
# # TODO: # start_time # end_time
|
||||
|
||||
# # TODO: # start_time
|
||||
# # end_time
|
||||
def compute(self, task_uuid):
|
||||
self.redis_logger.warning(f'{self.module_name}, starting Retro hunt task {task_uuid}')
|
||||
print(f'starting Retro hunt task {task_uuid}')
|
||||
self.task_uuid = task_uuid
|
||||
self.progress = 0
|
||||
# First launch
|
||||
# restart
|
||||
retro_hunt = Tracker.RetroHunt(task_uuid) # TODO SELF
|
||||
self.retro_hunt = Tracker.RetroHunt(task_uuid)
|
||||
|
||||
rule = retro_hunt.get_rule(r_compile=True)
|
||||
rule = self.retro_hunt.get_rule(r_compile=True)
|
||||
timeout = self.retro_hunt.get_timeout()
|
||||
self.tags = self.retro_hunt.get_tags()
|
||||
|
||||
timeout = retro_hunt.get_timeout()
|
||||
self.redis_logger.debug(f'{self.module_name}, Retro Hunt rule {task_uuid} timeout {timeout}')
|
||||
sources = retro_hunt.get_sources(r_sort=True)
|
||||
|
||||
self.date_from = retro_hunt.get_date_from()
|
||||
self.date_to = retro_hunt.get_date_to()
|
||||
self.tags = retro_hunt.get_tags()
|
||||
curr_date = Tracker.get_retro_hunt_task_current_date(task_uuid)
|
||||
self.nb_src_done = Tracker.get_retro_hunt_task_nb_src_done(task_uuid, sources=sources)
|
||||
self.update_progress(sources, curr_date)
|
||||
# iterate on date
|
||||
filter_last = True
|
||||
while int(curr_date) <= int(self.date_to):
|
||||
print(curr_date)
|
||||
dirs_date = Tracker.get_retro_hunt_dir_day_to_analyze(task_uuid, curr_date, filter_last=filter_last, sources=sources)
|
||||
filter_last = False
|
||||
nb_id = 0
|
||||
self.nb_src_done = 0
|
||||
self.update_progress(sources, curr_date)
|
||||
# # TODO: Filter previous item
|
||||
for dir in dirs_date:
|
||||
print(dir)
|
||||
self.redis_logger.debug(f'{self.module_name}, Retro Hunt searching in directory {dir}')
|
||||
l_obj = Tracker.get_items_to_analyze(dir)
|
||||
for id in l_obj:
|
||||
# print(f'{dir} / {id}')
|
||||
self.item = Item(id)
|
||||
# save current item in cache
|
||||
Tracker.set_cache_retro_hunt_task_id(task_uuid, id)
|
||||
# Filters
|
||||
filters = self.retro_hunt.get_filters()
|
||||
if not filters:
|
||||
filters = {}
|
||||
for obj_type in get_objects_retro_hunted():
|
||||
filters[obj_type] = {}
|
||||
|
||||
self.redis_logger.debug(f'{self.module_name}, Retro Hunt rule {task_uuid}, searching item {id}')
|
||||
self.nb_objs = ail_objects.card_objs_iterators(filters)
|
||||
|
||||
yara_match = rule.match(data=self.item.get_content(), callback=self.yara_rules_match,
|
||||
which_callbacks=yara.CALLBACK_MATCHES, timeout=timeout)
|
||||
# Resume
|
||||
last_obj = self.retro_hunt.get_last_analyzed()
|
||||
if last_obj:
|
||||
last_obj_type, last_obj_subtype, last_obj_id = last_obj.split(':', 2)
|
||||
else:
|
||||
last_obj_type = None
|
||||
last_obj_subtype = None
|
||||
last_obj_id = None
|
||||
|
||||
# save last item
|
||||
if nb_id % 10 == 0: # # TODO: Add nb before save in DB
|
||||
Tracker.set_retro_hunt_last_analyzed(task_uuid, id)
|
||||
nb_id += 1
|
||||
self.update_progress(sources, curr_date)
|
||||
self.nb_done = 0
|
||||
self.update_progress()
|
||||
|
||||
# PAUSE
|
||||
self.update_progress(sources, curr_date)
|
||||
if retro_hunt.to_pause():
|
||||
Tracker.set_retro_hunt_last_analyzed(task_uuid, id)
|
||||
# self.update_progress(sources, curr_date, save_db=True)
|
||||
retro_hunt.pause()
|
||||
return None
|
||||
for obj_type in filters:
|
||||
if last_obj_type:
|
||||
filters['start'] = f'{last_obj_subtype}:{last_obj_id}'
|
||||
last_obj_type = None
|
||||
for obj in ail_objects.obj_iterator(obj_type, filters):
|
||||
self.obj = obj
|
||||
content = obj.get_content(r_str=True)
|
||||
rule.match(data=content, callback=self.yara_rules_match,
|
||||
which_callbacks=yara.CALLBACK_MATCHES, timeout=timeout)
|
||||
|
||||
self.nb_src_done += 1
|
||||
self.update_progress(sources, curr_date)
|
||||
curr_date = Date.date_add_day(curr_date)
|
||||
print('-----')
|
||||
self.nb_done += 1
|
||||
if self.nb_done % 10 == 0:
|
||||
self.retro_hunt.set_last_analyzed(obj.get_type(), obj.get_subtype(r_str=True), obj.get_id())
|
||||
self.retro_hunt.set_last_analyzed_cache(obj.get_type(), obj.get_subtype(r_str=True), obj.get_id())
|
||||
|
||||
self.update_progress(sources, curr_date)
|
||||
# update progress
|
||||
self.update_progress()
|
||||
|
||||
retro_hunt.complete()
|
||||
# PAUSE
|
||||
if self.retro_hunt.to_pause():
|
||||
self.retro_hunt.set_last_analyzed(obj.get_type(), obj.get_subtype(r_str=True), obj.get_id())
|
||||
self.retro_hunt.pause()
|
||||
return None
|
||||
|
||||
# Completed
|
||||
self.retro_hunt.complete()
|
||||
print(f'Retro Hunt {task_uuid} completed')
|
||||
self.redis_logger.warning(f'{self.module_name}, Retro Hunt {task_uuid} completed')
|
||||
|
||||
# # TODO: stop
|
||||
|
||||
def update_progress(self, sources, curr_date, save_db=False):
|
||||
retro_hunt = Tracker.RetroHunt(retro_hubt) # TODO USE SELF
|
||||
progress = retro_hunt.compute_progress(date_from=self.date_from, date_to=self.date_to,
|
||||
sources=sources, curr_date=curr_date, nb_src_done=self.nb_src_done)
|
||||
if self.progress != progress:
|
||||
retro_hunt.set_progress(progress)
|
||||
self.progress = progress
|
||||
# if save_db:
|
||||
# Tracker.set_retro_hunt_task_progress(task_uuid, progress)
|
||||
def update_progress(self):
|
||||
new_progress = self.nb_done * 100 / self.nb_objs
|
||||
if int(self.progress) != int(new_progress):
|
||||
print(new_progress)
|
||||
self.retro_hunt.set_progress(new_progress)
|
||||
self.progress = new_progress
|
||||
|
||||
def yara_rules_match(self, data):
|
||||
id = self.item.get_id()
|
||||
obj_id = self.obj.get_id()
|
||||
# print(data)
|
||||
task_uuid = data['namespace']
|
||||
|
||||
self.redis_logger.info(f'{self.module_name}, Retro hunt {task_uuid} match found: {id}')
|
||||
print(f'Retro hunt {task_uuid} match found: {id}')
|
||||
self.redis_logger.info(f'{self.module_name}, Retro hunt {task_uuid} match found: {obj_id}')
|
||||
print(f'Retro hunt {task_uuid} match found: {self.obj.get_type()} {obj_id}')
|
||||
|
||||
Tracker.save_retro_hunt_match(task_uuid, id)
|
||||
self.retro_hunt.add(self.obj.get_type(), self.obj.get_subtype(), obj_id)
|
||||
|
||||
# TODO FILTER Tags
|
||||
# Tags
|
||||
for tag in self.tags:
|
||||
msg = f'{tag};{id}'
|
||||
self.add_message_to_queue(msg, 'Tags')
|
||||
|
||||
# # Mails
|
||||
# mail_to_notify = Tracker.get_tracker_mails(tracker_uuid)
|
||||
# if mail_to_notify:
|
||||
# mail_subject = Tracker.get_email_subject(tracker_uuid)
|
||||
# mail_body = Tracker_Yara.mail_body_template.format(data['rule'], item_id, self.full_item_url, item_id)
|
||||
# for mail in mail_to_notify:
|
||||
# self.redis_logger.debug(f'Send Mail {mail_subject}')
|
||||
# print(f'Send Mail {mail_subject}')
|
||||
# NotificationHelper.sendEmailNotification(mail, mail_subject, mail_body)
|
||||
# EXPORTER MAILS
|
||||
return yara.CALLBACK_CONTINUE
|
||||
|
||||
def run(self):
|
||||
|
|
@ -188,6 +161,5 @@ class Retro_Hunt(AbstractModule):
|
|||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
module = Retro_Hunt()
|
||||
module = Retro_Hunt_Module()
|
||||
module.run()
|
||||
|
|
|
|||
|
|
@ -40,7 +40,7 @@ publish = D4_client
|
|||
[D4Client]
|
||||
subscribe = D4_client
|
||||
|
||||
[Retro_Hunt]
|
||||
[Retro_Hunt_Module]
|
||||
publish = Tags
|
||||
|
||||
[Tracker_Typo_Squatting]
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ import os
|
|||
import sys
|
||||
import json
|
||||
|
||||
from flask import render_template, jsonify, request, Blueprint, redirect, url_for, Response
|
||||
from flask import render_template, jsonify, request, Blueprint, redirect, url_for, Response, escape
|
||||
from flask_login import login_required, current_user, login_user, logout_user
|
||||
|
||||
sys.path.append('modules')
|
||||
|
|
@ -23,6 +23,7 @@ sys.path.append(os.environ['AIL_BIN'])
|
|||
# Import Project packages
|
||||
##################################
|
||||
from lib import ail_core
|
||||
from lib.objects import ail_objects
|
||||
from lib import item_basic
|
||||
from lib import Tracker
|
||||
from lib import Tag
|
||||
|
|
@ -149,6 +150,8 @@ def add_tracked_menu():
|
|||
galaxies_tags = json.loads(galaxies_tags)
|
||||
except:
|
||||
galaxies_tags = []
|
||||
else:
|
||||
galaxies_tags = []
|
||||
# custom tags
|
||||
if tags:
|
||||
tags = tags.split()
|
||||
|
|
@ -242,7 +245,7 @@ def tracker_delete():
|
|||
@login_required
|
||||
@login_read_only
|
||||
def retro_hunt_all_tasks():
|
||||
retro_hunts = Tracker.get_retro_hunt_tasks_metas()
|
||||
retro_hunts = Tracker.get_retro_hunt_metas()
|
||||
return render_template("retro_hunt_tasks.html", retro_hunts=retro_hunts, bootstrap_label=bootstrap_label)
|
||||
|
||||
@hunters.route('/retro_hunt/task/show', methods=['GET'])
|
||||
|
|
@ -250,40 +253,35 @@ def retro_hunt_all_tasks():
|
|||
@login_read_only
|
||||
def retro_hunt_show_task():
|
||||
task_uuid = request.args.get('uuid', None)
|
||||
objs = request.args.get('objs', False)
|
||||
|
||||
date_from = request.args.get('date_from')
|
||||
date_to = request.args.get('date_to')
|
||||
if date_from:
|
||||
date_from = date_from.replace('-', '')
|
||||
if date_to:
|
||||
date_to = date_to.replace('-', '')
|
||||
date_from_item = request.args.get('date_from')
|
||||
date_to_item = request.args.get('date_to')
|
||||
if date_from_item:
|
||||
date_from_item = date_from_item.replace('-', '')
|
||||
if date_to_item:
|
||||
date_to_item = date_to_item.replace('-', '')
|
||||
|
||||
res = Tracker.api_check_retro_hunt_task_uuid(task_uuid)
|
||||
if res:
|
||||
return create_json_response(res[0], res[1])
|
||||
|
||||
retro_hunt = Tracker.RetroHunt(task_uuid)
|
||||
dict_task = retro_hunt.get_meta(options={'creator', 'date', 'description', 'progress', 'sources', 'tags'})
|
||||
dict_task = retro_hunt.get_meta(options={'creator', 'date', 'description', 'progress', 'filters', 'nb_objs', 'tags'})
|
||||
rule_content = Tracker.get_yara_rule_content(dict_task['rule'])
|
||||
dict_task['filters'] = json.dumps(dict_task['filters'], indent=4)
|
||||
|
||||
if date_from:
|
||||
res = Tracker.api_get_retro_hunt_items({'uuid': task_uuid, 'date_from': date_from, 'date_to': date_to})
|
||||
if res[1] != 200:
|
||||
return create_json_response(res[0], res[1])
|
||||
dict_task['items'] = res[0]['items']
|
||||
dict_task['date_from_input'] = res[0]['date_from']
|
||||
dict_task['date_to_input'] = res[0]['date_to']
|
||||
if objs:
|
||||
dict_task['objs'] = ail_objects.get_objects_meta(retro_hunt.get_objs(), flask_context=True)
|
||||
else:
|
||||
dict_task['items'] = []
|
||||
dict_task['date_from_input'] = dict_task['date_from']
|
||||
dict_task['date_to_input'] = dict_task['date_to']
|
||||
dict_task['objs'] = []
|
||||
|
||||
return render_template("show_retro_hunt.html", dict_task=dict_task,
|
||||
rule_content=rule_content,
|
||||
bootstrap_label=bootstrap_label)
|
||||
rule_content=rule_content,
|
||||
bootstrap_label=bootstrap_label)
|
||||
|
||||
|
||||
@hunters.route('/retro_hunt/task/add', methods=['GET', 'POST'])
|
||||
@hunters.route('/retro_hunt/add', methods=['GET', 'POST'])
|
||||
@login_required
|
||||
@login_analyst
|
||||
def retro_hunt_add_task():
|
||||
|
|
@ -291,23 +289,69 @@ def retro_hunt_add_task():
|
|||
name = request.form.get("name", '')
|
||||
description = request.form.get("description", '')
|
||||
timeout = request.form.get("timeout", 30)
|
||||
# TAGS
|
||||
tags = request.form.get("tags", [])
|
||||
taxonomies_tags = request.form.get('taxonomies_tags')
|
||||
if taxonomies_tags:
|
||||
try:
|
||||
taxonomies_tags = json.loads(taxonomies_tags)
|
||||
except:
|
||||
taxonomies_tags = []
|
||||
else:
|
||||
taxonomies_tags = []
|
||||
galaxies_tags = request.form.get('galaxies_tags')
|
||||
if galaxies_tags:
|
||||
try:
|
||||
galaxies_tags = json.loads(galaxies_tags)
|
||||
except:
|
||||
galaxies_tags = []
|
||||
else:
|
||||
galaxies_tags = []
|
||||
# custom tags
|
||||
if tags:
|
||||
tags = tags.split()
|
||||
escaped_tags = []
|
||||
for tag in tags:
|
||||
escaped_tags.append(escape(tag))
|
||||
tags = escaped_tags
|
||||
else:
|
||||
tags = []
|
||||
tags = tags + taxonomies_tags + galaxies_tags
|
||||
# mails = request.form.get("mails", [])
|
||||
# if mails:
|
||||
# mails = mails.split()
|
||||
|
||||
sources = request.form.get("sources", [])
|
||||
if sources:
|
||||
sources = json.loads(sources)
|
||||
|
||||
date_from = request.form.get('date_from')
|
||||
date_to = request.form.get('date_to')
|
||||
if date_from:
|
||||
date_from = date_from.replace('-', '')
|
||||
if date_to:
|
||||
date_to = date_to.replace('-', '')
|
||||
# FILTERS
|
||||
filters = {}
|
||||
for obj_type in Tracker.get_objects_tracked():
|
||||
new_filter = request.form.get(f'{obj_type}_obj')
|
||||
if new_filter == 'on':
|
||||
filters[obj_type] = {}
|
||||
# Date From
|
||||
date_from = request.form.get(f'date_from_{obj_type}', '').replace('-', '')
|
||||
if date_from:
|
||||
filters[obj_type]['date_from'] = date_from
|
||||
# Date to
|
||||
date_to = request.form.get(f'date_to_{obj_type}', '').replace('-', '')
|
||||
if date_to:
|
||||
filters[obj_type]['date_to'] = date_to
|
||||
# Mimetypes
|
||||
mimetypes = request.form.get(f'mimetypes_{obj_type}', [])
|
||||
if mimetypes:
|
||||
mimetypes = json.loads(mimetypes)
|
||||
filters[obj_type]['mimetypes'] = mimetypes
|
||||
# Sources
|
||||
sources = request.form.get(f'sources_{obj_type}', [])
|
||||
if sources:
|
||||
sources = json.loads(sources)
|
||||
filters[obj_type]['sources'] = sources
|
||||
# Subtypes
|
||||
for obj_subtype in ail_core.get_object_all_subtypes(obj_type):
|
||||
subtype = request.form.get(f'filter_{obj_type}_{obj_subtype}')
|
||||
if subtype == 'on':
|
||||
if 'subtypes' not in filters[obj_type]:
|
||||
filters[obj_type]['subtypes'] = []
|
||||
filters[obj_type]['subtypes'].append(obj_subtype)
|
||||
|
||||
# YARA #
|
||||
yara_default_rule = request.form.get("yara_default_rule")
|
||||
|
|
@ -322,9 +366,9 @@ def retro_hunt_add_task():
|
|||
user_id = current_user.get_id()
|
||||
|
||||
input_dict = {"name": name, "description": description, "creator": user_id,
|
||||
"rule": rule, "type": rule_type,
|
||||
"tags": tags, "sources": sources, "timeout": timeout, #"mails": mails,
|
||||
"date_from": date_from, "date_to": date_to}
|
||||
"rule": rule, "type": rule_type,
|
||||
"tags": tags, "filters": filters, "timeout": timeout, # "mails": mails
|
||||
}
|
||||
|
||||
res = Tracker.api_create_retro_hunt_task(input_dict, user_id)
|
||||
if res[1] == 200:
|
||||
|
|
@ -334,8 +378,9 @@ def retro_hunt_add_task():
|
|||
return create_json_response(res[0], res[1])
|
||||
else:
|
||||
return render_template("add_retro_hunt_task.html",
|
||||
all_yara_files=Tracker.get_all_default_yara_files(),
|
||||
all_sources=item_basic.get_all_items_sources(r_list=True))
|
||||
all_yara_files=Tracker.get_all_default_yara_files(),
|
||||
tags_selector_data=Tag.get_tags_selector_data(),
|
||||
items_sources=item_basic.get_all_items_sources(r_list=True))
|
||||
|
||||
@hunters.route('/retro_hunt/task/pause', methods=['GET'])
|
||||
@login_required
|
||||
|
|
@ -368,27 +413,4 @@ def retro_hunt_delete_task():
|
|||
return redirect(url_for('hunters.retro_hunt_all_tasks'))
|
||||
|
||||
|
||||
#### JSON ####
|
||||
|
||||
@hunters.route("/retro_hunt/nb_items/date/json", methods=['GET'])
|
||||
@login_required
|
||||
@login_read_only
|
||||
def get_json_retro_hunt_nb_items_by_date():
|
||||
date_from = request.args.get('date_from')
|
||||
date_to = request.args.get('date_to')
|
||||
|
||||
if date_from:
|
||||
date_from = date_from.replace('-', '')
|
||||
if date_to:
|
||||
date_to = date_to.replace('-', '')
|
||||
|
||||
task_uuid = request.args.get('uuid')
|
||||
|
||||
if date_from and date_to:
|
||||
res = Tracker.get_retro_hunt_nb_item_by_day([task_uuid], date_from=date_from, date_to=date_to)
|
||||
else:
|
||||
res = Tracker.get_retro_hunt_nb_item_by_day([task_uuid])
|
||||
return jsonify(res)
|
||||
|
||||
|
||||
## - - ##
|
||||
|
|
|
|||
|
|
@ -127,6 +127,7 @@ def show_tracker():
|
|||
meta['date_to'] = date_to
|
||||
print(meta['filters'])
|
||||
meta['item_sources'] = sorted(meta['filters'].get('item', {}).get('sources', []))
|
||||
# meta['filters'] = json.dumps(meta['filters'], indent=4)
|
||||
|
||||
return render_template("showTracker.html", tracker_metadata=meta,
|
||||
yara_rule_content=yara_rule_content,
|
||||
|
|
|
|||
|
|
@ -31,28 +31,22 @@
|
|||
|
||||
<div class="col-12 col-lg-10" id="core_content">
|
||||
|
||||
<div class="card my-3">
|
||||
<div class="card-header bg-dark text-white">
|
||||
<h5 class="card-title">Create a new Retro Hunt task</h5>
|
||||
</div>
|
||||
<div class="card-body">
|
||||
<div class="card my-3">
|
||||
<div class="card-header bg-dark text-white">
|
||||
<h5 class="card-title">Create a new Retro Hunt task</h5>
|
||||
</div>
|
||||
<div class="card-body">
|
||||
|
||||
<form action="{{ url_for('hunters.retro_hunt_add_task') }}" method='post'>
|
||||
<form action="{{ url_for('hunters.retro_hunt_add_task') }}" method='post'>
|
||||
|
||||
<div class="row">
|
||||
<div class="col-12 col-xl-9">
|
||||
<div class="input-group mb-2 mr-sm-2">
|
||||
<div class="input-group-prepend">
|
||||
<div class="input-group-text bg-secondary text-white"><i class="fas fa-quote-right"></i></div>
|
||||
</div>
|
||||
<input id="name" name="name" class="form-control" placeholder="Retro Hunt Name" type="text" required>
|
||||
</div>
|
||||
<div class="input-group mb-2 mr-sm-2">
|
||||
<div class="input-group-prepend">
|
||||
<div class="input-group-text bg-danger text-white"><i class="fas fa-tag"></i></div>
|
||||
</div>
|
||||
<input id="tags" name="tags" class="form-control" placeholder="Tags to add on match (optional, space separated)" type="text">
|
||||
</div>
|
||||
<div class="row">
|
||||
<div class="col-12 col-xl-9">
|
||||
<div class="input-group mb-2 mr-sm-2">
|
||||
<div class="input-group-prepend">
|
||||
<div class="input-group-text bg-secondary text-white"><i class="fas fa-quote-right"></i></div>
|
||||
</div>
|
||||
<input id="name" name="name" class="form-control" placeholder="Retro Hunt Name" type="text" required>
|
||||
</div>
|
||||
|
||||
<!-- <div class="input-group mb-2 mr-sm-2">
|
||||
<div class="input-group-prepend">
|
||||
|
|
@ -60,85 +54,168 @@
|
|||
</div>
|
||||
<input id="mails" name="mails" class="form-control" placeholder="E-Mails Notification (optional, space separated)" type="text"}>
|
||||
</div> -->
|
||||
<div class="input-group mb-2 mr-sm-2">
|
||||
<div class="input-group-prepend">
|
||||
<div class="input-group-text bg-info text-white"><i class="fas fa-pencil-alt"></i></div>
|
||||
</div>
|
||||
<textarea id="description" name="description" class="form-control" placeholder="Retro Hunt Description (optional)" rows="3"></textarea>
|
||||
</div>
|
||||
<div class="input-group mb-2 mr-sm-2">
|
||||
<div class="input-group-prepend">
|
||||
<div class="input-group-text bg-info text-white"><i class="fas fa-pencil-alt"></i></div>
|
||||
</div>
|
||||
<textarea id="description" name="description" class="form-control" placeholder="Retro Hunt Description (optional)" rows="3"></textarea>
|
||||
</div>
|
||||
|
||||
<div class="input-group mb-2 mr-sm-2">
|
||||
<div class="input-group-prepend">
|
||||
<div class="input-group-text bg-dark text-white"><i class="fas fa-folder"></i></div>
|
||||
</div>
|
||||
<input id="sources" class="form-control" type="text" name="sources" placeholder="Sources to track (ALL IF EMPTY)" autocomplete="off">
|
||||
</div>
|
||||
<div class="card my-4">
|
||||
<div class="card-header bg-info text-white">
|
||||
<b>Objects to Track:</b>
|
||||
</div>
|
||||
<div class="card-body">
|
||||
{# <div class="custom-control custom-switch mt-1">#}
|
||||
{# <input class="custom-control-input" type="checkbox" name="cve_obj" id="cve_obj" checked="">#}
|
||||
{# <label class="custom-control-label" for="cve_obj"><i class="fas fa-bug"></i> CVE</label>#}
|
||||
{# </div>#}
|
||||
{# <div class="custom-control custom-switch mt-1">#}
|
||||
{# <input class="custom-control-input" type="checkbox" name="crypto_obj" id="crypto_obj" checked="">#}
|
||||
{# <label class="custom-control-label" for="crypto_obj"><i class="fas fa-coins"></i> Cryptocurrency</label>#}
|
||||
{# </div>#}
|
||||
<div class="custom-control custom-switch mt-1">
|
||||
<input class="custom-control-input" type="checkbox" name="decoded_obj" id="decoded_obj">
|
||||
<label class="custom-control-label" for="decoded_obj"><i class="fas fa-lock-open"></i> Decoded</label>
|
||||
</div>
|
||||
{# <div class="custom-control custom-switch mt-1">#}
|
||||
{# <input class="custom-control-input" type="checkbox" name="domain_obj" id="domain_obj" checked="">#}
|
||||
{# <label class="custom-control-label" for="domain_obj"><i class="fas fa-spider"></i> Domain</label>#}
|
||||
{# </div>#}
|
||||
<div class="custom-control custom-switch mt-1">
|
||||
<input class="custom-control-input" type="checkbox" name="item_obj" id="item_obj" checked="">
|
||||
<label class="custom-control-label" for="item_obj"><i class="fas fa-file"></i> Item</label>
|
||||
</div>
|
||||
<div class="card border-dark mb-4" id="sources_item_div">
|
||||
<div class="card-body">
|
||||
<h5>Filter Item by sources</h5>
|
||||
<div class="input-group mb-2 mr-sm-2">
|
||||
<div class="input-group-prepend">
|
||||
<div class="input-group-text bg-dark text-white"><i class="fas fa-folder"></i></div>
|
||||
</div>
|
||||
<input id="sources_item" class="form-control" type="text" name="sources_item" placeholder="Item Sources to track (ALL IF EMPTY)" autocomplete="off">
|
||||
</div>
|
||||
<h5>Date range:</h5>
|
||||
<div class="row mb-2">
|
||||
<div class="col-lg-6">
|
||||
<div class="input-group" id="date-range-from">
|
||||
<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
|
||||
<input class="form-control" id="date-range-from-input" placeholder="yyyy-mm-dd" name="date_from_item" autocomplete="off">
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-lg-6">
|
||||
<div class="input-group" id="date-range-to">
|
||||
<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
|
||||
<input class="form-control" id="date-range-to-input" placeholder="yyyy-mm-dd" name="date_to_item" autocomplete="off">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<h6>Date range:</h6>
|
||||
<div class="row mb-2">
|
||||
<div class="col-lg-6">
|
||||
<div class="input-group" id="date-range-from">
|
||||
<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
|
||||
<input class="form-control" id="date-range-from-input" placeholder="yyyy-mm-dd" name="date_from" autocomplete="off" required>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-lg-6">
|
||||
<div class="input-group" id="date-range-to">
|
||||
<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
|
||||
<input class="form-control" id="date-range-to-input" placeholder="yyyy-mm-dd" name="date_to" autocomplete="off" required>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
<div class="col-12 col-xl-3">
|
||||
{# <div class="custom-control custom-switch mt-1">#}
|
||||
{# <input class="custom-control-input" type="checkbox" name="pgp_obj" id="pgp_obj" checked="">#}
|
||||
{# <label class="custom-control-label" for="pgp_obj"><i class="fas fa-key"></i> PGP</label>#}
|
||||
{# </div>#}
|
||||
{# <div class="card border-dark mb-4" id="sources_pgp_div">#}
|
||||
{# <div class="card-body">#}
|
||||
{# <h6>Filter PGP by subtype:</h6>#}
|
||||
{# <div class="custom-control custom-switch mt-1">#}
|
||||
{# <input class="custom-control-input" type="checkbox" name="filter_pgp_name" id="filter_pgp_name" checked="">#}
|
||||
{# <label class="custom-control-label" for="filter_pgp_name">#}
|
||||
{# <svg height="26" width="26">#}
|
||||
{# <g class="nodes">#}
|
||||
{# <circle cx="13" cy="13" r="13" fill="#44AA99"></circle>#}
|
||||
{# <text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fas" font-size="16px"></text>#}
|
||||
{# </g>#}
|
||||
{# </svg>#}
|
||||
{# name#}
|
||||
{# </label>#}
|
||||
{# </div>#}
|
||||
{# <div class="custom-control custom-switch mt-1">#}
|
||||
{# <input class="custom-control-input" type="checkbox" name="filter_pgp_mail" id="filter_pgp_mail" checked="">#}
|
||||
{# <label class="custom-control-label" for="filter_pgp_mail">#}
|
||||
{# <svg height="26" width="26">#}
|
||||
{# <g class="nodes">#}
|
||||
{# <circle cx="13" cy="13" r="13" fill="#44AA99"></circle>#}
|
||||
{# <text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="fas" font-size="16px"></text>#}
|
||||
{# </g>#}
|
||||
{# </svg>#}
|
||||
{# mail#}
|
||||
{# </label>#}
|
||||
{# </div>#}
|
||||
{# </div>#}
|
||||
{# </div>#}
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="card my-4">
|
||||
<div class="card-header bg-secondary text-white">
|
||||
<b>Tags</b>
|
||||
</div>
|
||||
<div class="card-body">
|
||||
<div class="input-group mb-2 mr-sm-2">
|
||||
<div class="input-group-prepend">
|
||||
<div class="input-group-text bg-danger text-white"><i class="fas fa-tag"></i></div>
|
||||
</div>
|
||||
<input id="tags" name="tags" class="form-control" placeholder="Custom Tags (optional, space separated)" type="text">
|
||||
</div>
|
||||
{% include 'tags/block_tags_selector.html' %}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
</div>
|
||||
<div class="col-12 col-xl-3">
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
|
||||
<div class="mb-2" id="yara_rule">
|
||||
<div class="" id="yara_default_rules">
|
||||
<div class="mb-2" id="yara_rule">
|
||||
<div class="" id="yara_default_rules">
|
||||
|
||||
<h6>Default YARA rules:</h6>
|
||||
<select class="custom-select w-100 mb-3" id="yara_default_rule" name="yara_default_rule" onchange="get_default_rule_content(this);">
|
||||
<option selected>Select a default rule</option>
|
||||
{% for yara_types in all_yara_files %}
|
||||
{% for yara_file_name in all_yara_files[yara_types] %}
|
||||
<option value="{{yara_types}}/{{yara_file_name}}">{{yara_types}} - {{yara_file_name}}</option>
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
</select>
|
||||
<h6>Default YARA rules:</h6>
|
||||
<select class="custom-select w-100 mb-3" id="yara_default_rule" name="yara_default_rule" onchange="get_default_rule_content(this);">
|
||||
<option selected>Select a default rule</option>
|
||||
{% for yara_types in all_yara_files %}
|
||||
{% for yara_file_name in all_yara_files[yara_types] %}
|
||||
<option value="{{yara_types}}/{{yara_file_name}}">{{yara_types}} - {{yara_file_name}}</option>
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
</select>
|
||||
|
||||
<pre class="border bg-light" id="default_yara_rule_content"></pre>
|
||||
<pre class="border bg-light" id="default_yara_rule_content"></pre>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<hr>
|
||||
<hr>
|
||||
|
||||
<h6>Custom YARA rules:</h6>
|
||||
<div class="row" id="textarea">
|
||||
<textarea class="form-control mx-3" id="text_input" name="yara_custom_rule" placeholder="Enter your own YARA rule" rows="5">{%if dict_tracker%}{%if dict_tracker['type']=='yara' and dict_tracker['content']%}{{dict_tracker['content']}}{%endif%}{%endif%}</textarea>
|
||||
</div>
|
||||
</div>
|
||||
<h6>Custom YARA rules:</h6>
|
||||
<div class="row" id="textarea">
|
||||
<textarea class="form-control mx-3" id="text_input" name="yara_custom_rule" placeholder="Enter your own YARA rule" rows="5"></textarea>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<br>
|
||||
<button class="btn btn-primary mt-2">
|
||||
<i class="fas fa-plus"></i> Create Retro Hunt Task
|
||||
</button>
|
||||
<br>
|
||||
<button class="btn btn-primary mt-2">
|
||||
<i class="fas fa-plus"></i> Create Retro Hunt Task
|
||||
</button>
|
||||
|
||||
</form>
|
||||
</form>
|
||||
|
||||
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
|
@ -177,22 +254,21 @@ $(document).ready(function(){
|
|||
}
|
||||
});
|
||||
|
||||
sources = $('#sources').tagSuggest({
|
||||
data: {{all_sources|safe}},
|
||||
{%if dict_tracker%}{%if dict_tracker['sources']%}value: {{dict_tracker['sources']|safe}},{%endif%}{%endif%}
|
||||
sources_item = $('#sources_item').tagSuggest({
|
||||
data: {{items_sources|safe}},
|
||||
sortOrder: 'name',
|
||||
maxDropHeight: 200,
|
||||
name: 'sources',
|
||||
emptyText: 'Sources to track (ALL IF EMPTY)',
|
||||
name: 'sources_item',
|
||||
emptyText: 'Item Sources to track (ALL IF EMPTY)',
|
||||
});
|
||||
|
||||
{%if dict_tracker%}
|
||||
$('#tracker_type').val('{{dict_tracker['type']}}').change();
|
||||
$('#item_obj').on("change", function () {
|
||||
item_source_input_controller();
|
||||
});
|
||||
/*$('#pgp_obj').on("change", function () {
|
||||
pgp_source_input_controller();
|
||||
});*/
|
||||
|
||||
{%if dict_tracker['type']=='yara' and dict_tracker['yara_file']%}
|
||||
$('#yara_default_rule').val('{{dict_tracker['yara_file']}}').change();
|
||||
{%endif%}
|
||||
{%endif%}
|
||||
|
||||
});
|
||||
|
||||
|
|
@ -210,7 +286,21 @@ function toggle_sidebar(){
|
|||
}
|
||||
}
|
||||
|
||||
function item_source_input_controller() {
|
||||
if($('#item_obj').is(':checked')){
|
||||
$("#sources_item_div").show();
|
||||
}else{
|
||||
$("#sources_item_div").hide();
|
||||
}
|
||||
}
|
||||
|
||||
function pgp_source_input_controller() {
|
||||
if($('#pgp_obj').is(':checked')){
|
||||
$("#sources_pgp_div").show();
|
||||
}else{
|
||||
$("#sources_pgp_div").hide();
|
||||
}
|
||||
}
|
||||
|
||||
function get_default_rule_content(selector){
|
||||
var yara_name = selector.value
|
||||
|
|
|
|||
|
|
@ -20,7 +20,6 @@
|
|||
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
|
||||
<script language="javascript" src="{{ url_for('static', filename='js/d3.min.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/d3/graphlinesgroup.js')}}"></script>
|
||||
<script language="javascript" src="{{ url_for('static', filename='js/moment.min.js') }}"></script>
|
||||
<script language="javascript" src="{{ url_for('static', filename='js/jquery.daterangepicker.min.js') }}"></script>
|
||||
|
||||
|
|
@ -48,9 +47,9 @@
|
|||
|
||||
<div class="col-12 col-lg-10" id="core_content">
|
||||
|
||||
<div class="row">
|
||||
<div class="col-lg-4">
|
||||
<div class="card my-2">
|
||||
<div class="row">
|
||||
<div class="col-lg-4">
|
||||
<div class="card my-2">
|
||||
<div class="card-header bg-dark text-white">
|
||||
<div class="">
|
||||
<span class="badge badge-pill badge-light lex-row-reverse float-right">
|
||||
|
|
@ -68,13 +67,13 @@
|
|||
<h3 class="card-title">
|
||||
{{dict_task['name']}}
|
||||
{%if dict_task['state']!='completed'%}
|
||||
<span class="text-info">[{{ dict_task['progress']}}%]<span>
|
||||
<span class="text-info">[{{ dict_task['progress']}}%]</span>
|
||||
{%endif%}
|
||||
</h3>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
<div class="card-body bg-light pt-2">
|
||||
<div class="card-body bg-light pt-2">
|
||||
<div class="d-flex justify-content-end">
|
||||
{%if dict_task['state']=='paused'%}
|
||||
<a href="{{ url_for('hunters.retro_hunt_resume_task') }}?uuid={{dict_task['uuid']}}" class="mx-1">
|
||||
|
|
@ -94,83 +93,85 @@
|
|||
</a>
|
||||
</div>
|
||||
<hr>
|
||||
<table class="table table-borderless">
|
||||
<tbody>
|
||||
<tr>
|
||||
<td class="text-right"><b>Date</b></td>
|
||||
<td>
|
||||
{{dict_task['date'][0:4]}}/{{dict_task['date'][4:6]}}/{{dict_task['date'][6:8]}}
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td class="text-right"><b>Search date</b></td>
|
||||
<td>
|
||||
{{dict_task['date_from'][0:4]}}/{{dict_task['date_from'][4:6]}}/{{dict_task['date_from'][6:8]}} -
|
||||
{{dict_task['date_to'][0:4]}}/{{dict_task['date_to'][4:6]}}/{{dict_task['date_to'][6:8]}}
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td class="text-right"><b>Description</b></td>
|
||||
<td>{{dict_task['description']}}</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td class="text-right"><b>Tags</b></td>
|
||||
<td>
|
||||
{%for tag in dict_task['tags']%}
|
||||
<a href="{{ url_for('tags_ui.get_obj_by_tags') }}?object_type=item<ags={{ tag }}">
|
||||
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span>
|
||||
</a>
|
||||
{%endfor%}
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td class="text-right"><b>Creator</b></td>
|
||||
<td>{{dict_task['creator']}}</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td class="text-right"><b>Sources</b></td>
|
||||
<td>
|
||||
<div class="">
|
||||
{%if not dict_task['sources']%}
|
||||
<span class="badge badge-secondary">All Souces</span>
|
||||
{%else%}
|
||||
{%for source in dict_task['sources']%}
|
||||
<span class="badge badge-secondary">{{source}}</span>
|
||||
{%endfor%}
|
||||
{%endif%}
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<table class="table table-borderless">
|
||||
<tbody>
|
||||
<tr>
|
||||
<td class="text-right"><b>Date</b></td>
|
||||
<td>
|
||||
{{dict_task['date'][0:4]}}/{{dict_task['date'][4:6]}}/{{dict_task['date'][6:8]}}
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td class="text-right"><b>Description</b></td>
|
||||
<td>{{dict_task['description']}}</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td class="text-right"><b>Tags</b></td>
|
||||
<td>
|
||||
{%for tag in dict_task['tags']%}
|
||||
<a href="{{ url_for('tags_ui.get_obj_by_tags') }}?object_type=item<ags={{ tag }}">
|
||||
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span>
|
||||
</a>
|
||||
{%endfor%}
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td class="text-right"><b>Creator</b></td>
|
||||
<td>{{dict_task['creator']}}</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td class="text-right"><b>Filters</b></td>
|
||||
<td>
|
||||
<div class="">
|
||||
{% if dict_task['filters'] %}
|
||||
<pre>{{ dict_task['filters'] }}</pre>
|
||||
{% else %}
|
||||
<span class="badge badge-secondary">No Filters</span><br>
|
||||
{% endif %}
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td class="text-right"><b>Objects Match</b></td>
|
||||
<td>
|
||||
{%for obj_type in dict_task['nb_objs']%}
|
||||
<h4><span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">
|
||||
{{ obj_type }}
|
||||
<span class="badge badge-light">{{ dict_task['nb_objs'][obj_type] }}</span>
|
||||
</span></h4>
|
||||
{%endfor%}
|
||||
</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="col-lg-8">
|
||||
<div class="col-lg-8">
|
||||
|
||||
<div class="card border-dark my-2">
|
||||
<div class="card-body">
|
||||
|
||||
<div class="row mb-2">
|
||||
<div class="col-lg-6">
|
||||
<div class="input-group" id="date-range-from">
|
||||
<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
|
||||
<input class="form-control" id="date-range-from-input" placeholder="yyyy-mm-dd" name="date_from" autocomplete="off"
|
||||
value="{{ dict_task['date_from_input'][0:4] }}-{{ dict_task['date_from_input'][4:6] }}-{{ dict_task['date_from_input'][6:8] }}">
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-lg-6">
|
||||
<div class="input-group" id="date-range-to">
|
||||
<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
|
||||
<input class="form-control" id="date-range-to-input" placeholder="yyyy-mm-dd" name="date_to" autocomplete="off"
|
||||
value="{{ dict_task['date_to_input'][0:4] }}-{{ dict_task['date_to_input'][4:6] }}-{{ dict_task['date_to_input'][6:8] }}">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
{# <div class="row mb-2">#}
|
||||
{# <div class="col-lg-6">#}
|
||||
{# <div class="input-group" id="date-range-from">#}
|
||||
{# <div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>#}
|
||||
{# <input class="form-control" id="date-range-from-input" placeholder="yyyy-mm-dd" name="date_from" autocomplete="off"> #}
|
||||
{# value="{{ dict_task['date_from_input'][0:4] }}-{{ dict_task['date_from_input'][4:6] }}-{{ dict_task['date_from_input'][6:8] }}"#}
|
||||
{# </div>#}
|
||||
{# </div>#}
|
||||
{# <div class="col-lg-6">#}
|
||||
{# <div class="input-group" id="date-range-to">#}
|
||||
{# <div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>#}
|
||||
{# <input class="form-control" id="date-range-to-input" placeholder="yyyy-mm-dd" name="date_to" autocomplete="off"> #}
|
||||
{# value="{{ dict_task['date_to_input'][0:4] }}-{{ dict_task['date_to_input'][4:6] }}-{{ dict_task['date_to_input'][6:8] }}"#}
|
||||
{# </div>#}
|
||||
{# </div>#}
|
||||
{# </div>#}
|
||||
|
||||
<button class="btn btn-info" type="button" id="button-search-tags" onclick="getItems();">
|
||||
<i class="fas fa-search"></i> Show Items
|
||||
<i class="fas fa-search"></i> Show Objects
|
||||
</button>
|
||||
|
||||
</div>
|
||||
|
|
@ -178,40 +179,52 @@
|
|||
|
||||
<p class="my-0"><pre class="border bg-light">{{ rule_content }}</pre></p>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{%if dict_task['items']%}
|
||||
{% if dict_task['objs'] %}
|
||||
<div class="mt-4">
|
||||
<table class="table table-bordered table-hover" id="items_table">
|
||||
<table class="table table-bordered table-hover" id="objs_table">
|
||||
<thead class="thead-dark">
|
||||
<tr>
|
||||
<th>Date</th>
|
||||
<th>Item Id</th>
|
||||
<th>Type</th>
|
||||
<th></th>
|
||||
<th>Id</th>
|
||||
<th>Tags</th>
|
||||
<th></th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
|
||||
{% for item in dict_task['items'] %}
|
||||
<tr>
|
||||
<td>
|
||||
{{item['date'][0:4]}}/{{item['date'][4:6]}}/{{item['date'][6:8]}}
|
||||
</td>
|
||||
<td>
|
||||
<a class="text-secondary" target="_blank" href="{{ url_for('objects_item.showItem') }}?id={{item['id']}}">
|
||||
<div style="line-height:0.9;">{{ item['id'] }}</div>
|
||||
</a>
|
||||
<div class="mb-2">
|
||||
{% for tag in item['tags'] %}
|
||||
<a href="{{ url_for('tags_ui.get_obj_by_tags') }}?object_type=item<ags={{ tag }}">
|
||||
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }}</span>
|
||||
</a>
|
||||
{% endfor %}
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
{% for object in dict_task['objs'] %}
|
||||
<tr class="border-color: blue;">
|
||||
<td>
|
||||
{% with style=object['icon']['style'], icon=object['icon']['icon'] , color=object['icon']['color'] %}
|
||||
{% include 'objects/obj_svg_block.html' %}
|
||||
{% endwith %}
|
||||
{{ object['type']}}
|
||||
</td>
|
||||
<td>
|
||||
{% if object['subtype'] %}
|
||||
{{ object['subtype']}}
|
||||
{% endif %}
|
||||
</td>
|
||||
<td>
|
||||
<a href="{{ object['link'] }}">
|
||||
{{ object['id']}}
|
||||
</a>
|
||||
</td>
|
||||
<td>
|
||||
{% for tag in object['tags'] %}
|
||||
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }}</span>
|
||||
{% endfor %}
|
||||
</td>
|
||||
<td class="text-right">
|
||||
{# <a href="{{ url_for('investigations_b.unregister_investigation') }}?uuid={{ tracker_metadata['uuid']}}&type={{ object['type'] }}&subtype={{ object['subtype']}}&id={{ object['id']}}">#}
|
||||
{# <button type="button" class="btn btn-danger"><i class="fas fa-trash-alt"></i></button>#}
|
||||
{# </a>#}
|
||||
</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
|
@ -219,13 +232,6 @@
|
|||
<hr>
|
||||
{% endif %}
|
||||
|
||||
|
||||
<div id="graphline" class="text-center"></div>
|
||||
<br>
|
||||
<br>
|
||||
<br>
|
||||
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
|
@ -234,7 +240,7 @@
|
|||
$(document).ready(function(){
|
||||
$('#nav_title_retro_hunt').removeClass("text-muted");
|
||||
|
||||
$('#date-range-from').dateRangePicker({
|
||||
/*$('#date-range-from').dateRangePicker({
|
||||
separator : ' to ',
|
||||
getValue: function(){
|
||||
if ($('#date-range-from-input').val() && $('#date-range-to-input').val() )
|
||||
|
|
@ -259,18 +265,15 @@ $(document).ready(function(){
|
|||
$('#date-range-from-input').val(s1);
|
||||
$('#date-range-to-input').val(s2);
|
||||
}
|
||||
});
|
||||
});*/
|
||||
|
||||
$('#items_table').DataTable({
|
||||
{% if dict_task['objs'] %}
|
||||
$('#objs_table').DataTable({
|
||||
"aLengthMenu": [[5, 10, 15, -1], [5, 10, 15, "All"]],
|
||||
"iDisplayLength": 10,
|
||||
"order": [[ 0, "asc" ]]
|
||||
});
|
||||
|
||||
let div_width = $("#graphline").width();
|
||||
$.getJSON( "{{ url_for('hunters.get_json_retro_hunt_nb_items_by_date') }}?uuid={{ dict_task['uuid'] }}&date_from={{ dict_task['date_from_input'] }}&date_to={{ dict_task['date_to_input'] }}",
|
||||
function( data ) {multilines_group("graphline", data, {"width": div_width});}
|
||||
);
|
||||
});
|
||||
{% endif%}
|
||||
|
||||
});
|
||||
|
||||
|
|
@ -291,7 +294,8 @@ function toggle_sidebar(){
|
|||
function getItems() {
|
||||
var date_from = $('#date-range-from-input').val();
|
||||
var date_to =$('#date-range-to-input').val();
|
||||
window.location.replace("{{ url_for('hunters.retro_hunt_show_task') }}?uuid={{ dict_task['uuid'] }}&date_from="+date_from+"&date_to="+date_to);
|
||||
{#window.location.replace("{{ url_for('hunters.retro_hunt_show_task') }}?uuid={{ dict_task['uuid'] }}&date_from="+date_from+"&date_to="+date_to);#}
|
||||
window.location.replace("{{ url_for('hunters.retro_hunt_show_task') }}?uuid={{ dict_task['uuid'] }}&objs=True");
|
||||
}
|
||||
|
||||
</script>
|
||||
|
|
|
|||
Loading…
Reference in New Issue