CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

chg: [trackers] refactor trackers: track objects + filter by object types/sources/subtypes

pull/594/head
Terrtia 2023-05-04 16:35:56 +02:00
parent e363dcda62
commit 0daf5bad44
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
41 changed files with 2549 additions and 2124 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
bin/DB_KVROCKS_MIGRATION.py
View File

@ -233,17 +233,61 @@ def ail_2_ail_migration():
# item in queue # item in queue
ail_2_ail.set_last_updated_sync_config() ail_2_ail.set_last_updated_sync_config()
###############################
# #
# TRACKER MIGRATION #
# #
###############################
def get_tracker_level(tracker_uuid):
level = r_serv_tracker.hget(f'tracker:{tracker_uuid}', 'level')
if not level:
level = 0
return int(level)
def get_tracker_metadata(tracker_uuid):
meta = {'uuid': tracker_uuid,
'tracked': r_serv_tracker.hget('tracker:{tracker_uuid}', 'tracked'),
'type': r_serv_tracker.hget('tracker:{tracker_uuid}', 'type'),
'date': r_serv_tracker.hget(f'tracker:{tracker_uuid}', 'date'),
'first_seen': r_serv_tracker.hget(f'tracker:{tracker_uuid}', 'first_seen'),
'last_seen': r_serv_tracker.hget(f'tracker:{tracker_uuid}', 'last_seen'),
'user_id': r_serv_tracker.hget('tracker:{tracker_uuid}', 'user_id'),
'level': get_tracker_level(tracker_uuid),
'mails': list(r_serv_tracker.smembers('tracker:mail:{tracker_uuid}')),
'sources': list(r_serv_tracker.smembers(f'tracker:sources:{tracker_uuid}')),
'tags': list(r_serv_tracker.smembers(f'tracker:tags:{tracker_uuid}')),
'description': r_serv_tracker.hget(f'tracker:{tracker_uuid}', 'description'),
'webhook': r_serv_tracker.hget(f'tracker:{tracker_uuid}', 'webhook')}
return meta
def get_tracker_items_by_daterange(tracker_uuid, date_from, date_to):
all_item_id = set()
if date_from and date_to:
l_date_match = r_serv_tracker.zrange(f'tracker:stat:{tracker_uuid}', 0, -1, withscores=True)
if l_date_match:
dict_date_match = dict(l_date_match)
for date_day in Date.substract_date(date_from, date_to):
if date_day in dict_date_match:
all_item_id |= r_serv_tracker.smembers(f'tracker:item:{tracker_uuid}:{date_day}')
return all_item_id
# trackers + retro_hunts # trackers + retro_hunts
def trackers_migration(): def trackers_migration():
print('TRACKERS MIGRATION...') print('TRACKERS MIGRATION...')
for tracker_uuid in old_Tracker.get_all_tracker_uuid(): for tracker_uuid in old_Tracker.get_all_tracker_uuid():
meta = old_Tracker.get_tracker_metadata(tracker_uuid, user_id=True, description=True, level=True, tags=True, mails=True, sources=True, sparkline=False, webhook=True) meta = get_tracker_metadata(tracker_uuid)
Tracker._re_create_tracker(meta['tracker'], meta['type'], meta['user_id'], meta['level'], meta['tags'], meta['mails'], meta['description'], meta['webhook'], 0, meta['uuid'], meta['sources'], meta['first_seen'], meta['last_seen']) Tracker._re_create_tracker(meta['type'], meta['uuid'], meta['tracked'], meta['user_id'], meta['level'],
tags=meta['tags'], mails=meta['mails'], description=meta['description'],
webhook=meta['webhook'], sources=meta['sources'],
first_seen=meta['first_seen'], last_seen=meta['last_seen'])
tracker = Tracker.Tracker(tracker_uuid)
# object migration # # TODO: in background # object migration # # TODO: in background
for item_id in old_Tracker.get_tracker_items_by_daterange(tracker_uuid, meta['first_seen'], meta['last_seen']): for item_id in old_Tracker.get_tracker_items_by_daterange(tracker_uuid, meta['first_seen'], meta['last_seen']):
print(item_id) print(item_id)
Tracker.add_tracked_item(tracker_uuid, item_id) item_date = get_item_date(item_id)
tracker.add('item', '', item_id, date=item_date)
print('RETRO HUNT MIGRATION...') print('RETRO HUNT MIGRATION...')
@ -929,13 +973,13 @@ if __name__ == '__main__':
# user_migration() # user_migration()
#tags_migration() #tags_migration()
# items_migration() # items_migration()
crawler_migration() # crawler_migration()
# domain_migration() # TO TEST ########################### # domain_migration() # TO TEST ###########################
# decodeds_migration() # decodeds_migration()
# screenshots_migration() # screenshots_migration()
# subtypes_obj_migration() # subtypes_obj_migration()
# ail_2_ail_migration() # ail_2_ail_migration()
# trackers_migration() trackers_migration()
# investigations_migration() # investigations_migration()
## statistics_migration() ## statistics_migration()

16
bin/core/DbCleaner.py
View File

@ -14,23 +14,9 @@ sys.path.append(os.environ['AIL_BIN'])
################################## ##################################
# Import Project packages # Import Project packages
################################## ##################################
from packages import Date
from packages import Term
from pubsublogger import publisher from pubsublogger import publisher
def clean_term_db_stat_token():
all_stat_date = Term.get_all_token_stat_history()
list_date_to_keep = Date.get_date_range(31)
for date in all_stat_date:
if date not in list_date_to_keep:
# remove history
Term.delete_token_statistics_by_date(date)
print('Term Stats Cleaned')
if __name__ == "__main__": if __name__ == "__main__":
publisher.port = 6380 publisher.port = 6380
@ -46,7 +32,7 @@ if __name__ == "__main__":
while True: while True:
if daily_cleaner: if daily_cleaner:
clean_term_db_stat_token()
daily_cleaner = False daily_cleaner = False
else: else:
sys.exit(0) sys.exit(0)

4
bin/exporter/MailExporter.py
View File

@ -105,7 +105,7 @@ class MailExporterTracker(MailExporter):
def export(self, tracker, obj): # TODO match def export(self, tracker, obj): # TODO match
tracker_type = tracker.get_type() tracker_type = tracker.get_type()
tracker_name = tracker.get_tracker() tracker_name = tracker.get_tracked()
subject = f'AIL Framework Tracker: {tracker_name}' # TODO custom subject subject = f'AIL Framework Tracker: {tracker_name}' # TODO custom subject
body = f"AIL Framework, New occurrence for {tracker_type} tracker: {tracker_name}\n" body = f"AIL Framework, New occurrence for {tracker_type} tracker: {tracker_name}\n"
body += f'Item: {obj.id}\nurl:{obj.get_link()}' body += f'Item: {obj.id}\nurl:{obj.get_link()}'
@ -115,4 +115,4 @@ class MailExporterTracker(MailExporter):
# body += f'Tracker Match:\n\n{escape(match)}' # body += f'Tracker Match:\n\n{escape(match)}'
for mail in tracker.get_mails(): for mail in tracker.get_mails():
self._export(mail, subject, body) self._export(mail, subject, body)

2
bin/exporter/WebHookExporter.py
View File

@ -53,7 +53,7 @@ class WebHookExporterTracker(WebHookExporter):
data = {'trackerId': tracker.get_uuid(), data = {'trackerId': tracker.get_uuid(),
'trackerType': tracker.get_type(), 'trackerType': tracker.get_type(),
'tags': tracker.get_tags(), 'tags': tracker.get_tags(),
'tracker': tracker.get_tracker(), 'tracker': tracker.get_tracked(),
# object # object
'itemId': obj.get_id(), 'itemId': obj.get_id(),
'itemURL': obj.get_link()} 'itemURL': obj.get_link()}

8
bin/lib/Investigations.py
View File

@ -186,9 +186,9 @@ class Investigation(object):
def set_threat_level(self, threat_level): def set_threat_level(self, threat_level):
try: try:
threat_level = int(threat_level) threat_level = int(threat_level)
except: except TypeError:
raise UpdateInvestigationError('threat_level Not an integer') raise UpdateInvestigationError('threat_level Not an integer')
if threat_level >= 1 and threat_level <= 4: if 1 <= threat_level <= 4:
r_tracking.hset(f'investigations:data:{self.uuid}', 'threat_level', threat_level) r_tracking.hset(f'investigations:data:{self.uuid}', 'threat_level', threat_level)
else: else:
raise UpdateInvestigationError(f'Invalid threat_level: {threat_level}') raise UpdateInvestigationError(f'Invalid threat_level: {threat_level}')
@ -196,9 +196,9 @@ class Investigation(object):
def set_analysis(self, analysis): def set_analysis(self, analysis):
try: try:
analysis = int(analysis) analysis = int(analysis)
except: except TypeError:
raise UpdateInvestigationError('analysis Not an integer') raise UpdateInvestigationError('analysis Not an integer')
if analysis >= 0 and analysis <= 2: if 0 <= analysis <= 2:
r_tracking.hset(f'investigations:data:{self.uuid}', 'analysis', analysis) r_tracking.hset(f'investigations:data:{self.uuid}', 'analysis', analysis)
else: else:
raise UpdateInvestigationError(f'Invalid analysis: {analysis}') raise UpdateInvestigationError(f'Invalid analysis: {analysis}')

1746
bin/lib/Tracker.py
View File

File diff suppressed because it is too large Load Diff

3
bin/lib/ail_core.py
View File

@ -40,6 +40,9 @@ def get_object_all_subtypes(obj_type):
return ['telegram', 'twitter', 'jabber'] return ['telegram', 'twitter', 'jabber']
return [] return []
def get_objects_tracked():
return ['decoded', 'item', 'pgp']
def get_all_objects_with_subtypes_tuple(): def get_all_objects_with_subtypes_tuple():
str_objs = [] str_objs = []
for obj_type in get_all_objects(): for obj_type in get_all_objects():

23
bin/lib/module_extractor.py
View File

@ -95,17 +95,18 @@ def convert_byte_offset_to_string(b_content, offset):
def get_tracker_match(obj_id, content): def get_tracker_match(obj_id, content):
extracted = [] extracted = []
extracted_yara = [] extracted_yara = []
trackers = Tracker.get_obj_all_trackers('item', '', obj_id) trackers = Tracker.get_obj_trackers('item', '', obj_id)
for tracker_uuid in trackers: for tracker_uuid in trackers:
tracker_type = Tracker.get_tracker_type(tracker_uuid) tracker = Tracker.Tracker(tracker_uuid)
tracker_type = tracker.get_type()
# print(tracker_type) # print(tracker_type)
tracker = Tracker.get_tracker_by_uuid(tracker_uuid) tracked = tracker.get_tracked()
if tracker_type == 'regex': # TODO Improve word detection -> word delimiter if tracker_type == 'regex': # TODO Improve word detection -> word delimiter
regex_match = regex_helper.regex_finditer(r_key, tracker, obj_id, content) regex_match = regex_helper.regex_finditer(r_key, tracked, obj_id, content)
for match in regex_match: for match in regex_match:
extracted.append([int(match[0]), int(match[1]), match[2], f'tracker:{tracker_uuid}']) extracted.append([int(match[0]), int(match[1]), match[2], f'tracker:{tracker.uuid}'])
elif tracker_type == 'yara': elif tracker_type == 'yara':
rule = Tracker.get_yara_rule_by_uuid(tracker_uuid) rule = tracker.get_rule()
rule.match(data=content.encode(), callback=_get_yara_match, rule.match(data=content.encode(), callback=_get_yara_match,
which_callbacks=yara.CALLBACK_MATCHES, timeout=30) which_callbacks=yara.CALLBACK_MATCHES, timeout=30)
yara_match = r_cache.smembers(f'extractor:yara:match:{r_key}') yara_match = r_cache.smembers(f'extractor:yara:match:{r_key}')
@ -113,20 +114,20 @@ def get_tracker_match(obj_id, content):
extracted = [] extracted = []
for match in yara_match: for match in yara_match:
start, end, value = match.split(':', 2) start, end, value = match.split(':', 2)
extracted_yara.append([int(start), int(end), value, f'tracker:{tracker_uuid}']) extracted_yara.append([int(start), int(end), value, f'tracker:{tracker.uuid}'])
elif tracker_type == 'word' or tracker_type == 'set': elif tracker_type == 'word' or tracker_type == 'set':
if tracker_type == 'set': if tracker_type == 'set':
tracker = tracker.rsplit(';', 1)[0] tracked = tracked.rsplit(';', 1)[0]
words = tracker.split(',') words = tracked.split(',')
else: else:
words = [tracker] words = [tracked]
for word in words: for word in words:
regex = _get_word_regex(word) regex = _get_word_regex(word)
regex_match = regex_helper.regex_finditer(r_key, regex, obj_id, content) regex_match = regex_helper.regex_finditer(r_key, regex, obj_id, content)
# print(regex_match) # print(regex_match)
for match in regex_match: for match in regex_match:
extracted.append([int(match[0]), int(match[1]), match[2], f'tracker:{tracker_uuid}']) extracted.append([int(match[0]), int(match[1]), match[2], f'tracker:{tracker.uuid}'])
# Convert byte offset to string offset # Convert byte offset to string offset
if extracted_yara: if extracted_yara:

73
bin/lib/objects/CryptoCurrencies.py
View File

@ -2,12 +2,13 @@
# -*-coding:UTF-8 -* # -*-coding:UTF-8 -*
import os import os
import re
import sys import sys
from flask import url_for from flask import url_for
from hashlib import sha256 from hashlib import sha256
from pymisp import MISPObject, MISPAttribute from pymisp import MISPObject
sys.path.append(os.environ['AIL_BIN']) sys.path.append(os.environ['AIL_BIN'])
################################## ##################################
@ -175,31 +176,57 @@ def get_all_cryptocurrencies():
cryptos[subtype] = get_all_cryptocurrencies_by_subtype(subtype) cryptos[subtype] = get_all_cryptocurrencies_by_subtype(subtype)
return cryptos return cryptos
def get_all_cryptocurrencies_by_subtype(subtype): def get_all_cryptocurrencies_by_subtype(subtype):
return get_all_id('cryptocurrency', subtype) return get_all_id('cryptocurrency', subtype)
def sanitize_cryptocurrency_name_to_search(name_to_search, subtype): # TODO FILTER NAME + Key + mail
if subtype == '':
pass
elif subtype == 'name':
pass
elif subtype == 'mail':
pass
return name_to_search
# TODO save object def search_cryptocurrency_by_name(name_to_search, subtype, r_pos=False):
def import_misp_object(misp_obj): cryptocurrencies = {}
""" # for subtype in subtypes:
:type misp_obj: MISPObject r_name = sanitize_cryptocurrency_name_to_search(name_to_search, subtype)
""" if not name_to_search or isinstance(r_name, dict):
obj_id = None # break
obj_subtype = None return cryptocurrencies
for attribute in misp_obj.attributes: r_name = re.compile(r_name)
if attribute.object_relation == 'address': # TODO: handle xmr address field for crypto_name in get_all_cryptocurrencies_by_subtype(subtype):
obj_id = attribute.value res = re.search(r_name, crypto_name)
elif attribute.object_relation == 'symbol': if res:
obj_subtype = get_subtype_by_symbol(attribute.value) cryptocurrencies[crypto_name] = {}
if obj_id and obj_subtype: if r_pos:
obj = CryptoCurrency(obj_id, obj_subtype) cryptocurrencies[crypto_name]['hl-start'] = res.start()
first_seen, last_seen = obj.get_misp_object_first_last_seen(misp_obj) cryptocurrencies[crypto_name]['hl-end'] = res.end()
tags = obj.get_misp_object_tags(misp_obj) return cryptocurrencies
# for tag in tags:
# obj.add_tag()
if __name__ == '__main__': # # TODO save object
res = get_all_cryptocurrencies() # def import_misp_object(misp_obj):
print(res) # """
# :type misp_obj: MISPObject
# """
# obj_id = None
# obj_subtype = None
# for attribute in misp_obj.attributes:
# if attribute.object_relation == 'address': # TODO: handle xmr address field
# obj_id = attribute.value
# elif attribute.object_relation == 'symbol':
# obj_subtype = get_subtype_by_symbol(attribute.value)
# if obj_id and obj_subtype:
# obj = CryptoCurrency(obj_id, obj_subtype)
# first_seen, last_seen = obj.get_misp_object_first_last_seen(misp_obj)
# tags = obj.get_misp_object_tags(misp_obj)
# # for tag in tags:
# # obj.add_tag()
# if __name__ == '__main__':
# name_to_search = '3c'
# subtype = 'bitcoin'
# print(search_cryptocurrency_by_name(name_to_search, subtype))

23
bin/lib/objects/Cves.py
View File

@ -2,9 +2,11 @@
# -*-coding:UTF-8 -* # -*-coding:UTF-8 -*
import os import os
import re
import sys import sys
from flask import url_for from flask import url_for
from pymisp import MISPObject from pymisp import MISPObject
import requests import requests
@ -110,6 +112,25 @@ def get_cves_meta(cves_id, options=set()):
dict_cve[cve_id] = cve.get_meta(options=options) dict_cve[cve_id] = cve.get_meta(options=options)
return dict_cve return dict_cve
def sanitize_cve_name_to_search(name_to_search): # TODO FILTER NAME
return name_to_search
def search_cves_by_name(name_to_search, r_pos=False):
cves = {}
# for subtype in subtypes:
r_name = sanitize_cve_name_to_search(name_to_search)
if not name_to_search or isinstance(r_name, dict):
return cves
r_name = re.compile(r_name)
for cve_name in get_all_cves():
res = re.search(r_name, cve_name)
if res:
cves[cve_name] = {}
if r_pos:
cves[cve_name]['hl-start'] = res.start()
cves[cve_name]['hl-end'] = res.end()
return cves
def api_get_cves_range_by_daterange(date_from, date_to): def api_get_cves_range_by_daterange(date_from, date_to):
cves = [] cves = []
for date in Date.substract_date(date_from, date_to): for date in Date.substract_date(date_from, date_to):
@ -133,3 +154,5 @@ def get_cve_graphline(cve_id):
# if __name__ == '__main__': # if __name__ == '__main__':
# name_to_search = '98'
# print(search_cves_by_name(name_to_search))

36
bin/lib/objects/Decodeds.py
View File

@ -2,6 +2,7 @@
# -*-coding:UTF-8 -* # -*-coding:UTF-8 -*
import os import os
import re
import sys import sys
import magic import magic
import requests import requests
@ -9,6 +10,7 @@ import zipfile
from flask import url_for from flask import url_for
from io import BytesIO from io import BytesIO
from pymisp import MISPObject from pymisp import MISPObject
sys.path.append(os.environ['AIL_BIN']) sys.path.append(os.environ['AIL_BIN'])
@ -114,11 +116,15 @@ class Decoded(AbstractDaterangeObject):
def get_filepath(self, mimetype=None): def get_filepath(self, mimetype=None):
return os.path.join(os.environ['AIL_HOME'], self.get_rel_path(mimetype=mimetype)) return os.path.join(os.environ['AIL_HOME'], self.get_rel_path(mimetype=mimetype))
def get_content(self, mimetype=None): def get_content(self, mimetype=None, r_str=False):
filepath = self.get_filepath(mimetype=mimetype) filepath = self.get_filepath(mimetype=mimetype)
with open(filepath, 'rb') as f: if r_str:
file_content = BytesIO(f.read()) with open(filepath, 'r') as f:
return file_content content = f.read()
else:
with open(filepath, 'rb') as f:
content = BytesIO(f.read())
return content
def get_zip_content(self): def get_zip_content(self):
# mimetype = self.get_estimated_type() # mimetype = self.get_estimated_type()
@ -347,6 +353,25 @@ def sanitise_mimetype(mimetype):
else: else:
return None return None
def sanitize_decoded_name_to_search(name_to_search): # TODO FILTER NAME
return name_to_search
def search_decodeds_by_name(name_to_search, r_pos=False):
decodeds = {}
# for subtype in subtypes:
r_name = sanitize_decoded_name_to_search(name_to_search)
if not name_to_search or isinstance(r_name, dict):
return decodeds
r_name = re.compile(r_name)
for decoded_name in get_all_decodeds():
res = re.search(r_name, decoded_name)
if res:
decodeds[decoded_name] = {}
if r_pos:
decodeds[decoded_name]['hl-start'] = res.start()
decodeds[decoded_name]['hl-end'] = res.end()
return decodeds
############################################################################ ############################################################################
def sanityze_decoder_names(decoder_name): def sanityze_decoder_names(decoder_name):
@ -512,4 +537,7 @@ def get_all_decodeds_files():
decodeds.append(file) decodeds.append(file)
return decodeds return decodeds
# if __name__ == '__main__': # if __name__ == '__main__':
# name_to_search = '4d36'
# print(search_decodeds_by_name(name_to_search))

3
bin/lib/objects/Domains.py
View File

@ -96,6 +96,9 @@ class Domain(AbstractObject):
elif int(last_check) < date: elif int(last_check) < date:
self._set_last_check(date) self._set_last_check(date)
def get_content(self):
return self.id
def get_last_origin(self, obj=False): def get_last_origin(self, obj=False):
origin = {'item': r_crawler.hget(f'domain:meta:{self.id}', 'last_origin')} origin = {'item': r_crawler.hget(f'domain:meta:{self.id}', 'last_origin')}
if obj and origin['item']: if obj and origin['item']:

2
bin/lib/objects/Items.py
View File

@ -79,7 +79,7 @@ class Item(AbstractObject):
else: else:
return filename return filename
def get_content(self, binary=False): def get_content(self, r_str=True, binary=False):
""" """
Returns Item content Returns Item content
""" """

31
bin/lib/objects/Pgps.py
View File

@ -2,6 +2,7 @@
# -*-coding:UTF-8 -* # -*-coding:UTF-8 -*
import os import os
import re
import sys import sys
from flask import url_for from flask import url_for
@ -100,5 +101,35 @@ def get_all_pgps():
def get_all_pgps_by_subtype(subtype): def get_all_pgps_by_subtype(subtype):
return get_all_id('pgp', subtype) return get_all_id('pgp', subtype)
# TODO FILTER NAME + Key + mail
def sanitize_pgp_name_to_search(name_to_search, subtype): # TODO FILTER NAME + Key + mail
if subtype == 'key':
pass
elif subtype == 'name':
pass
elif subtype == 'mail':
pass
return name_to_search
def search_pgps_by_name(name_to_search, subtype, r_pos=False):
pgps = {}
# for subtype in subtypes:
r_name = sanitize_pgp_name_to_search(name_to_search, subtype)
if not name_to_search or isinstance(r_name, dict):
# break
return pgps
r_name = re.compile(r_name)
for pgp_name in get_all_pgps_by_subtype(subtype):
res = re.search(r_name, pgp_name)
if res:
pgps[pgp_name] = {}
if r_pos:
pgps[pgp_name]['hl-start'] = res.start()
pgps[pgp_name]['hl-end'] = res.end()
return pgps
# if __name__ == '__main__': # if __name__ == '__main__':
# name_to_search = 'ex'
# subtype = 'name'
# print(search_pgps_by_name(name_to_search, subtype))

28
bin/lib/objects/Screenshots.py
View File

@ -3,6 +3,7 @@
import base64 import base64
import os import os
import re
import sys import sys
from hashlib import sha256 from hashlib import sha256
@ -71,6 +72,9 @@ class Screenshot(AbstractObject):
file_content = BytesIO(f.read()) file_content = BytesIO(f.read())
return file_content return file_content
def get_content(self):
return self.get_file_content()
def get_misp_object(self): def get_misp_object(self):
obj_attrs = [] obj_attrs = []
obj = MISPObject('file') obj = MISPObject('file')
@ -129,4 +133,26 @@ def create_screenshot(content, size_limit=5000000, b64=True, force=False):
return screenshot return screenshot
return None return None
#if __name__ == '__main__': def sanitize_screenshot_name_to_search(name_to_search): # TODO FILTER NAME
return name_to_search
def search_screenshots_by_name(name_to_search, r_pos=False):
screenshots = {}
# for subtype in subtypes:
r_name = sanitize_screenshot_name_to_search(name_to_search)
if not name_to_search or isinstance(r_name, dict):
return screenshots
r_name = re.compile(r_name)
for screenshot_name in get_all_screenshots():
res = re.search(r_name, screenshot_name)
if res:
screenshots[screenshot_name] = {}
if r_pos:
screenshots[screenshot_name]['hl-start'] = res.start()
screenshots[screenshot_name]['hl-end'] = res.end()
return screenshots
# if __name__ == '__main__':
# name_to_search = '29ba'
# print(search_screenshots_by_name(name_to_search))

41
bin/lib/objects/Usernames.py
View File

@ -3,13 +3,11 @@
import os import os
import sys import sys
import redis import re
from flask import url_for from flask import url_for
from pymisp import MISPObject from pymisp import MISPObject
# sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
sys.path.append(os.environ['AIL_BIN']) sys.path.append(os.environ['AIL_BIN'])
################################## ##################################
# Import Project packages # Import Project packages
@ -61,7 +59,7 @@ class Username(AbstractSubtypeObject):
else: else:
style = 'fas' style = 'fas'
icon = '\uf007' icon = '\uf007'
return {'style': style, 'icon': icon, 'color': '#4dffff', 'radius':5} return {'style': style, 'icon': icon, 'color': '#4dffff', 'radius': 5}
def get_meta(self, options=set()): def get_meta(self, options=set()):
meta = self._get_meta(options=options) meta = self._get_meta(options=options)
@ -74,15 +72,15 @@ class Username(AbstractSubtypeObject):
obj_attrs = [] obj_attrs = []
if self.subtype == 'telegram': if self.subtype == 'telegram':
obj = MISPObject('telegram-account', standalone=True) obj = MISPObject('telegram-account', standalone=True)
obj_attrs.append( obj.add_attribute('username', value=self.id) ) obj_attrs.append(obj.add_attribute('username', value=self.id))
elif self.subtype == 'twitter': elif self.subtype == 'twitter':
obj = MISPObject('twitter-account', standalone=True) obj = MISPObject('twitter-account', standalone=True)
obj_attrs.append( obj.add_attribute('name', value=self.id) ) obj_attrs.append(obj.add_attribute('name', value=self.id))
else: else:
obj = MISPObject('user-account', standalone=True) obj = MISPObject('user-account', standalone=True)
obj_attrs.append( obj.add_attribute('username', value=self.id) ) obj_attrs.append(obj.add_attribute('username', value=self.id))
obj.first_seen = self.get_first_seen() obj.first_seen = self.get_first_seen()
obj.last_seen = self.get_last_seen() obj.last_seen = self.get_last_seen()
@ -107,9 +105,30 @@ def get_all_usernames():
def get_all_usernames_by_subtype(subtype): def get_all_usernames_by_subtype(subtype):
return get_all_id('username', subtype) return get_all_id('username', subtype)
# TODO FILTER NAME + Key + mail
def sanitize_username_name_to_search(name_to_search, subtype): # TODO FILTER NAME
return name_to_search
def search_usernames_by_name(name_to_search, subtype, r_pos=False):
usernames = {}
# for subtype in subtypes:
r_name = sanitize_username_name_to_search(name_to_search, subtype)
if not name_to_search or isinstance(r_name, dict):
# break
return usernames
r_name = re.compile(r_name)
for user_name in get_all_usernames_by_subtype(subtype):
res = re.search(r_name, user_name)
if res:
usernames[user_name] = {}
if r_pos:
usernames[user_name]['hl-start'] = res.start()
usernames[user_name]['hl-end'] = res.end()
return usernames
if __name__ == '__main__': # if __name__ == '__main__':
# name_to_search = 'co'
obj = Username('ninechantw', 'telegram') # subtype = 'telegram'
print(obj.get_misp_object().to_json()) # print(search_usernames_by_name(name_to_search, subtype))

3
bin/lib/objects/abstract_daterange_object.py
View File

@ -108,6 +108,9 @@ class AbstractDaterangeObject(AbstractObject, ABC):
sparkline.append(self.get_nb_seen_by_date(date)) sparkline.append(self.get_nb_seen_by_date(date))
return sparkline return sparkline
def get_content(self):
return self.id
def _add_create(self): def _add_create(self):
r_object.sadd(f'{self.type}:all', self.id) r_object.sadd(f'{self.type}:all', self.id)

20
bin/lib/objects/abstract_object.py
View File

@ -19,9 +19,9 @@ sys.path.append(os.environ['AIL_BIN'])
################################## ##################################
from lib import Tag from lib import Tag
from lib import Duplicate from lib import Duplicate
from lib.correlations_engine import get_nb_correlations, get_correlations, add_obj_correlation, delete_obj_correlation, exists_obj_correlation, is_obj_correlated, get_nb_correlation_by_correl_type from lib.correlations_engine import get_nb_correlations, get_correlations, add_obj_correlation, delete_obj_correlation, delete_obj_correlations, exists_obj_correlation, is_obj_correlated, get_nb_correlation_by_correl_type
from lib.Investigations import is_object_investigated, get_obj_investigations, delete_obj_investigations from lib.Investigations import is_object_investigated, get_obj_investigations, delete_obj_investigations
from lib.Tracker import is_obj_tracked, get_obj_all_trackers, delete_obj_trackers from lib.Tracker import is_obj_tracked, get_obj_trackers, delete_obj_trackers
class AbstractObject(ABC): class AbstractObject(ABC):
@ -85,6 +85,13 @@ class AbstractObject(ABC):
#- Tags -# #- Tags -#
@abstractmethod
def get_content(self):
"""
Get Object Content
"""
pass
## Duplicates ## ## Duplicates ##
def get_duplicates(self): def get_duplicates(self):
return Duplicate.get_obj_duplicates(self.type, self.get_subtype(r_str=True), self.id) return Duplicate.get_obj_duplicates(self.type, self.get_subtype(r_str=True), self.id)
@ -125,7 +132,7 @@ class AbstractObject(ABC):
return is_obj_tracked(self.type, self.subtype, self.id) return is_obj_tracked(self.type, self.subtype, self.id)
def get_trackers(self): def get_trackers(self):
return get_obj_all_trackers(self.type, self.subtype, self.id) return get_obj_trackers(self.type, self.subtype, self.id)
def delete_trackers(self): def delete_trackers(self):
return delete_obj_trackers(self.type, self.subtype, self.id) return delete_obj_trackers(self.type, self.subtype, self.id)
@ -134,13 +141,14 @@ class AbstractObject(ABC):
def _delete(self): def _delete(self):
# DELETE TAGS # DELETE TAGS
Tag.delete_obj_all_tags(self.id, self.type) # ########### # TODO: # TODO: # FIXME: Tag.delete_object_tags(self.type, self.get_subtype(r_str=True), self.id)
# remove from tracker # remove from tracker
self.delete_trackers() self.delete_trackers()
# remove from retro hunt currently item only TODO
# remove from investigations # remove from investigations
self.delete_investigations() self.delete_investigations()
# Delete Correlations
# # TODO: remove from correlation delete_obj_correlations(self.type, self.get_subtype(r_str=True), self.id)
@abstractmethod @abstractmethod
def delete(self): def delete(self):

3
bin/lib/objects/abstract_subtype_object.py
View File

@ -116,6 +116,9 @@ class AbstractSubtypeObject(AbstractObject, ABC):
if date > last_seen: if date > last_seen:
self.set_last_seen(date) self.set_last_seen(date)
def get_content(self):
return self.id
def get_sparkline(self): def get_sparkline(self):
sparkline = [] sparkline = []
for date in Date.get_previous_date_list(6): for date in Date.get_previous_date_list(6):

38
bin/lib/objects/ail_objects.py
View File

@ -138,9 +138,14 @@ def get_object_meta(obj_type, subtype, id, options=set(), flask_context=False):
def get_objects_meta(objs, options=set(), flask_context=False): def get_objects_meta(objs, options=set(), flask_context=False):
metas = [] metas = []
for obj_dict in objs: for obj in objs:
metas.append(get_object_meta(obj_dict['type'], obj_dict['subtype'], obj_dict['id'], options=options, if isinstance(obj, dict):
flask_context=flask_context)) obj_type = obj['type']
subtype = obj['subtype']
obj_id = obj['id']
else:
obj_type, subtype, obj_id = obj.split(':', 2)
metas.append(get_object_meta(obj_type, subtype, obj_id, options=options, flask_context=flask_context))
return metas return metas
@ -162,7 +167,25 @@ def get_object_card_meta(obj_type, subtype, id, related_btc=False):
return meta return meta
def get_ui_obj_tag_table_keys(obj_type): #### OBJ FILTERS ####
def is_filtered(obj, filters):
if 'mimetypes' in filters:
mimetype = obj.get_mimetype()
if mimetype not in filters['mimetypes']:
return True
if 'sources' in filters:
obj_source = obj.get_source()
if obj_source not in filters['sources']:
return True
if 'subtypes' in filters:
subtype = obj.get_subtype(r_str=True)
if subtype not in filters['subtypes']:
return True
return False
def get_ui_obj_tag_table_keys(obj_type): # TODO REMOVE ME
""" """
Warning: use only in flask (dynamic templates) Warning: use only in flask (dynamic templates)
""" """
@ -170,6 +193,7 @@ def get_ui_obj_tag_table_keys(obj_type):
return ['id', 'first_seen', 'last_check', 'status'] # # TODO: add root screenshot return ['id', 'first_seen', 'last_check', 'status'] # # TODO: add root screenshot
# # # # MISP OBJECTS # # # # # # # # MISP OBJECTS # # # #
# # TODO: CHECK IF object already have an UUID # # TODO: CHECK IF object already have an UUID
@ -265,6 +289,7 @@ def api_sanitize_object_type(obj_type):
if not is_valid_object_type(obj_type): if not is_valid_object_type(obj_type):
return {'status': 'error', 'reason': 'Incorrect object type'}, 400 return {'status': 'error', 'reason': 'Incorrect object type'}, 400
#### CORRELATION ####
def get_obj_correlations(obj_type, subtype, obj_id): def get_obj_correlations(obj_type, subtype, obj_id):
obj = get_object(obj_type, subtype, obj_id) obj = get_object(obj_type, subtype, obj_id)
@ -292,13 +317,14 @@ def get_obj_correlations_objs(obj_type, subtype, obj_id, filter_types=[], lvl=0,
def obj_correlations_objs_add_tags(obj_type, subtype, obj_id, tags, filter_types=[], lvl=0, nb_max=300): def obj_correlations_objs_add_tags(obj_type, subtype, obj_id, tags, filter_types=[], lvl=0, nb_max=300):
objs = get_obj_correlations_objs(obj_type, subtype, obj_id, filter_types=filter_types, lvl=lvl, nb_max=nb_max) objs = get_obj_correlations_objs(obj_type, subtype, obj_id, filter_types=filter_types, lvl=lvl, nb_max=nb_max)
# print(objs)
for obj_tuple in objs: for obj_tuple in objs:
obj1_type, subtype1, id1 = obj_tuple obj1_type, subtype1, id1 = obj_tuple
add_obj_tags(obj1_type, subtype1, id1, tags) add_obj_tags(obj1_type, subtype1, id1, tags)
return objs return objs
################################################################################ ################################################################################
################################################################################ ################################################################################ TODO
################################################################################ ################################################################################
def delete_obj_correlations(obj_type, subtype, obj_id): def delete_obj_correlations(obj_type, subtype, obj_id):
@ -357,7 +383,7 @@ def get_correlations_graph_node(obj_type, subtype, obj_id, filter_types=[], max_
"links": create_correlation_graph_links(links)} "links": create_correlation_graph_links(links)}
############### # --- CORRELATION --- #
# if __name__ == '__main__': # if __name__ == '__main__':

18
bin/modules/Decoder.py
View File

@ -23,6 +23,9 @@ from modules.abstract_module import AbstractModule
from lib.ConfigLoader import ConfigLoader from lib.ConfigLoader import ConfigLoader
from lib.objects.Items import Item from lib.objects.Items import Item
from lib.objects.Decodeds import Decoded from lib.objects.Decodeds import Decoded
from trackers.Tracker_Term import Tracker_Term
from trackers.Tracker_Regex import Tracker_Regex
from trackers.Tracker_Yara import Tracker_Yara
config_loader = ConfigLoader() config_loader = ConfigLoader()
hex_max_execution_time = config_loader.get_config_int("Decoder", "max_execution_time_hexadecimal") hex_max_execution_time = config_loader.get_config_int("Decoder", "max_execution_time_hexadecimal")
@ -76,6 +79,10 @@ class Decoder(AbstractModule):
# Waiting time in seconds between to message processed # Waiting time in seconds between to message processed
self.pending_seconds = 1 self.pending_seconds = 1
self.tracker_term = Tracker_Term(queue=False)
self.tracker_regex = Tracker_Regex(queue=False)
self.tracker_yara = Tracker_Yara(queue=False)
# Send module state to logs # Send module state to logs
self.redis_logger.info(f'Module {self.module_name} initialized') self.redis_logger.info(f'Module {self.module_name} initialized')
@ -84,6 +91,7 @@ class Decoder(AbstractModule):
item = Item(message) item = Item(message)
content = item.get_content() content = item.get_content()
date = item.get_date() date = item.get_date()
new_decodeds = []
for decoder in self.decoder_order: for decoder in self.decoder_order:
find = False find = False
@ -96,8 +104,8 @@ class Decoder(AbstractModule):
encodeds = set(encodeds) encodeds = set(encodeds)
for encoded in encodeds: for encoded in encodeds:
find = False
if len(encoded) >= decoder['encoded_min_size']: if len(encoded) >= decoder['encoded_min_size']:
find = True
decoded_file = self.decoder_function[dname](encoded) decoded_file = self.decoder_function[dname](encoded)
sha1_string = sha1(decoded_file).hexdigest() sha1_string = sha1(decoded_file).hexdigest()
@ -109,6 +117,7 @@ class Decoder(AbstractModule):
print(sha1_string, item.id) print(sha1_string, item.id)
raise Exception(f'Invalid mimetype: {decoded.id} {item.id}') raise Exception(f'Invalid mimetype: {decoded.id} {item.id}')
decoded.save_file(decoded_file, mimetype) decoded.save_file(decoded_file, mimetype)
new_decodeds.append(decoded.id)
else: else:
mimetype = decoded.get_mimetype() mimetype = decoded.get_mimetype()
decoded.add(dname, date, item.id, mimetype=mimetype) decoded.add(dname, date, item.id, mimetype=mimetype)
@ -125,6 +134,13 @@ class Decoder(AbstractModule):
msg = f'infoleak:automatic-detection="{dname}";{item.id}' msg = f'infoleak:automatic-detection="{dname}";{item.id}'
self.add_message_to_queue(msg, 'Tags') self.add_message_to_queue(msg, 'Tags')
####################
# TRACKERS DECODED
for decoded_id in new_decodeds:
self.tracker_term.compute(decoded_id, obj_type='decoded')
self.tracker_regex.compute(decoded_id, obj_type='decoded')
self.tracker_yara.compute(decoded_id, obj_type='decoded')
if __name__ == '__main__': if __name__ == '__main__':
module = Decoder() module = Decoder()

12
bin/modules/PgpDump.py
View File

@ -216,16 +216,16 @@ class PgpDump(AbstractModule):
pgp = Pgps.Pgp(name, 'name') pgp = Pgps.Pgp(name, 'name')
pgp.add(date, self.item_id) pgp.add(date, self.item_id)
print(f' name: {name}') print(f' name: {name}')
self.tracker_term.compute(self.item_id, item_content=name) self.tracker_term.compute(name, obj_type='pgp', subtype='name')
self.tracker_regex.compute(self.item_id, content=name) self.tracker_regex.compute(name, obj_type='pgp', subtype='name')
self.tracker_yara.compute(self.item_id, item_content=name) self.tracker_yara.compute(name, obj_type='pgp', subtype='name')
for mail in self.mails: for mail in self.mails:
pgp = Pgps.Pgp(mail, 'mail') pgp = Pgps.Pgp(mail, 'mail')
pgp.add(date, self.item_id) pgp.add(date, self.item_id)
print(f' mail: {mail}') print(f' mail: {mail}')
self.tracker_term.compute(self.item_id, item_content=mail) self.tracker_term.compute(mail, obj_type='pgp', subtype='mail')
self.tracker_regex.compute(self.item_id, content=mail) self.tracker_regex.compute(mail, obj_type='pgp', subtype='mail')
self.tracker_yara.compute(self.item_id, item_content=mail) self.tracker_yara.compute(mail, obj_type='pgp', subtype='mail')
# Keys extracted from PGP PRIVATE KEY BLOCK # Keys extracted from PGP PRIVATE KEY BLOCK
for key in self.private_keys: for key in self.private_keys:

532
bin/packages/Term.py
View File

@ -1,532 +0,0 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
import os
import re
import sys
import time
import uuid
import datetime
from collections import defaultdict
from flask import escape
from nltk.tokenize import RegexpTokenizer
from textblob import TextBlob
sys.path.append(os.environ['AIL_BIN'])
##################################
# Import Project packages
##################################
from lib import ConfigLoader
from lib import Tracker
from packages import Date
from lib.objects import Items
config_loader = ConfigLoader.ConfigLoader()
r_serv_term = config_loader.get_db_conn("Kvrocks_Trackers")
config_loader = None
email_regex = r'[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,6}'
email_regex = re.compile(email_regex)
special_characters = set('[<>~!?@#$%^&*|()_-+={}":;,.\'\n\r\t]/\\')
special_characters.add('\\s')
# NLTK tokenizer
tokenizer = RegexpTokenizer('[\&\~\:\;\,\.\(\)\{\}\|\[\]\\\\/\-/\=\'\"\%\$\?\@\+\#\_\^\<\>\!\*\n\r\t\s]+',
gaps=True, discard_empty=True)
def is_valid_uuid_v4(UUID):
if not UUID:
return False
UUID = UUID.replace('-', '')
try:
uuid_test = uuid.UUID(hex=UUID, version=4)
return uuid_test.hex == UUID
except:
return False
# # TODO: use new package => duplicate fct
def is_in_role(user_id, role):
if r_serv_term.sismember('user_role:{}'.format(role), user_id):
return True
else:
return False
def check_term_uuid_valid_access(term_uuid, user_id):
if not is_valid_uuid_v4(term_uuid):
return {"status": "error", "reason": "Invalid uuid"}, 400
level = r_serv_term.hget('tracker:{}'.format(term_uuid), 'level')
if not level:
return {"status": "error", "reason": "Unknown uuid"}, 404
if level == 0:
if r_serv_term.hget('tracker:{}'.format(term_uuid), 'user_id') != user_id:
if not is_in_role(user_id, 'admin'):
return {"status": "error", "reason": "Unknown uuid"}, 404
return None
def is_valid_mail(email):
result = email_regex.match(email)
if result:
return True
else:
return False
def verify_mail_list(mail_list):
for mail in mail_list:
if not is_valid_mail(mail):
return {'status': 'error', 'reason': 'Invalid email', 'value': mail}, 400
return None
def is_valid_regex(term_regex):
try:
re.compile(term_regex)
return True
except:
return False
def get_text_word_frequency(item_content, filtering=True):
item_content = item_content.lower()
words_dict = defaultdict(int)
if filtering:
blob = TextBlob(item_content, tokenizer=tokenizer)
else:
blob = TextBlob(item_content)
for word in blob.tokens:
words_dict[word] += 1
return words_dict
# # TODO: create all tracked words
def get_tracked_words_list():
return list(r_serv_term.smembers('all:tracker:word'))
def get_set_tracked_words_list():
set_list = r_serv_term.smembers('all:tracker:set')
all_set_list = []
for elem in set_list:
res = elem.split(';')
num_words = int(res[1])
ter_set = res[0].split(',')
all_set_list.append((ter_set, num_words, elem))
return all_set_list
def get_regex_tracked_words_dict():
regex_list = r_serv_term.smembers('all:tracker:regex')
dict_tracked_regex = {}
for regex in regex_list:
dict_tracked_regex[regex] = re.compile(regex)
return dict_tracked_regex
def get_tracked_term_list_item(term_uuid, date_from, date_to):
all_item_id = []
if date_from and date_to:
for date in r_serv_term.zrangebyscore('tracker:stat:{}'.format(term_uuid), int(date_from), int(date_to)):
all_item_id = all_item_id + list(r_serv_term.smembers('tracker:item:{}:{}'.format(term_uuid, date)))
return all_item_id
def is_term_tracked_in_global_level(term, term_type):
res = r_serv_term.smembers('all:tracker_uuid:{}:{}'.format(term_type, term))
if res:
for elem_uuid in res:
if r_serv_term.hget('tracker:{}'.format(elem_uuid), 'level') == '1':
return True
return False
def is_term_tracked_in_user_level(term, term_type, user_id):
res = r_serv_term.smembers('user:tracker:{}'.format(user_id))
if res:
for elem_uuid in res:
if r_serv_term.hget('tracker:{}'.format(elem_uuid), 'tracked') == term:
if r_serv_term.hget('tracker:{}'.format(elem_uuid), 'type') == term_type:
return True
return False
def parse_json_term_to_add(dict_input, user_id):
term = dict_input.get('term', None)
if not term:
return {"status": "error", "reason": "Term not provided"}, 400
term_type = dict_input.get('type', None)
if not term_type:
return {"status": "error", "reason": "Term type not provided"}, 400
nb_words = dict_input.get('nb_words', 1)
description = dict_input.get('description', '')
description = escape(description)
webhook = dict_input.get('webhook', '')
webhook = escape(webhook)
res = parse_tracked_term_to_add(term, term_type, nb_words=nb_words)
if res[1] != 200:
return res
term = res[0]['term']
term_type = res[0]['type']
tags = dict_input.get('tags', [])
mails = dict_input.get('mails', [])
res = verify_mail_list(mails)
if res:
return res
level = dict_input.get('level', 1)
try:
level = int(level)
if level not in range(0, 1):
level = 1
except:
level = 1
# check if term already tracked in global
if level == 1:
if is_term_tracked_in_global_level(term, term_type):
return {"status": "error", "reason": "Term already tracked"}, 409
else:
if is_term_tracked_in_user_level(term, term_type, user_id):
return {"status": "error", "reason": "Term already tracked"}, 409
term_uuid = add_tracked_term(term, term_type, user_id, level, tags, mails, description, webhook)
return {'term': term, 'type': term_type, 'uuid': term_uuid}, 200
def parse_tracked_term_to_add(term, term_type, nb_words=1):
if term_type == 'regex':
if not is_valid_regex(term):
return {"status": "error", "reason": "Invalid regex"}, 400
elif term_type == 'word' or term_type == 'set':
# force lowercase
term = term.lower()
word_set = set(term)
set_inter = word_set.intersection(special_characters)
if set_inter:
return {"status": "error", "reason": "special character not allowed", "message": "Please use a regex or remove all special characters"}, 400
words = term.split()
# not a word
if term_type == 'word' and len(words) > 1:
term_type = 'set'
# output format: term1,term2,term3;2
if term_type == 'set':
try:
nb_words = int(nb_words)
except:
nb_words = 1
if nb_words == 0:
nb_words = 1
words_set = set(words)
words_set = sorted(words_set)
if nb_words > len(words_set):
nb_words = len(words_set)
term = ",".join(words_set)
term = "{};{}".format(term, nb_words)
elif term_type == 'yara_custom':
if not Tracker.is_valid_yara_rule(term):
return {"status": "error", "reason": "Invalid custom Yara Rule"}, 400
elif term_type == 'yara_default':
if not Tracker.is_valid_default_yara_rule(term):
return {"status": "error", "reason": "The Yara Rule doesn't exist"}, 400
else:
return {"status": "error", "reason": "Incorrect type"}, 400
return {"status": "success", "term": term, "type": term_type}, 200
def add_tracked_term(term, term_type, user_id, level, tags, mails, description, webhook, dashboard=0):
term_uuid = str(uuid.uuid4())
# YARA
if term_type == 'yara_custom' or term_type == 'yara_default':
term = Tracker.save_yara_rule(term_type, term, tracker_uuid=term_uuid)
term_type = 'yara'
# create metadata
r_serv_term.hset('tracker:{}'.format(term_uuid), 'tracked', term) # # TODO: use hash
r_serv_term.hset('tracker:{}'.format(term_uuid), 'type', term_type)
r_serv_term.hset('tracker:{}'.format(term_uuid), 'date', datetime.date.today().strftime("%Y%m%d"))
r_serv_term.hset('tracker:{}'.format(term_uuid), 'user_id', user_id)
r_serv_term.hset('tracker:{}'.format(term_uuid), 'level', level)
r_serv_term.hset('tracker:{}'.format(term_uuid), 'dashboard', dashboard)
if description:
r_serv_term.hset('tracker:{}'.format(term_uuid), 'description', description)
if webhook:
r_serv_term.hset('tracker:{}'.format(term_uuid), 'webhook', webhook)
# create all term set
r_serv_term.sadd('all:tracker:{}'.format(term_type), term)
# create term - uuid map
r_serv_term.sadd('all:tracker_uuid:{}:{}'.format(term_type, term), term_uuid)
# add display level set
if level == 0: # user only
r_serv_term.sadd('user:tracker:{}'.format(user_id), term_uuid)
r_serv_term.sadd('user:tracker:{}:{}'.format(user_id, term_type), term_uuid)
elif level == 1: # global
r_serv_term.sadd('global:tracker', term_uuid)
r_serv_term.sadd('global:tracker:{}'.format(term_type), term_uuid)
# create term tags list
for tag in tags:
r_serv_term.sadd('tracker:tags:{}'.format(term_uuid), escape(tag))
# create term tags mail notification list
for mail in mails:
r_serv_term.sadd('tracker:mail:{}'.format(term_uuid), escape(mail))
# toggle refresh module tracker list/set
r_serv_term.set('tracker:refresh:{}'.format(term_type), time.time())
return term_uuid
def parse_tracked_term_to_delete(dict_input, user_id):
term_uuid = dict_input.get("uuid", None)
res = check_term_uuid_valid_access(term_uuid, user_id)
if res:
return res
delete_term(term_uuid)
return {"uuid": term_uuid}, 200
# # TODO: MOVE IN TRACKER
def delete_term(term_uuid):
term = r_serv_term.hget('tracker:{}'.format(term_uuid), 'tracked')
term_type = r_serv_term.hget('tracker:{}'.format(term_uuid), 'type')
level = r_serv_term.hget('tracker:{}'.format(term_uuid), 'level')
r_serv_term.srem('all:tracker_uuid:{}:{}'.format(term_type, term), term_uuid)
r_serv_term.srem(f'trackers:all', term_uuid)
r_serv_term.srem(f'trackers:all:{term_type}', term_uuid)
# Term not tracked by other users
if not r_serv_term.exists('all:tracker_uuid:{}:{}'.format(term_type, term)):
r_serv_term.srem('all:tracker:{}'.format(term_type), term)
# toggle refresh module tracker list/set
r_serv_term.set('tracker:refresh:{}'.format(term_type), time.time())
if level == '0': # user only
user_id = term_type = r_serv_term.hget('tracker:{}'.format(term_uuid), 'user_id')
r_serv_term.srem('user:tracker:{}'.format(user_id), term_uuid)
r_serv_term.srem('user:tracker:{}:{}'.format(user_id, term_type), term_uuid)
elif level == '1': # global
r_serv_term.srem('global:tracker', term_uuid)
r_serv_term.srem('global:tracker:{}'.format(term_type), term_uuid)
# delete metatadata
r_serv_term.delete('tracker:{}'.format(term_uuid))
# remove tags
r_serv_term.delete('tracker:tags:{}'.format(term_uuid))
# remove mails
r_serv_term.delete('tracker:mail:{}'.format(term_uuid))
# remove sources
r_serv_term.delete('tracker:sources:{}'.format(term_uuid))
# remove item set
all_item_date = r_serv_term.zrange(f'tracker:stat:{term_uuid}', 0, -1, withscores=True)
if all_item_date:
all_item_date = dict(all_item_date)
for date in all_item_date:
for item_id in r_serv_term.smembers(f'tracker:item:{term_uuid}:{date}'):
r_serv_term.srem(f'obj:trackers:item:{item_id}', term_uuid)
r_serv_term.delete(f'tracker:item:{term_uuid}:{date}')
r_serv_term.delete('tracker:stat:{}'.format(term_uuid))
if term_type == 'yara':
# delete custom rule
if not Tracker.is_default_yara_rule(term):
filepath = Tracker.get_yara_rule_file_by_tracker_name(term)
if filepath:
os.remove(filepath)
def replace_tracker_description(term_uuid, description):
description = escape(description)
r_serv_term.hset('tracker:{}'.format(term_uuid), 'description', description)
def replace_tracked_term_tags(term_uuid, tags):
r_serv_term.delete('tracker:tags:{}'.format(term_uuid))
for tag in tags:
tag = escape(tag)
r_serv_term.sadd('tracker:tags:{}'.format(term_uuid), tag)
def replace_tracked_term_mails(term_uuid, mails):
res = verify_mail_list(mails)
if res:
return res
else:
r_serv_term.delete('tracker:mail:{}'.format(term_uuid))
for mail in mails:
mail = escape(mail)
r_serv_term.sadd('tracker:mail:{}'.format(term_uuid), mail)
def get_term_uuid_list(term, term_type):
return list(r_serv_term.smembers('all:tracker_uuid:{}:{}'.format(term_type, term)))
def get_term_tags(term_uuid):
return list(r_serv_term.smembers('tracker:tags:{}'.format(term_uuid)))
def get_term_mails(term_uuid):
return list(r_serv_term.smembers('tracker:mail:{}'.format(term_uuid)))
def get_term_webhook(term_uuid):
return r_serv_term.hget('tracker:{}'.format(term_uuid), "webhook")
def add_tracked_item(term_uuid, item_id, item_date):
# track item
r_serv_term.sadd('tracker:item:{}:{}'.format(term_uuid, item_date), item_id)
# track nb item by date
r_serv_term.zadd('tracker:stat:{}'.format(term_uuid), {item_date: item_date})
def create_token_statistics(item_date, word, nb):
r_serv_term.zincrby('stat_token_per_item_by_day:{}'.format(item_date), 1, word)
r_serv_term.zincrby('stat_token_total_by_day:{}'.format(item_date), nb, word)
r_serv_term.sadd('stat_token_history', item_date)
def delete_token_statistics_by_date(item_date):
r_serv_term.delete('stat_token_per_item_by_day:{}'.format(item_date))
r_serv_term.delete('stat_token_total_by_day:{}'.format(item_date))
r_serv_term.srem('stat_token_history', item_date)
def get_all_token_stat_history():
return r_serv_term.smembers('stat_token_history')
def get_tracked_term_last_updated_by_type(term_type):
epoch_update = r_serv_term.get('tracker:refresh:{}'.format(term_type))
if not epoch_update:
epoch_update = 0
return float(epoch_update)
def parse_get_tracker_term_item(dict_input, user_id):
term_uuid = dict_input.get('uuid', None)
res = check_term_uuid_valid_access(term_uuid, user_id)
if res:
return res
date_from = dict_input.get('date_from', None)
date_to = dict_input.get('date_to', None)
if date_from is None:
date_from = get_tracked_term_first_seen(term_uuid)
if date_from:
date_from = date_from[0]
if date_to is None:
date_to = date_from
if date_from > date_to:
date_from = date_to
all_item_id = Tracker.get_tracker_items_by_daterange(term_uuid, date_from, date_to)
all_item_id = Items.get_item_list_desc(all_item_id)
res_dict = {'uuid': term_uuid, 'date_from': date_from, 'date_to': date_to, 'items': all_item_id}
return res_dict, 200
def get_tracked_term_first_seen(term_uuid):
return Tracker.get_tracker_first_seen(term_uuid)
def get_tracked_term_last_seen(term_uuid):
return Tracker.get_tracker_last_seen(term_uuid)
def get_term_metedata(term_uuid, user_id=False, description=False, level=False, tags=False, mails=False, sparkline=False):
dict_uuid = {}
dict_uuid['term'] = r_serv_term.hget('tracker:{}'.format(term_uuid), 'tracked')
dict_uuid['type'] = r_serv_term.hget('tracker:{}'.format(term_uuid), 'type')
dict_uuid['date'] = r_serv_term.hget('tracker:{}'.format(term_uuid), 'date')
dict_uuid['first_seen'] = get_tracked_term_first_seen(term_uuid)
dict_uuid['last_seen'] = get_tracked_term_last_seen(term_uuid)
if user_id:
dict_uuid['user_id'] = r_serv_term.hget('tracker:{}'.format(term_uuid), 'user_id')
if level:
dict_uuid['level'] = r_serv_term.hget('tracker:{}'.format(term_uuid), 'level')
if mails:
dict_uuid['mails'] = get_list_trackeed_term_mails(term_uuid)
if tags:
dict_uuid['tags'] = get_list_trackeed_term_tags(term_uuid)
if sparkline:
dict_uuid['sparkline'] = get_tracked_term_sparkline(term_uuid)
if description:
dict_uuid['description'] = r_serv_term.hget('tracker:{}'.format(term_uuid), 'description')
dict_uuid['uuid'] = term_uuid
return dict_uuid
def get_tracked_term_sparkline(tracker_uuid, num_day=6):
date_range_sparkline = Date.get_date_range(num_day)
sparklines_value = []
for date_day in date_range_sparkline:
nb_seen_this_day = r_serv_term.scard('tracker:item:{}:{}'.format(tracker_uuid, date_day))
if nb_seen_this_day is None:
nb_seen_this_day = 0
sparklines_value.append(int(nb_seen_this_day))
return sparklines_value
def get_list_tracked_term_stats_by_day(list_tracker_uuid, num_day=31, date_from=None, date_to=None):
if date_from and date_to:
date_range = Date.substract_date(date_from, date_to)
else:
date_range = Date.get_date_range(num_day)
list_tracker_stats = []
for tracker_uuid in list_tracker_uuid:
dict_tracker_data = []
tracker = r_serv_term.hget('tracker:{}'.format(tracker_uuid), 'tracked')
for date_day in date_range:
nb_seen_this_day = r_serv_term.scard('tracker:item:{}:{}'.format(tracker_uuid, date_day))
if nb_seen_this_day is None:
nb_seen_this_day = 0
dict_tracker_data.append({"date": date_day, "value": int(nb_seen_this_day)})
list_tracker_stats.append({"name": tracker, "Data": dict_tracker_data})
return list_tracker_stats
def get_list_trackeed_term_tags(term_uuid):
res = r_serv_term.smembers('tracker:tags:{}'.format(term_uuid))
if res:
return list(res)
else:
return []
def get_list_trackeed_term_mails(term_uuid):
res = r_serv_term.smembers('tracker:mail:{}'.format(term_uuid))
if res:
return list(res)
else:
return []
def get_user_tracked_term_uuid(user_id, filter_type=None):
if filter_type:
return list(r_serv_term.smembers('user:tracker:{}:{}'.format(user_id, filter_type)))
else:
return list(r_serv_term.smembers('user:tracker:{}'.format(user_id)))
def get_global_tracked_term_uuid(filter_type=None):
if filter_type:
return list(r_serv_term.smembers('global:tracker:{}'.format(filter_type)))
else:
return list(r_serv_term.smembers('global:tracker'))
def get_all_user_tracked_terms(user_id, filter_type=None):
all_user_term = []
all_user_term_uuid = get_user_tracked_term_uuid(user_id, filter_type=filter_type)
for term_uuid in all_user_term_uuid:
all_user_term.append(get_term_metedata(term_uuid, tags=True, mails=True, sparkline=True))
return all_user_term
def get_all_global_tracked_terms(filter_type=None):
all_user_term = []
all_user_term_uuid = get_global_tracked_term_uuid(filter_type=filter_type)
for term_uuid in all_user_term_uuid:
all_user_term.append(get_term_metedata(term_uuid, user_id=True, tags=True, mails=True, sparkline=True))
return all_user_term

2
bin/packages/lib_words.py
View File

@ -3,8 +3,6 @@
import os import os
import string import string
from pubsublogger import publisher
import calendar import calendar
from datetime import date from datetime import date
from dateutil.rrule import rrule, DAILY from dateutil.rrule import rrule, DAILY

30
bin/trackers/Retro_Hunt.py
View File

@ -61,15 +61,17 @@ class Retro_Hunt(AbstractModule):
self.progress = 0 self.progress = 0
# First launch # First launch
# restart # restart
rule = Tracker.get_retro_hunt_task_rule(task_uuid, r_compile=True) retro_hunt = Tracker.RetroHunt(task_uuid) # TODO SELF
timeout = Tracker.get_retro_hunt_task_timeout(task_uuid) rule = retro_hunt.get_rule(r_compile=True)
timeout = retro_hunt.get_timeout()
self.redis_logger.debug(f'{self.module_name}, Retro Hunt rule {task_uuid} timeout {timeout}') self.redis_logger.debug(f'{self.module_name}, Retro Hunt rule {task_uuid} timeout {timeout}')
sources = Tracker.get_retro_hunt_task_sources(task_uuid, r_sort=True) sources = retro_hunt.get_sources(r_sort=True)
self.date_from = Tracker.get_retro_hunt_task_date_from(task_uuid) self.date_from = retro_hunt.get_date_from()
self.date_to = Tracker.get_retro_hunt_task_date_to(task_uuid) self.date_to = retro_hunt.get_date_to()
self.tags = Tracker.get_retro_hunt_task_tags(task_uuid) self.tags = retro_hunt.get_tags()
curr_date = Tracker.get_retro_hunt_task_current_date(task_uuid) curr_date = Tracker.get_retro_hunt_task_current_date(task_uuid)
self.nb_src_done = Tracker.get_retro_hunt_task_nb_src_done(task_uuid, sources=sources) self.nb_src_done = Tracker.get_retro_hunt_task_nb_src_done(task_uuid, sources=sources)
self.update_progress(sources, curr_date) self.update_progress(sources, curr_date)
@ -106,11 +108,10 @@ class Retro_Hunt(AbstractModule):
# PAUSE # PAUSE
self.update_progress(sources, curr_date) self.update_progress(sources, curr_date)
if Tracker.check_retro_hunt_pause(task_uuid): if retro_hunt.to_pause():
Tracker.set_retro_hunt_last_analyzed(task_uuid, id) Tracker.set_retro_hunt_last_analyzed(task_uuid, id)
# self.update_progress(sources, curr_date, save_db=True) # self.update_progress(sources, curr_date, save_db=True)
Tracker.pause_retro_hunt_task(task_uuid) retro_hunt.pause()
Tracker.clear_retro_hunt_task_cache(task_uuid)
return None return None
self.nb_src_done += 1 self.nb_src_done += 1
@ -120,9 +121,7 @@ class Retro_Hunt(AbstractModule):
self.update_progress(sources, curr_date) self.update_progress(sources, curr_date)
Tracker.set_retro_hunt_task_state(task_uuid, 'completed') retro_hunt.complete()
Tracker.set_retro_hunt_nb_match(task_uuid)
Tracker.clear_retro_hunt_task_cache(task_uuid)
print(f'Retro Hunt {task_uuid} completed') print(f'Retro Hunt {task_uuid} completed')
self.redis_logger.warning(f'{self.module_name}, Retro Hunt {task_uuid} completed') self.redis_logger.warning(f'{self.module_name}, Retro Hunt {task_uuid} completed')
@ -130,10 +129,11 @@ class Retro_Hunt(AbstractModule):
# # TODO: stop # # TODO: stop
def update_progress(self, sources, curr_date, save_db=False): def update_progress(self, sources, curr_date, save_db=False):
progress = Tracker.compute_retro_hunt_task_progress(self.task_uuid, date_from=self.date_from, date_to=self.date_to, retro_hunt = Tracker.RetroHunt(retro_hubt) # TODO USE SELF
sources=sources, curr_date=curr_date, nb_src_done=self.nb_src_done) progress = retro_hunt.compute_progress(date_from=self.date_from, date_to=self.date_to,
sources=sources, curr_date=curr_date, nb_src_done=self.nb_src_done)
if self.progress != progress: if self.progress != progress:
Tracker.set_cache_retro_hunt_task_progress(self.task_uuid, progress) retro_hunt.set_progress(progress)
self.progress = progress self.progress = progress
# if save_db: # if save_db:
# Tracker.set_retro_hunt_task_progress(task_uuid, progress) # Tracker.set_retro_hunt_task_progress(task_uuid, progress)

70
bin/trackers/Tracker_Regex.py
View File

@ -17,9 +17,8 @@ sys.path.append(os.environ['AIL_BIN'])
# Import Project packages # Import Project packages
################################## ##################################
from modules.abstract_module import AbstractModule from modules.abstract_module import AbstractModule
from lib.objects.Items import Item from lib.objects import ail_objects
from lib.ConfigLoader import ConfigLoader from lib.ConfigLoader import ConfigLoader
from packages import Term
from lib import Tracker from lib import Tracker
from exporter.MailExporter import MailExporterTracker from exporter.MailExporter import MailExporterTracker
@ -39,7 +38,7 @@ class Tracker_Regex(AbstractModule):
self.max_execution_time = config_loader.get_config_int(self.module_name, "max_execution_time") self.max_execution_time = config_loader.get_config_int(self.module_name, "max_execution_time")
# refresh Tracked Regex # refresh Tracked Regex
self.dict_regex_tracked = Term.get_regex_tracked_words_dict() self.tracked_regexs = Tracker.get_tracked_regexs()
self.last_refresh = time.time() self.last_refresh = time.time()
# Exporter # Exporter
@ -48,56 +47,65 @@ class Tracker_Regex(AbstractModule):
self.redis_logger.info(f"Module: {self.module_name} Launched") self.redis_logger.info(f"Module: {self.module_name} Launched")
def compute(self, item_id, content=None): def compute(self, obj_id, obj_type='item', subtype=''):
# refresh Tracked regex # refresh Tracked regex
if self.last_refresh < Tracker.get_tracker_last_updated_by_type('regex'): if self.last_refresh < Tracker.get_tracker_last_updated_by_type('regex'):
self.dict_regex_tracked = Term.get_regex_tracked_words_dict() self.tracked_regexs = Tracker.get_tracked_regexs()
self.last_refresh = time.time() self.last_refresh = time.time()
self.redis_logger.debug('Tracked regex refreshed') self.redis_logger.debug('Tracked regex refreshed')
print('Tracked regex refreshed') print('Tracked regex refreshed')
item = Item(item_id) obj = ail_objects.get_object(obj_type, subtype, obj_id)
item_id = item.get_id() obj_id = obj.get_id()
if not content: obj_type = obj.get_type()
content = item.get_content()
for regex in self.dict_regex_tracked: # Object Filter
matched = self.regex_findall(self.dict_regex_tracked[regex], item_id, content) if obj_type not in self.tracked_regexs:
return None
content = obj.get_content(r_str=True)
for dict_regex in self.tracked_regexs[obj_type]:
matched = self.regex_findall(dict_regex['regex'], obj_id, content)
if matched: if matched:
self.new_tracker_found(regex, 'regex', item) self.new_tracker_found(dict_regex['tracked'], 'regex', obj)
def new_tracker_found(self, tracker_name, tracker_type, item): def new_tracker_found(self, tracker_name, tracker_type, obj):
uuid_list = Tracker.get_tracker_uuid_list(tracker_name, tracker_type) obj_id = obj.get_id()
for tracker_uuid in Tracker.get_trackers_by_tracked_obj_type(tracker_type, obj.get_type(), tracker_name):
item_id = item.get_id()
# date = item.get_date()
item_source = item.get_source()
for tracker_uuid in uuid_list:
tracker = Tracker.Tracker(tracker_uuid) tracker = Tracker.Tracker(tracker_uuid)
# Source Filtering # Filter Object
tracker_sources = tracker.get_sources() filters = tracker.get_filters()
if tracker_sources and item_source not in tracker_sources: if ail_objects.is_filtered(obj, filters):
continue continue
print(f'new tracked regex found: {tracker_name} in {item_id}') print(f'new tracked regex found: {tracker_name} in {obj_id}')
self.redis_logger.warning(f'new tracked regex found: {tracker_name} in {item_id}') self.redis_logger.warning(f'new tracked regex found: {tracker_name} in {obj_id}')
# TODO
Tracker.add_tracked_item(tracker_uuid, item_id) if obj.get_type() == 'item':
date = obj.get_date()
else:
date = None
tracker.add(obj.get_type(), obj.get_subtype(r_str=True), obj_id, date=date)
for tag in tracker.get_tags(): for tag in tracker.get_tags():
msg = f'{tag};{item_id}' if obj.get_type() == 'item':
self.add_message_to_queue(msg, 'Tags') msg = f'{tag};{obj_id}'
self.add_message_to_queue(msg, 'Tags')
else:
obj.add_tag(tag)
if tracker.mail_export(): if tracker.mail_export():
# TODO add matches + custom subjects # TODO add matches + custom subjects
self.exporters['mail'].export(tracker, item) self.exporters['mail'].export(tracker, obj)
if tracker.webhook_export(): if tracker.webhook_export():
self.exporters['webhook'].export(tracker, item) self.exporters['webhook'].export(tracker, obj)
if __name__ == "__main__": if __name__ == "__main__":
module = Tracker_Regex() module = Tracker_Regex()
module.run() module.run()
# module.compute('submitted/2023/05/02/submitted_b1e518f1-703b-40f6-8238-d1c22888197e.gz')

92
bin/trackers/Tracker_Term.py
View File

@ -21,8 +21,7 @@ sys.path.append(os.environ['AIL_BIN'])
################################## ##################################
from modules.abstract_module import AbstractModule from modules.abstract_module import AbstractModule
from lib.ConfigLoader import ConfigLoader from lib.ConfigLoader import ConfigLoader
from lib.objects.Items import Item from lib.objects import ail_objects
from packages import Term
from lib import Tracker from lib import Tracker
from exporter.MailExporter import MailExporterTracker from exporter.MailExporter import MailExporterTracker
@ -54,9 +53,9 @@ class Tracker_Term(AbstractModule):
self.max_execution_time = config_loader.get_config_int('Tracker_Term', "max_execution_time") self.max_execution_time = config_loader.get_config_int('Tracker_Term', "max_execution_time")
# loads tracked words # loads tracked words
self.list_tracked_words = Term.get_tracked_words_list() self.tracked_words = Tracker.get_tracked_words()
self.last_refresh_word = time.time() self.last_refresh_word = time.time()
self.set_tracked_words_list = Term.get_set_tracked_words_list() self.tracked_sets = Tracker.get_tracked_sets()
self.last_refresh_set = time.time() self.last_refresh_set = time.time()
# Exporter # Exporter
@ -65,93 +64,94 @@ class Tracker_Term(AbstractModule):
self.redis_logger.info(f"Module: {self.module_name} Launched") self.redis_logger.info(f"Module: {self.module_name} Launched")
def compute(self, item_id, item_content=None): def compute(self, obj_id, obj_type='item', subtype=''):
# refresh Tracked term # refresh Tracked term
if self.last_refresh_word < Term.get_tracked_term_last_updated_by_type('word'): if self.last_refresh_word < Tracker.get_tracker_last_updated_by_type('word'):
self.list_tracked_words = Term.get_tracked_words_list() self.tracked_words = Tracker.get_tracked_words()
self.last_refresh_word = time.time() self.last_refresh_word = time.time()
self.redis_logger.debug('Tracked word refreshed') self.redis_logger.debug('Tracked word refreshed')
print('Tracked word refreshed') print('Tracked word refreshed')
if self.last_refresh_set < Term.get_tracked_term_last_updated_by_type('set'): if self.last_refresh_set < Tracker.get_tracker_last_updated_by_type('set'):
self.set_tracked_words_list = Term.get_set_tracked_words_list() self.tracked_sets = Tracker.get_tracked_sets()
self.last_refresh_set = time.time() self.last_refresh_set = time.time()
self.redis_logger.debug('Tracked set refreshed') self.redis_logger.debug('Tracked set refreshed')
print('Tracked set refreshed') print('Tracked set refreshed')
# Cast message as Item obj = ail_objects.get_object(obj_type, subtype, obj_id)
item = Item(item_id) obj_type = obj.get_type()
if not item_content:
item_content = item.get_content() # Object Filter
if obj_type not in self.tracked_words and obj_type not in self.tracked_sets:
return None
content = obj.get_content(r_str=True)
signal.alarm(self.max_execution_time) signal.alarm(self.max_execution_time)
dict_words_freq = None dict_words_freq = None
try: try:
dict_words_freq = Term.get_text_word_frequency(item_content) dict_words_freq = Tracker.get_text_word_frequency(content)
except TimeoutException: except TimeoutException:
self.redis_logger.warning(f"{item.get_id()} processing timeout") self.redis_logger.warning(f"{obj.get_id()} processing timeout")
else: else:
signal.alarm(0) signal.alarm(0)
if dict_words_freq: if dict_words_freq:
# create token statistics
# for word in dict_words_freq:
# Term.create_token_statistics(item_date, word, dict_words_freq[word])
# check solo words # check solo words
# ###### # TODO: check if source needed ####### for word in self.tracked_words[obj_type]:
for word in self.list_tracked_words:
if word in dict_words_freq: if word in dict_words_freq:
self.new_term_found(word, 'word', item) self.new_tracker_found(word, 'word', obj)
# check words set # check words set
for elem in self.set_tracked_words_list: for tracked_set in self.tracked_sets[obj_type]:
list_words = elem[0]
nb_words_threshold = elem[1]
word_set = elem[2]
nb_uniq_word = 0 nb_uniq_word = 0
for word in tracked_set['words']:
for word in list_words:
if word in dict_words_freq: if word in dict_words_freq:
nb_uniq_word += 1 nb_uniq_word += 1
if nb_uniq_word >= nb_words_threshold: if nb_uniq_word >= tracked_set['nb']:
self.new_term_found(word_set, 'set', item) self.new_tracker_found(tracked_set['tracked'], 'set', obj)
def new_term_found(self, tracker_name, tracker_type, item): def new_tracker_found(self, tracker_name, tracker_type, obj): # TODO FILTER
uuid_list = Tracker.get_tracker_uuid_list(tracker_name, tracker_type) obj_id = obj.get_id()
item_id = item.get_id() for tracker_uuid in Tracker.get_trackers_by_tracked_obj_type(tracker_type, obj.get_type(), tracker_name):
item_source = item.get_source()
for tracker_uuid in uuid_list:
tracker = Tracker.Tracker(tracker_uuid) tracker = Tracker.Tracker(tracker_uuid)
# Source Filtering # Filter Object
tracker_sources = tracker.get_sources() filters = tracker.get_filters()
if tracker_sources and item_source not in tracker_sources: if ail_objects.is_filtered(obj, filters):
continue continue
print(f'new tracked term found: {tracker_name} in {item_id}') print(f'new tracked term found: {tracker_name} in {obj_id}')
self.redis_logger.warning(f'new tracked term found: {tracker_name} in {item_id}') self.redis_logger.warning(f'new tracked term found: {tracker_name} in {obj_id}')
# TODO
Tracker.add_tracked_item(tracker_uuid, item_id) if obj.get_type() == 'item':
date = obj.get_date()
else:
date = None
tracker.add(obj.get_type(), obj.get_subtype(), obj_id, date=date)
# Tags # Tags
for tag in tracker.get_tags(): for tag in tracker.get_tags():
msg = f'{tag};{item_id}' if obj.get_type() == 'item':
self.add_message_to_queue(msg, 'Tags') msg = f'{tag};{obj_id}'
self.add_message_to_queue(msg, 'Tags')
else:
obj.add_tag(tag)
# Mail # Mail
if tracker.mail_export(): if tracker.mail_export():
# TODO add matches + custom subjects # TODO add matches + custom subjects
self.exporters['mail'].export(tracker, item) self.exporters['mail'].export(tracker, obj)
# Webhook # Webhook
if tracker.webhook_export(): if tracker.webhook_export():
self.exporters['webhook'].export(tracker, item) self.exporters['webhook'].export(tracker, obj)
if __name__ == '__main__': if __name__ == '__main__':
module = Tracker_Term() module = Tracker_Term()
module.run() module.run()
# module.compute('submitted/2023/05/02/submitted_b1e518f1-703b-40f6-8238-d1c22888197e.gz')

68
bin/trackers/Tracker_Typo_Squatting.py
View File

@ -19,7 +19,7 @@ sys.path.append(os.environ['AIL_BIN'])
# Import Project packages # Import Project packages
################################## ##################################
from modules.abstract_module import AbstractModule from modules.abstract_module import AbstractModule
from lib.objects.Items import Item from lib.objects import ail_objects
from lib import Tracker from lib import Tracker
from exporter.MailExporter import MailExporterTracker from exporter.MailExporter import MailExporterTracker
@ -36,8 +36,8 @@ class Tracker_Typo_Squatting(AbstractModule):
self.pending_seconds = 5 self.pending_seconds = 5
# Refresh typo squatting # Refresh typo squatting
self.typosquat_tracked_words_list = Tracker.get_typosquatting_tracked_words_list() self.tracked_typosquattings = Tracker.get_tracked_typosquatting()
self.last_refresh_typosquat = time.time() self.last_refresh_typosquatting = time.time()
# Exporter # Exporter
self.exporters = {'mail': MailExporterTracker(), self.exporters = {'mail': MailExporterTracker(),
@ -45,50 +45,62 @@ class Tracker_Typo_Squatting(AbstractModule):
self.redis_logger.info(f"Module: {self.module_name} Launched") self.redis_logger.info(f"Module: {self.module_name} Launched")
def compute(self, message): def compute(self, message, obj_type='item', subtype=''):
# refresh Tracked typo # refresh Tracked typo
if self.last_refresh_typosquat < Tracker.get_tracker_last_updated_by_type('typosquatting'): if self.last_refresh_typosquatting < Tracker.get_tracker_last_updated_by_type('typosquatting'):
self.typosquat_tracked_words_list = Tracker.get_typosquatting_tracked_words_list() self.tracked_typosquattings = Tracker.get_tracked_typosquatting()
self.last_refresh_typosquat = time.time() self.last_refresh_typosquatting = time.time()
self.redis_logger.debug('Tracked typosquatting refreshed') self.redis_logger.debug('Tracked typosquatting refreshed')
print('Tracked typosquatting refreshed') print('Tracked typosquatting refreshed')
host, item_id = message.split() host, obj_id = message.split()
obj = ail_objects.get_object(obj_type, subtype, obj_id)
obj_type = obj.get_type()
# Cast message as Item # Object Filter
for tracker in self.typosquat_tracked_words_list: if obj_type not in self.tracked_typosquattings:
if host in self.typosquat_tracked_words_list[tracker]: return None
item = Item(item_id)
self.new_tracker_found(tracker, 'typosquatting', item)
def new_tracker_found(self, tracker, tracker_type, item): for typo in self.tracked_typosquattings[obj_type]:
item_id = item.get_id() if host in typo['domains']:
item_source = item.get_source() self.new_tracker_found(typo['tracked'], 'typosquatting', obj)
print(f'new tracked typosquatting found: {tracker} in {item_id}')
self.redis_logger.warning(f'tracker typosquatting: {tracker} in {item_id}')
for tracker_uuid in Tracker.get_tracker_uuid_list(tracker, tracker_type): def new_tracker_found(self, tracked, tracker_type, obj):
obj_id = obj.get_id()
for tracker_uuid in Tracker.get_trackers_by_tracked_obj_type(tracker_type, obj.get_type(), tracked):
tracker = Tracker.Tracker(tracker_uuid) tracker = Tracker.Tracker(tracker_uuid)
# Source Filtering # Filter Object
tracker_sources = tracker.get_sources() filters = tracker.get_filters()
if tracker_sources and item_source not in tracker_sources: if ail_objects.is_filtered(obj, filters):
continue continue
Tracker.add_tracked_item(tracker_uuid, item_id) print(f'new tracked typosquatting found: {tracked} in {obj_id}')
self.redis_logger.warning(f'tracker typosquatting: {tracked} in {obj_id}')
if obj.get_type() == 'item':
date = obj.get_date()
else:
date = None
tracker.add(obj.get_type(), obj.get_subtype(r_str=True), obj_id, date=date)
# Tags
for tag in tracker.get_tags(): for tag in tracker.get_tags():
msg = f'{tag};{item_id}' if obj.get_type() == 'item':
self.add_message_to_queue(msg, 'Tags') msg = f'{tag};{obj_id}'
self.add_message_to_queue(msg, 'Tags')
else:
obj.add_tag(tag)
if tracker.mail_export(): if tracker.mail_export():
self.exporters['mail'].export(tracker, item) self.exporters['mail'].export(tracker, obj)
if tracker.webhook_export(): if tracker.webhook_export():
self.exporters['webhook'].export(tracker, item) self.exporters['webhook'].export(tracker, obj)
if __name__ == '__main__': if __name__ == '__main__':
module = Tracker_Typo_Squatting() module = Tracker_Typo_Squatting()
module.run() module.run()
#module.compute('g00gle.com tests/2020/01/01/test.gz') # module.compute('g00gle.com tests/2020/01/01/test.gz')

84
bin/trackers/Tracker_Yara.py
View File

@ -19,7 +19,7 @@ sys.path.append(os.environ['AIL_BIN'])
# Import Project packages # Import Project packages
################################## ##################################
from modules.abstract_module import AbstractModule from modules.abstract_module import AbstractModule
from lib.objects.Items import Item from lib.objects import ail_objects
from lib import Tracker from lib import Tracker
from exporter.MailExporter import MailExporterTracker from exporter.MailExporter import MailExporterTracker
@ -35,10 +35,10 @@ class Tracker_Yara(AbstractModule):
self.pending_seconds = 5 self.pending_seconds = 5
# Load Yara rules # Load Yara rules
self.rules = Tracker.reload_yara_rules() self.rules = Tracker.get_tracked_yara_rules()
self.last_refresh = time.time() self.last_refresh = time.time()
self.item = None self.obj = None
# Exporter # Exporter
self.exporters = {'mail': MailExporterTracker(), self.exporters = {'mail': MailExporterTracker(),
@ -46,58 +46,67 @@ class Tracker_Yara(AbstractModule):
self.redis_logger.info(f"Module: {self.module_name} Launched") self.redis_logger.info(f"Module: {self.module_name} Launched")
def compute(self, item_id, item_content=None): def compute(self, obj_id, obj_type='item', subtype=''):
# refresh YARA list # refresh YARA list
if self.last_refresh < Tracker.get_tracker_last_updated_by_type('yara'): if self.last_refresh < Tracker.get_tracker_last_updated_by_type('yara'):
self.rules = Tracker.reload_yara_rules() self.rules = Tracker.get_tracked_yara_rules()
self.last_refresh = time.time() self.last_refresh = time.time()
self.redis_logger.debug('Tracked set refreshed') self.redis_logger.debug('Tracked set refreshed')
print('Tracked set refreshed') print('Tracked set refreshed')
self.item = Item(item_id) self.obj = ail_objects.get_object(obj_type, subtype, obj_id)
if not item_content: obj_type = self.obj.get_type()
item_content = self.item.get_content()
# Object Filter
if obj_type not in self.rules:
return None
content = self.obj.get_content(r_str=True)
try: try:
yara_match = self.rules.match(data=item_content, callback=self.yara_rules_match, yara_match = self.rules[obj_type].match(data=content, callback=self.yara_rules_match,
which_callbacks=yara.CALLBACK_MATCHES, timeout=60) which_callbacks=yara.CALLBACK_MATCHES, timeout=60)
if yara_match: if yara_match:
self.redis_logger.warning(f'tracker yara: new match {self.item.get_id()}: {yara_match}') self.redis_logger.warning(f'tracker yara: new match {self.obj.get_id()}: {yara_match}')
print(f'{self.item.get_id()}: {yara_match}') print(f'{self.obj.get_id()}: {yara_match}')
except yara.TimeoutError: except yara.TimeoutError:
print(f'{self.item.get_id()}: yara scanning timed out') print(f'{self.obj.get_id()}: yara scanning timed out')
self.redis_logger.info(f'{self.item.get_id()}: yara scanning timed out') self.redis_logger.info(f'{self.obj.get_id()}: yara scanning timed out')
def yara_rules_match(self, data): def yara_rules_match(self, data):
tracker_uuid = data['namespace'] tracker_name = data['namespace']
item_id = self.item.get_id() obj_id = self.obj.get_id()
for tracker_uuid in Tracker.get_trackers_by_tracked_obj_type('yara', self.obj.get_type(), tracker_name):
tracker = Tracker.Tracker(tracker_uuid)
item = Item(item_id) # Filter Object
item_source = self.item.get_source() filters = tracker.get_filters()
if ail_objects.is_filtered(self.obj, filters):
continue
tracker = Tracker.Tracker(tracker_uuid) if self.obj.get_type() == 'item':
date = self.obj.get_date()
else:
date = None
# Source Filtering tracker.add(self.obj.get_type(), self.obj.get_subtype(r_str=True), obj_id, date=date)
tracker_sources = tracker.get_sources()
if tracker_sources and item_source not in tracker_sources:
print(f'Source Filtering: {data["rule"]}')
return yara.CALLBACK_CONTINUE
Tracker.add_tracked_item(tracker_uuid, item_id) # TODO # Tags
for tag in tracker.get_tags():
if self.obj.get_type() == 'item':
msg = f'{tag};{obj_id}'
self.add_message_to_queue(msg, 'Tags')
else:
self.obj.add_tag(tag)
# Tags # Mails
for tag in tracker.get_tags(): if tracker.mail_export():
msg = f'{tag};{item_id}' # TODO add matches + custom subjects
self.add_message_to_queue(msg, 'Tags') self.exporters['mail'].export(tracker, self.obj)
# Mails # Webhook
if tracker.mail_export(): if tracker.webhook_export():
# TODO add matches + custom subjects self.exporters['webhook'].export(tracker, self.obj)
self.exporters['mail'].export(tracker, item)
# Webhook
if tracker.webhook_export():
self.exporters['webhook'].export(tracker, item)
return yara.CALLBACK_CONTINUE return yara.CALLBACK_CONTINUE
@ -105,3 +114,4 @@ class Tracker_Yara(AbstractModule):
if __name__ == '__main__': if __name__ == '__main__':
module = Tracker_Yara() module = Tracker_Yara()
module.run() module.run()
# module.compute('archive/gist.github.com/2023/04/13/chipzoller_d8d6d2d737d02ad4fe9d30a897170761.gz')

137
update/v2.2/Update.py
View File

@ -39,74 +39,75 @@ if __name__ == '__main__':
r_serv_term_stats.flushdb() r_serv_term_stats.flushdb()
# convert all regex: # Disabled. Checkout the v2.2 branch if you need it
all_regex = r_serv_termfreq.smembers('TrackedRegexSet') # # convert all regex:
for regex in all_regex: # all_regex = r_serv_termfreq.smembers('TrackedRegexSet')
tags = list(r_serv_termfreq.smembers('TrackedNotificationTags_{}'.format(regex))) # for regex in all_regex:
mails = list(r_serv_termfreq.smembers('TrackedNotificationEmails_{}'.format(regex))) # tags = list(r_serv_termfreq.smembers('TrackedNotificationTags_{}'.format(regex)))
# mails = list(r_serv_termfreq.smembers('TrackedNotificationEmails_{}'.format(regex)))
new_term = regex[1:-1] #
res = Term.parse_json_term_to_add({"term": new_term, "type": 'regex', "tags": tags, "mails": mails, "level": 1}, # new_term = regex[1:-1]
'admin@admin.test') # res = Term.parse_json_term_to_add({"term": new_term, "type": 'regex', "tags": tags, "mails": mails, "level": 1},
if res[1] == 200: # 'admin@admin.test')
term_uuid = res[0]['uuid'] # if res[1] == 200:
list_items = r_serv_termfreq.smembers('regex_{}'.format(regex)) # term_uuid = res[0]['uuid']
for paste_item in list_items: # list_items = r_serv_termfreq.smembers('regex_{}'.format(regex))
item_id = get_item_id(paste_item) # for paste_item in list_items:
item_date = get_item_date(item_id) # item_id = get_item_id(paste_item)
Term.add_tracked_item(term_uuid, item_id, item_date) # item_date = get_item_date(item_id)
# Term.add_tracked_item(term_uuid, item_id, item_date)
# Invalid Tracker => remove it #
else: # # Invalid Tracker => remove it
print('Invalid Regex Removed: {}'.format(regex)) # else:
print(res[0]) # print('Invalid Regex Removed: {}'.format(regex))
# allow reprocess # print(res[0])
r_serv_termfreq.srem('TrackedRegexSet', regex) # # allow reprocess
# r_serv_termfreq.srem('TrackedRegexSet', regex)
all_tokens = r_serv_termfreq.smembers('TrackedSetTermSet') #
for token in all_tokens: # all_tokens = r_serv_termfreq.smembers('TrackedSetTermSet')
tags = list(r_serv_termfreq.smembers('TrackedNotificationTags_{}'.format(token))) # for token in all_tokens:
mails = list(r_serv_termfreq.smembers('TrackedNotificationEmails_{}'.format(token))) # tags = list(r_serv_termfreq.smembers('TrackedNotificationTags_{}'.format(token)))
# mails = list(r_serv_termfreq.smembers('TrackedNotificationEmails_{}'.format(token)))
res = Term.parse_json_term_to_add({"term": token, "type": 'word', "tags": tags, "mails": mails, "level": 1}, 'admin@admin.test') #
if res[1] == 200: # res = Term.parse_json_term_to_add({"term": token, "type": 'word', "tags": tags, "mails": mails, "level": 1}, 'admin@admin.test')
term_uuid = res[0]['uuid'] # if res[1] == 200:
list_items = r_serv_termfreq.smembers('tracked_{}'.format(token)) # term_uuid = res[0]['uuid']
for paste_item in list_items: # list_items = r_serv_termfreq.smembers('tracked_{}'.format(token))
item_id = get_item_id(paste_item) # for paste_item in list_items:
item_date = get_item_date(item_id) # item_id = get_item_id(paste_item)
Term.add_tracked_item(term_uuid, item_id, item_date) # item_date = get_item_date(item_id)
# Invalid Tracker => remove it # Term.add_tracked_item(term_uuid, item_id, item_date)
else: # # Invalid Tracker => remove it
print('Invalid Token Removed: {}'.format(token)) # else:
print(res[0]) # print('Invalid Token Removed: {}'.format(token))
# allow reprocess # print(res[0])
r_serv_termfreq.srem('TrackedSetTermSet', token) # # allow reprocess
# r_serv_termfreq.srem('TrackedSetTermSet', token)
all_set = r_serv_termfreq.smembers('TrackedSetSet') #
for curr_set in all_set: # all_set = r_serv_termfreq.smembers('TrackedSetSet')
tags = list(r_serv_termfreq.smembers('TrackedNotificationTags_{}'.format(curr_set))) # for curr_set in all_set:
mails = list(r_serv_termfreq.smembers('TrackedNotificationEmails_{}'.format(curr_set))) # tags = list(r_serv_termfreq.smembers('TrackedNotificationTags_{}'.format(curr_set)))
# mails = list(r_serv_termfreq.smembers('TrackedNotificationEmails_{}'.format(curr_set)))
to_remove = ',{}'.format(curr_set.split(',')[-1]) #
new_set = rreplace(curr_set, to_remove, '', 1) # to_remove = ',{}'.format(curr_set.split(',')[-1])
new_set = new_set[2:] # new_set = rreplace(curr_set, to_remove, '', 1)
new_set = new_set.replace(',', '') # new_set = new_set[2:]
# new_set = new_set.replace(',', '')
res = Term.parse_json_term_to_add({"term": new_set, "type": 'set', "nb_words": 1, "tags": tags, "mails": mails, "level": 1}, 'admin@admin.test') #
if res[1] == 200: # res = Term.parse_json_term_to_add({"term": new_set, "type": 'set', "nb_words": 1, "tags": tags, "mails": mails, "level": 1}, 'admin@admin.test')
term_uuid = res[0]['uuid'] # if res[1] == 200:
list_items = r_serv_termfreq.smembers('tracked_{}'.format(curr_set)) # term_uuid = res[0]['uuid']
for paste_item in list_items: # list_items = r_serv_termfreq.smembers('tracked_{}'.format(curr_set))
item_id = get_item_id(paste_item) # for paste_item in list_items:
item_date = get_item_date(item_id) # item_id = get_item_id(paste_item)
Term.add_tracked_item(term_uuid, item_id, item_date) # item_date = get_item_date(item_id)
# Invalid Tracker => remove it # Term.add_tracked_item(term_uuid, item_id, item_date)
else: # # Invalid Tracker => remove it
print('Invalid Set Removed: {}'.format(curr_set)) # else:
print(res[0]) # print('Invalid Set Removed: {}'.format(curr_set))
# allow reprocess # print(res[0])
r_serv_termfreq.srem('TrackedSetSet', curr_set) # # allow reprocess
# r_serv_termfreq.srem('TrackedSetSet', curr_set)
r_serv_termfreq.flushdb() r_serv_termfreq.flushdb()

2
update/v3.7/Update.py
View File

@ -24,7 +24,7 @@ class Updater(AIL_Updater):
print('Fixing Tracker_uuid list ...') print('Fixing Tracker_uuid list ...')
Tracker.fix_all_tracker_uuid_list() Tracker.fix_all_tracker_uuid_list()
nb = 0 nb = 0
for tracker_uuid in Tracker.get_all_tracker_uuid(): for tracker_uuid in Tracker.get_trackers():
self.r_serv.sadd('trackers_update_v3.7', tracker_uuid) self.r_serv.sadd('trackers_update_v3.7', tracker_uuid)
nb += 1 nb += 1

209
var/www/blueprints/hunters.py
View File

@ -1,15 +1,15 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*-coding:UTF-8 -* # -*-coding:UTF-8 -*
''' """
Blueprint Flask: crawler splash endpoints: dashboard, onion crawler ... Blueprint Flask: crawler splash endpoints: dashboard, onion crawler ...
''' """
import os import os
import sys import sys
import json import json
from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response, make_response from flask import render_template, jsonify, request, Blueprint, redirect, url_for, Response
from flask_login import login_required, current_user, login_user, logout_user from flask_login import login_required, current_user, login_user, logout_user
sys.path.append('modules') sys.path.append('modules')
@ -22,8 +22,10 @@ sys.path.append(os.environ['AIL_BIN'])
################################## ##################################
# Import Project packages # Import Project packages
################################## ##################################
from lib import ail_core
from lib import item_basic from lib import item_basic
from lib import Tracker from lib import Tracker
from lib import Tag
bootstrap_label = Flask_config.bootstrap_label bootstrap_label = Flask_config.bootstrap_label
@ -34,7 +36,6 @@ hunters = Blueprint('hunters', __name__, template_folder=os.path.join(os.environ
# ============ VARIABLES ============ # ============ VARIABLES ============
# ============ FUNCTIONS ============ # ============ FUNCTIONS ============
def api_validator(api_response): def api_validator(api_response):
if api_response: if api_response:
@ -45,11 +46,203 @@ def create_json_response(data, status_code):
# ============= ROUTES ============== # ============= ROUTES ==============
##################
# TRACKERS #
##################
@hunters.route('/trackers', methods=['GET'])
@login_required
@login_read_only
def trackers_dashboard():
user_id = current_user.get_id() # TODO
trackers = Tracker.get_trackers_dashboard()
stats = Tracker.get_trackers_stats(user_id)
return render_template("trackers_dashboard.html", trackers=trackers, stats=stats, bootstrap_label=bootstrap_label)
@hunters.route("/trackers/all")
@login_required
@login_read_only
def tracked_menu():
user_id = current_user.get_id()
user_trackers = Tracker.get_user_trackers_meta(user_id)
global_trackers = Tracker.get_global_trackers_meta()
return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers, bootstrap_label=bootstrap_label)
@hunters.route("/trackers/word")
@login_required
@login_read_only
def tracked_menu_word():
tracker_type = 'word'
user_id = current_user.get_id()
user_trackers = Tracker.get_user_trackers_meta(user_id, tracker_type='word')
global_trackers = Tracker.get_global_trackers_meta(tracker_type='word')
return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers, bootstrap_label=bootstrap_label, tracker_type=tracker_type)
@hunters.route("/trackers/set")
@login_required
@login_read_only
def tracked_menu_set():
tracker_type = 'set'
user_id = current_user.get_id()
user_trackers = Tracker.get_user_trackers_meta(user_id, tracker_type=tracker_type)
global_trackers = Tracker.get_global_trackers_meta(tracker_type=tracker_type)
return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers, bootstrap_label=bootstrap_label, tracker_type=tracker_type)
@hunters.route("/trackers/regex")
@login_required
@login_read_only
def tracked_menu_regex():
tracker_type = 'regex'
user_id = current_user.get_id()
user_trackers = Tracker.get_user_trackers_meta(user_id, tracker_type=tracker_type)
global_trackers = Tracker.get_global_trackers_meta(tracker_type=tracker_type)
return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers, bootstrap_label=bootstrap_label, tracker_type=tracker_type)
@hunters.route("/trackers/yara")
@login_required
@login_read_only
def tracked_menu_yara():
tracker_type = 'yara'
user_id = current_user.get_id()
user_trackers = Tracker.get_user_trackers_meta(user_id, tracker_type=tracker_type)
global_trackers = Tracker.get_global_trackers_meta(tracker_type=tracker_type)
return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers, bootstrap_label=bootstrap_label, tracker_type=tracker_type)
@hunters.route("/trackers/typosquatting")
@login_required
@login_read_only
def tracked_menu_typosquatting():
tracker_type = 'typosquatting'
user_id = current_user.get_id()
user_trackers = Tracker.get_user_trackers_meta(user_id, tracker_type=tracker_type)
global_trackers = Tracker.get_global_trackers_meta(tracker_type=tracker_type)
return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers,
bootstrap_label=bootstrap_label, tracker_type=tracker_type)
@hunters.route("/tracker/add", methods=['GET', 'POST'])
@login_required
@login_analyst
def add_tracked_menu():
if request.method == 'POST':
to_track = request.form.get("tracker")
tracker_uuid = request.form.get("tracker_uuid")
tracker_type = request.form.get("tracker_type")
nb_words = request.form.get("nb_word", 1)
description = request.form.get("description", '')
webhook = request.form.get("webhook", '')
level = request.form.get("level", 0)
mails = request.form.get("mails", [])
# TAGS
tags = request.form.get("tags", [])
taxonomies_tags = request.form.get('taxonomies_tags')
if taxonomies_tags:
try:
taxonomies_tags = json.loads(taxonomies_tags)
except:
taxonomies_tags = []
else:
taxonomies_tags = []
galaxies_tags = request.form.get('galaxies_tags')
if galaxies_tags:
try:
galaxies_tags = json.loads(galaxies_tags)
except:
galaxies_tags = []
# custom tags
if tags:
tags = tags.split()
else:
tags = []
tags = tags + taxonomies_tags + galaxies_tags
# YARA #
if tracker_type == 'yara':
yara_default_rule = request.form.get("yara_default_rule")
yara_custom_rule = request.form.get("yara_custom_rule")
if yara_custom_rule:
to_track = yara_custom_rule
tracker_type = 'yara_custom'
else:
to_track = yara_default_rule
tracker_type = 'yara_default'
if level == 'on':
level = 1
else:
level = 0
if mails:
mails = mails.split()
else:
tags = []
# FILTERS
filters = {}
for obj_type in Tracker.get_objects_tracked():
new_filter = request.form.get(f'{obj_type}_obj')
if new_filter == 'on':
filters[obj_type] = {}
# Mimetypes
mimetypes = request.form.get(f'mimetypes_{obj_type}', [])
if mimetypes:
mimetypes = json.loads(mimetypes)
filters[obj_type]['mimetypes'] = mimetypes
# Sources
sources = request.form.get(f'sources_{obj_type}', [])
if sources:
sources = json.loads(sources)
filters[obj_type]['sources'] = sources
# Subtypes
for obj_subtype in ail_core.get_object_all_subtypes(obj_type):
subtype = request.form.get(f'filter_{obj_type}_{obj_subtype}')
if subtype == 'on':
if 'subtypes' not in filters[obj_type]:
filters[obj_type]['subtypes'] = []
filters[obj_type]['subtypes'].append(obj_subtype)
input_dict = {"tracked": to_track, "type": tracker_type,
"tags": tags, "mails": mails, "filters": filters,
"level": level, "description": description, "webhook": webhook}
if tracker_type == 'set':
try:
input_dict['nb_words'] = int(nb_words)
except TypeError:
input_dict['nb_words'] = 1
user_id = current_user.get_id()
res = Tracker.api_add_tracker(input_dict, user_id)
if res[1] == 200:
return redirect(url_for('hunters.trackers_dashboard'))
else:
return create_json_response(res[0], res[1])
else:
return render_template("tracker_add.html",
all_sources=item_basic.get_all_items_sources(r_list=True),
tags_selector_data=Tag.get_tags_selector_data(),
all_yara_files=Tracker.get_all_default_yara_files())
@hunters.route('/tracker/delete', methods=['GET'])
@login_required
@login_analyst
def tracker_delete():
user_id = current_user.get_id()
tracker_uuid = request.args.get('uuid')
res = Tracker.api_delete_tracker({'uuid': tracker_uuid}, user_id)
if res[1] != 200:
return create_json_response(res[0], res[1])
else:
return redirect(url_for('hunter.tracked_menu'))
####################
# RETRO HUNT #
####################
@hunters.route('/retro_hunt/tasks', methods=['GET']) @hunters.route('/retro_hunt/tasks', methods=['GET'])
@login_required @login_required
@login_read_only @login_read_only
def retro_hunt_all_tasks(): def retro_hunt_all_tasks():
retro_hunts = Tracker.get_all_retro_hunt_tasks_with_metadata() retro_hunts = Tracker.get_retro_hunt_tasks_metas()
return render_template("retro_hunt_tasks.html", retro_hunts=retro_hunts, bootstrap_label=bootstrap_label) return render_template("retro_hunt_tasks.html", retro_hunts=retro_hunts, bootstrap_label=bootstrap_label)
@hunters.route('/retro_hunt/task/show', methods=['GET']) @hunters.route('/retro_hunt/task/show', methods=['GET'])
@ -69,8 +262,8 @@ def retro_hunt_show_task():
if res: if res:
return create_json_response(res[0], res[1]) return create_json_response(res[0], res[1])
dict_task = Tracker.get_retro_hunt_task_metadata(task_uuid, date=True, progress=True, creator=True, retro_hunt = Tracker.RetroHunt(task_uuid)
sources=True, tags=True, description=True) dict_task = retro_hunt.get_meta(options={'creator', 'date', 'description', 'progress', 'sources', 'tags'})
rule_content = Tracker.get_yara_rule_content(dict_task['rule']) rule_content = Tracker.get_yara_rule_content(dict_task['rule'])
if date_from: if date_from:
@ -177,7 +370,7 @@ def retro_hunt_delete_task():
#### JSON #### #### JSON ####
@hunters.route("/tracker/get_json_retro_hunt_nb_items_by_date", methods=['GET']) @hunters.route("/retro_hunt/nb_items/date/json", methods=['GET'])
@login_required @login_required
@login_read_only @login_read_only
def get_json_retro_hunt_nb_items_by_date(): def get_json_retro_hunt_nb_items_by_date():

208
var/www/modules/hunter/Flask_hunter.py
View File

@ -17,10 +17,11 @@ sys.path.append(os.environ['AIL_BIN'])
################################## ##################################
# Import Project packages # Import Project packages
################################## ##################################
from lib.objects import ail_objects
from lib import item_basic from lib import item_basic
from lib import Tracker from lib import Tracker
from lib import Tag from lib import Tag
from packages import Term from packages import Date
# ============ VARIABLES ============ # ============ VARIABLES ============
@ -34,142 +35,11 @@ hunter = Blueprint('hunter', __name__, template_folder='templates')
# ============ FUNCTIONS ============ # ============ FUNCTIONS ============
def create_json_response(data, status_code):
return Response(json.dumps(data, indent=2, sort_keys=True), mimetype='application/json'), status_code
# ============ ROUTES ============ # ============ ROUTES ============
@hunter.route("/trackers")
@login_required
@login_read_only
def tracked_menu():
user_id = current_user.get_id()
user_trackers = Tracker.get_user_trackers_metadata(user_id)
global_trackers = Tracker.get_global_trackers_metadata()
return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers, bootstrap_label=bootstrap_label)
@hunter.route("/trackers/word")
@login_required
@login_read_only
def tracked_menu_word():
tracker_type = 'word'
user_id = current_user.get_id()
user_trackers = Tracker.get_user_trackers_metadata(user_id, tracker_type='word')
global_trackers = Tracker.get_global_trackers_metadata(tracker_type='word')
return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers, bootstrap_label=bootstrap_label, tracker_type=tracker_type)
@hunter.route("/trackers/set")
@login_required
@login_read_only
def tracked_menu_set():
tracker_type = 'set'
user_id = current_user.get_id()
user_trackers = Tracker.get_user_trackers_metadata(user_id, tracker_type=tracker_type)
global_trackers = Tracker.get_global_trackers_metadata(tracker_type=tracker_type)
return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers, bootstrap_label=bootstrap_label, tracker_type=tracker_type)
@hunter.route("/trackers/regex")
@login_required
@login_read_only
def tracked_menu_regex():
tracker_type = 'regex'
user_id = current_user.get_id()
user_trackers = Tracker.get_user_trackers_metadata(user_id, tracker_type=tracker_type)
global_trackers = Tracker.get_global_trackers_metadata(tracker_type=tracker_type)
return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers, bootstrap_label=bootstrap_label, tracker_type=tracker_type)
@hunter.route("/trackers/yara")
@login_required
@login_read_only
def tracked_menu_yara():
tracker_type = 'yara'
user_id = current_user.get_id()
user_trackers = Tracker.get_user_trackers_metadata(user_id, tracker_type=tracker_type)
global_trackers = Tracker.get_global_trackers_metadata(tracker_type=tracker_type)
return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers, bootstrap_label=bootstrap_label, tracker_type=tracker_type)
@hunter.route("/trackers/typosquatting")
@login_required
@login_read_only
def tracked_menu_typosquatting():
tracker_type = 'typosquatting'
user_id = current_user.get_id()
user_trackers = Tracker.get_user_trackers_metadata(user_id, tracker_type=tracker_type)
global_trackers = Tracker.get_global_trackers_metadata(tracker_type=tracker_type)
return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers,
bootstrap_label=bootstrap_label, tracker_type=tracker_type)
@hunter.route("/tracker/add", methods=['GET', 'POST'])
@login_required
@login_analyst
def add_tracked_menu():
if request.method == 'POST':
tracker = request.form.get("tracker")
tracker_uuid = request.form.get("tracker_uuid")
tracker_type = request.form.get("tracker_type")
nb_words = request.form.get("nb_word", 1)
description = request.form.get("description", '')
webhook = request.form.get("webhook", '')
level = request.form.get("level", 0)
mails = request.form.get("mails", [])
sources = request.form.get("sources", [])
tags = request.form.get("tags", [])
taxonomies_tags = request.form.get('taxonomies_tags')
if taxonomies_tags:
try:
taxonomies_tags = json.loads(taxonomies_tags)
except Exception:
taxonomies_tags = []
else:
taxonomies_tags = []
galaxies_tags = request.form.get('galaxies_tags')
if galaxies_tags:
try:
galaxies_tags = json.loads(galaxies_tags)
except Exception:
galaxies_tags = []
# YARA #
if tracker_type == 'yara':
yara_default_rule = request.form.get("yara_default_rule")
yara_custom_rule = request.form.get("yara_custom_rule")
if yara_custom_rule:
tracker = yara_custom_rule
tracker_type='yara_custom'
else:
tracker = yara_default_rule
tracker_type='yara_default'
if level == 'on':
level = 1
if mails:
mails = mails.split()
if tags:
tags = tags.split()
if sources:
sources = json.loads(sources)
input_dict = {"tracker": tracker, "type": tracker_type, "nb_words": nb_words,
"tags": tags, "mails": mails, "sources": sources,
"level": level, "description": description, "webhook": webhook}
user_id = current_user.get_id()
# edit tracker
if tracker_uuid:
input_dict['uuid'] = tracker_uuid
res = Tracker.api_add_tracker(input_dict, user_id)
if res[1] == 200:
if 'uuid' in res[0]:
return redirect(url_for('hunter.show_tracker', uuid=res[0]['uuid']))
else:
return redirect(url_for('hunter.tracked_menu'))
else:
## TODO: use modal
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
else:
return render_template("edit_tracker.html",
all_sources=item_basic.get_all_items_sources(r_list=True),
tags_selector_data=Tag.get_tags_selector_data(),
all_yara_files=Tracker.get_all_default_yara_files())
@hunter.route("/tracker/edit", methods=['GET', 'POST']) @hunter.route("/tracker/edit", methods=['GET', 'POST'])
@login_required @login_required
@login_analyst @login_analyst
@ -181,7 +51,8 @@ def edit_tracked_menu():
if res[1] != 200: # invalid access if res[1] != 200: # invalid access
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
dict_tracker = Tracker.get_tracker_metadata(tracker_uuid, user_id=True, level=True, description=True, tags=True, mails=True, sources=True, webhook=True) tracker = Tracker.Tracker(tracker_uuid)
dict_tracker = tracker.get_meta(options={'description', 'level', 'mails', 'sources', 'tags', 'user', 'webhook'})
dict_tracker['tags'] = ' '.join(dict_tracker['tags']) dict_tracker['tags'] = ' '.join(dict_tracker['tags'])
dict_tracker['mails'] = ' '.join(dict_tracker['mails']) dict_tracker['mails'] = ' '.join(dict_tracker['mails'])
@ -205,7 +76,7 @@ def edit_tracked_menu():
# word # word
# set of word + nb words # set of word + nb words
# regex # regex
# yara custum # yara custom
# yara default ???? => allow edit ? # yara default ???? => allow edit ?
#### EDIT SHow Trackers ?????????????????????????????????????????????????? #### EDIT SHow Trackers ??????????????????????????????????????????????????
@ -228,34 +99,36 @@ def show_tracker():
if date_to: if date_to:
date_to = date_to.replace('-', '') date_to = date_to.replace('-', '')
tracker_metadata = Tracker.get_tracker_metadata(tracker_uuid, user_id=True, level=True, description=True, tags=True, mails=True, sources=True, sparkline=True, webhook=True) tracker = Tracker.Tracker(tracker_uuid)
meta = tracker.get_meta(options={'description', 'level', 'mails', 'filters', 'sparkline', 'tags',
'user', 'webhook'})
if tracker_metadata['type'] == 'yara': if meta['type'] == 'yara':
yara_rule_content = Tracker.get_yara_rule_content(tracker_metadata['tracker']) yara_rule_content = Tracker.get_yara_rule_content(meta['tracked'])
else: else:
yara_rule_content = None yara_rule_content = None
if tracker_metadata['type'] == 'typosquatting': if meta['type'] == 'typosquatting':
typo_squatting = list(Tracker.get_tracker_typosquatting_domains(tracker_uuid)) typo_squatting = Tracker.get_tracked_typosquatting_domains(meta['tracked'])
typo_squatting.sort() sorted(typo_squatting)
else: else:
typo_squatting = None typo_squatting = set()
if date_from: if date_from:
res = Term.parse_get_tracker_term_item({'uuid': tracker_uuid, 'date_from': date_from, 'date_to': date_to}, user_id) date_from, date_to = Date.sanitise_daterange(meta['first_seen'], meta['last_seen'])
if res[1] != 200: objs = tracker.get_objs_by_daterange(date_from, date_to)
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] meta['objs'] = ail_objects.get_objects_meta(objs, flask_context=True)
tracker_metadata['items'] = res[0]['items']
tracker_metadata['date_from'] = res[0]['date_from']
tracker_metadata['date_to'] = res[0]['date_to']
else: else:
tracker_metadata['items'] = [] date_from = ''
tracker_metadata['date_from'] = '' date_to = ''
tracker_metadata['date_to'] = '' meta['objs'] = []
tracker_metadata['sources'] = sorted(tracker_metadata['sources']) meta['date_from'] = date_from
meta['date_to'] = date_to
print(meta['filters'])
meta['item_sources'] = sorted(meta['filters'].get('item', {}).get('sources', []))
return render_template("showTracker.html", tracker_metadata=tracker_metadata, return render_template("showTracker.html", tracker_metadata=meta,
yara_rule_content=yara_rule_content, yara_rule_content=yara_rule_content,
typo_squatting=typo_squatting, typo_squatting=typo_squatting,
bootstrap_label=bootstrap_label) bootstrap_label=bootstrap_label)
@ -309,21 +182,16 @@ def update_tracker_mails():
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
return redirect(url_for('hunter.show_tracker', uuid=term_uuid)) return redirect(url_for('hunter.show_tracker', uuid=term_uuid))
@hunter.route("/tracker/delete", methods=['GET'])
@login_required
@login_analyst
def delete_tracker():
user_id = current_user.get_id()
term_uuid = request.args.get('uuid')
res = Term.parse_tracked_term_to_delete({'uuid': term_uuid}, user_id)
if res[1] !=200:
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
return redirect(url_for('hunter.tracked_menu'))
@hunter.route("/tracker/get_json_tracker_stats", methods=['GET']) @hunter.route("/tracker/get_json_tracker_stats", methods=['GET'])
@login_required @login_required
@login_read_only @login_read_only
def get_json_tracker_stats(): def get_json_tracker_stats():
user_id = current_user.get_id()
tracker_uuid = request.args.get('uuid')
res = Tracker.api_check_tracker_acl(tracker_uuid, user_id)
if res:
return create_json_response(res[0], res[1])
date_from = request.args.get('date_from') date_from = request.args.get('date_from')
date_to = request.args.get('date_to') date_to = request.args.get('date_to')
@ -331,13 +199,10 @@ def get_json_tracker_stats():
date_from = date_from.replace('-', '') date_from = date_from.replace('-', '')
if date_to: if date_to:
date_to = date_to.replace('-', '') date_to = date_to.replace('-', '')
tracker_uuid = request.args.get('uuid')
if date_from and date_to: if date_from and date_to:
res = Term.get_list_tracked_term_stats_by_day([tracker_uuid], date_from=date_from, date_to=date_to) res = Tracker.get_trackers_graph_by_day([tracker_uuid], date_from=date_from, date_to=date_to)
else: else:
res = Term.get_list_tracked_term_stats_by_day([tracker_uuid]) res = Tracker.get_trackers_graph_by_day([tracker_uuid])
return jsonify(res) return jsonify(res)
@hunter.route("/tracker/yara/default_rule/content", methods=['GET']) @hunter.route("/tracker/yara/default_rule/content", methods=['GET'])
@ -348,5 +213,6 @@ def get_default_yara_rule_content():
res = Tracker.api_get_default_rule_content(default_yara_rule) res = Tracker.api_get_default_rule_content(default_yara_rule)
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
# ========= REGISTRATION ========= # ========= REGISTRATION =========
app.register_blueprint(hunter, url_prefix=baseUrl) app.register_blueprint(hunter, url_prefix=baseUrl)

260
var/www/modules/hunter/templates/edit_tracker.html
View File

@ -1,260 +0,0 @@
<!DOCTYPE html>
<html>
<head>
<title>AIL-Framework</title>
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png')}}">
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/daterangepicker.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/tags.css') }}" rel="stylesheet">
<!-- JS -->
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/tags.js') }}"></script>
</head>
<body>
{% include 'nav_bar.html' %}
<div class="container-fluid">
<div class="row">
{% include 'hunter/menu_sidebar.html' %}
<div class="col-12 col-lg-10" id="core_content">
<div class="card my-3">
<div class="card-header bg-dark text-white">
<h5 class="card-title">{%if dict_tracker%}Edit a{%else%}Create a new{%endif%} Tracker</h5>
</div>
<div class="card-body">
<form action="{{ url_for('hunter.add_tracked_menu') }}" method='post'>
{%if dict_tracker%}
<input id="tracker_uuid" name="tracker_uuid" class="form-control" type="text" value="{{dict_tracker['uuid']}}" hidden>
{%endif%}
<div class="row">
<div class="col-12 col-xl-9">
<div class="input-group mb-2 mr-sm-2">
<div class="input-group-prepend">
<div class="input-group-text bg-secondary text-white"><i class="fas fa-at"></i></div>
</div>
<input id="mails" name="mails" class="form-control" placeholder="E-Mails Notification (optional, space separated)" type="text" {%if dict_tracker%}{%if dict_tracker['mails']%}value="{{dict_tracker['mails']}}"{%endif%}{%endif%}>
</div>
<div class="input-group mb-2 mr-sm-2">
<div class="input-group-prepend">
<div class="input-group-text bg-info text-white"><i class="fas fa-anchor"></i></div>
</div>
<input id="webhook" name="webhook" class="form-control" placeholder="Webhook URL" type="text" {%if dict_tracker%}{%if dict_tracker['webhook']%}value="{{dict_tracker['webhook']}}"{%endif%}{%endif%}>
</div>
<div class="input-group mb-2 mr-sm-2">
<div class="input-group-prepend">
<div class="input-group-text bg-info text-white"><i class="fas fa-pencil-alt"></i></div>
</div>
<input id="description" name="description" class="form-control" placeholder="Tracker Description (optional)" type="text" {%if dict_tracker%}{%if dict_tracker['description']%}value="{{dict_tracker['description']}}"{%endif%}{%endif%}>
</div>
<div class="input-group mb-2 mr-sm-2">
<div class="input-group-prepend">
<div class="input-group-text bg-dark text-white"><i class="fas fa-folder"></i></div>
</div>
<input id="sources" class="form-control" type="text" name="sources" placeholder="Sources to track (ALL IF EMPTY)" autocomplete="off">
</div>
<div class="card my-4">
<div class="card-header bg-secondary text-white">
<b>Tags</b>
</div>
<div class="card-body">
<div class="input-group mb-2 mr-sm-2">
<div class="input-group-prepend">
<div class="input-group-text bg-danger text-white"><i class="fas fa-tag"></i></div>
</div>
<input id="tags" name="tags" class="form-control" placeholder="Custom Tags (optional, space separated)" type="text" {%if dict_tracker%}{%if dict_tracker['tags']%}value="{{dict_tracker['tags']}}"{%endif%}{%endif%}>
</div>
{% include 'tags/block_tags_selector.html' %}
</div>
</div>
</div>
<div class="col-12 col-xl-3">
<div class="custom-control custom-switch mt-1">
<input class="custom-control-input" type="checkbox" name="level" id="id_level" {%if dict_tracker%}{%if dict_tracker['level']==1%}checked{%endif%}{%else%}checked{%endif%}>
<label class="custom-control-label" for="id_level">
<i class="fas fa-users"></i>&nbsp;Show tracker to all Users
</label>
</div>
</div>
</div>
<hr>
<h4>Tracker Type:</h4>
<select id="tracker_type" name="tracker_type" class="custom-select w-25 mb-3">
<option disabled selected value> -- Select a tracker type -- </option>
<option value="word">Word</option>
<option value="set">Set</option>
<option value="regex">Regex</option>
<option value="yara">YARA rule</option>
<option value="typosquatting">Typo-squatting</option>
</select>
<p id="tracker_desc">Terms to track (space separated)</p>
<div class="row" id="simple_input">
<div class="col-12 col-lg-10">
<input id="tracker" name="tracker" class="form-control" placeholder="Terms to track (space separated)" type="text" {%if dict_tracker%}{%if dict_tracker['tracker']!='yara'%}value="{{dict_tracker['tracker']}}"{%endif%}{%endif%}>
</div>
<div class="col-12 col-lg-2">
<input type="number" id="nb_word" name="nb_word" name="quantity" min="1" placeholder="Nb of keywords" {%if dict_tracker%}{%if dict_tracker['nb_words']%}value="{{dict_tracker['nb_words']}}"{%endif%}{%endif%}>
</div>
</div>
<div class="mb-2" id="yara_rule">
<div class="" id="yara_default_rules">
<h6>Default YARA rules:</h6>
<select class="custom-select w-100 mb-3" id="yara_default_rule" name="yara_default_rule" onchange="get_default_rule_content(this);">
<option selected>Select a default rule</option>
{% for yara_types in all_yara_files %}
{% for yara_file_name in all_yara_files[yara_types] %}
<option value="{{yara_types}}/{{yara_file_name}}">{{yara_types}} - {{yara_file_name}}</option>
{% endfor %}
{% endfor %}
</select>
<pre class="border bg-light" id="default_yara_rule_content"></pre>
</div>
<hr>
<h6>Custom YARA rules:</h6>
<div class="row" id="textarea">
<textarea class="form-control mx-3" id="text_input" name="yara_custom_rule" placeholder="Enter your own YARA rule" rows="5">{%if dict_tracker%}{%if dict_tracker['type']=='yara' and dict_tracker['content']%}{{dict_tracker['content']}}{%endif%}{%endif%}</textarea>
</div>
</div>
<br>
<button class="btn btn-success mt-2">
<i class="fas fa-plus"></i> {%if dict_tracker%}Edit{%else%}Create{%endif%} Tracker
</button>
</form>
</div>
</div>
</div>
</div>
</div>
</body>
<script>
var ltags;
var chart = {};
$(document).ready(function(){
$("#page-Tracker").addClass("active");
$("#nav_manual_crawler").addClass("active");
$("#tracker_desc").hide();
$("#tracker").hide();
$("#nb_word").hide();
$("#yara_rule").hide();
sources = $('#sources').tagSuggest({
data: {{all_sources|safe}},
{%if dict_tracker%}{%if dict_tracker['sources']%}value: {{dict_tracker['sources']|safe}},{%endif%}{%endif%}
sortOrder: 'name',
maxDropHeight: 200,
name: 'sources',
emptyText: 'Sources to track (ALL IF EMPTY)',
});
$('#tracker_type').on('change', function() {
var tracker_type = this.value;
if (tracker_type=="word") {
$("#tracker_desc").text("Token to track. You need to use a regex if you want to use one of the following special characters [<>~!?@#$%^&*|()_-+={}\":;,.\'\n\r\t]/\\ ");
$("#tracker_desc").show();
$("#tracker").show();
$("#nb_word").hide();
$("#yara_rule").hide();
} else if (tracker_type=="set") {
$("#tracker_desc").text("Set of Terms to track (space separated). This tracker is used to check if an item contain one or more terms specified in a set. If an item contain NB unique terms (by default NB of unique keywords = 1), this tracker is triggered. You need to use a regex if you want to use one of the following special characters [<>~!?@#$%^&*|()_-+={}\":;,.\'\n\r\t]/\\ ");
$("#tracker_desc").show();
$("#tracker").show();
$("#nb_word").show();
$("#yara_rule").hide();
} else if (tracker_type=="regex") {
$("#tracker_desc").text("Enter a valid Python regex");
$("#tracker_desc").show();
$("#tracker").show();
$("#nb_word").hide();
$("#yara_rule").hide();
} else if (tracker_type=="yara") {
$("#tracker_desc").text("Select a default yara rule or create your own rule:");
$("#tracker_desc").show();
$("#tracker").hide();
$("#nb_word").hide();
$("#yara_rule").show();
} else if (tracker_type=="typosquatting") {
$("#tracker_desc").text("Generation of variation for domain name. Only one domain name at a time.");
$("#tracker_desc").show();
$("#tracker").show();
$("#nb_word").hide();
$("#yara_rule").hide();
}
});
{%if dict_tracker%}
$('#tracker_type').val('{{dict_tracker['type']}}').change();
{%if dict_tracker['type']=='yara' and dict_tracker['yara_file']%}
$('#yara_default_rule').val('{{dict_tracker['yara_file']}}').change();
{%endif%}
{%endif%}
});
function toggle_sidebar(){
if($('#nav_menu').is(':visible')){
$('#nav_menu').hide();
$('#side_menu').removeClass('border-right')
$('#side_menu').removeClass('col-lg-2')
$('#core_content').removeClass('col-lg-10')
}else{
$('#nav_menu').show();
$('#side_menu').addClass('border-right')
$('#side_menu').addClass('col-lg-2')
$('#core_content').addClass('col-lg-10')
}
}
function get_default_rule_content(selector){
var yara_name = selector.value
if (yara_name === "Select a default rule") {
jQuery("#default_yara_rule_content").text("")
} else {
$.getJSON("{{ url_for('hunter.get_default_yara_rule_content') }}?rule_name=" + yara_name,
function(data) {
jQuery("#default_yara_rule_content").text(data['content'])
});
}
}
</script>

124
var/www/modules/hunter/templates/showTracker.html
View File

@ -72,7 +72,7 @@
<tr> <tr>
<th>Type</th> <th>Type</th>
<th>Tracker</th> <th>Tracker</th>
<th>Date added</th> <th>Created</th>
<th>Access Level</th> <th>Access Level</th>
<th>Created by</th> <th>Created by</th>
<th>First seen</th> <th>First seen</th>
@ -94,7 +94,7 @@
{% if tracker_metadata['type'] == 'typosquatting' %} {% if tracker_metadata['type'] == 'typosquatting' %}
<td> <td>
<a class="btn btn-primary" data-toggle="collapse" href="#collapseTypo" role="button" aria-expanded="false" aria-controls="collapseTypo"> <a class="btn btn-primary" data-toggle="collapse" href="#collapseTypo" role="button" aria-expanded="false" aria-controls="collapseTypo">
{{ tracker_metadata['tracker'].split(",")[0] }} {{ tracker_metadata['tracked'].split(",")[0] }}
</a> </a>
<div class="collapse" id="collapseTypo"> <div class="collapse" id="collapseTypo">
<div class="card card-body"> <div class="card card-body">
@ -108,7 +108,7 @@
</div> </div>
</td> </td>
{% else %} {% else %}
<td>{{ tracker_metadata['tracker'] }}</td> <td>{{ tracker_metadata['tracked'] }}</td>
{% endif %} {% endif %}
<td>{{ tracker_metadata['date'][0:4] }}/{{ tracker_metadata['date'][4:6] }}/{{ tracker_metadata['date'][6:8] }}</td> <td>{{ tracker_metadata['date'][0:4] }}/{{ tracker_metadata['date'][4:6] }}/{{ tracker_metadata['date'][6:8] }}</td>
<td> <td>
@ -118,19 +118,15 @@
Global Global
{% endif %} {% endif %}
</td> </td>
<td>{{ tracker_metadata['user_id'] }}</td> <td>{{ tracker_metadata['user'] }}</td>
<td> <td>
{% if tracker_metadata['first_seen'] %} {% if tracker_metadata['first_seen'] %}
{{ tracker_metadata['first_seen'][0:4] }}/ {{ tracker_metadata['first_seen'][0:4] }}/{{ tracker_metadata['first_seen'][4:6] }}/{{ tracker_metadata['first_seen'][6:8] }}
{{ tracker_metadata['first_seen'][4:6] }}/
{{ tracker_metadata['first_seen'][6:8] }}
{% endif %} {% endif %}
</td> </td>
<td> <td>
{% if tracker_metadata['last_seen'] %} {% if tracker_metadata['last_seen'] %}
{{ tracker_metadata['last_seen'][0:4] }}/ {{ tracker_metadata['last_seen'][0:4] }}/{{ tracker_metadata['last_seen'][4:6] }}/{{ tracker_metadata['last_seen'][6:8] }}
{{ tracker_metadata['last_seen'][4:6] }}/
{{ tracker_metadata['last_seen'][6:8] }}
{% endif %} {% endif %}
</td> </td>
{% if tracker_metadata['webhook'] %} {% if tracker_metadata['webhook'] %}
@ -159,14 +155,22 @@
<div id="sparkline"></div> <div id="sparkline"></div>
</div> </div>
</div> </div>
<h6>Sources:</h6>
{% if tracker_metadata['sources'] %} <h6>Filters:</h6>
{% for sources in tracker_metadata['sources'] %} {% if tracker_metadata['filters'] %}
<span class="badge badge-secondary">{{ sources }}</span><br> <pre>{{ tracker_metadata['filters'] }}</pre>
{% endfor %}
{% else %} {% else %}
<span class="badge badge-secondary">All Souces</span><br> <span class="badge badge-secondary">No Filters</span><br>
{% endif %} {% endif %}
{# <h6>Sources:</h6>#}
{# {% if tracker_metadata['sources'] %}#}
{# {% for sources in tracker_metadata['sources'] %}#}
{# <span class="badge badge-secondary">{{ sources }}</span><br>#}
{# {% endfor %}#}
{# {% else %}#}
{# <span class="badge badge-secondary">All Sources</span><br>#}
{# {% endif %}#}
</li> </li>
</ul> </ul>
@ -232,7 +236,7 @@
</div> </div>
<div class="d-flex flex-row-reverse"> <div class="d-flex flex-row-reverse">
<a href="{{ url_for('hunter.delete_tracker') }}?uuid={{ tracker_metadata['uuid'] }}" <a href="{{ url_for('hunters.tracker_delete') }}?uuid={{ tracker_metadata['uuid'] }}"
style="font-size: 15px"> style="font-size: 15px">
<button class='btn btn-danger'><i class="fas fa-trash-alt"></i></button> <button class='btn btn-danger'><i class="fas fa-trash-alt"></i></button>
</a> </a>
@ -262,8 +266,9 @@
class="far fa-calendar-alt" aria-hidden="true"></i></span></div> class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
<input class="form-control" id="date-range-from-input" placeholder="yyyy-mm-dd" <input class="form-control" id="date-range-from-input" placeholder="yyyy-mm-dd"
name="date_from" autocomplete="off" name="date_from" autocomplete="off"
{% if tracker_metadata['date_from'] %}value="{{ tracker_metadata['date_from'] }}" {% if tracker_metadata['date_from'] %}value="{{ tracker_metadata['date_from'][0:4] }}-{{ tracker_metadata['date_from'][4:6] }}-{{ tracker_metadata['date_from'][6:8] }}"
{% else %}value="{{ tracker_metadata['first_seen'] }}"{% endif %}> {% elif tracker_metadata['first_seen'] %}value="{{ tracker_metadata['first_seen'][0:4] }}-{{ tracker_metadata['first_seen'][4:6] }}-{{ tracker_metadata['first_seen'][6:8] }}"
{% endif %}>
</div> </div>
</div> </div>
<div class="col-md-6"> <div class="col-md-6">
@ -272,8 +277,9 @@
class="far fa-calendar-alt" aria-hidden="true"></i></span></div> class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
<input class="form-control" id="date-range-to-input" placeholder="yyyy-mm-dd" <input class="form-control" id="date-range-to-input" placeholder="yyyy-mm-dd"
name="date_to" autocomplete="off" name="date_to" autocomplete="off"
{% if tracker_metadata['date_to'] %}value="{{ tracker_metadata['date_to'] }}" {% if tracker_metadata['date_to'] %}value="{{ tracker_metadata['date_to'][0:4] }}-{{ tracker_metadata['date_to'][4:6] }}-{{ tracker_metadata['date_to'][6:8] }}"
{% else %}value="{{ tracker_metadata['last_seen'] }}"{% endif %}> {% elif tracker_metadata['last_seen'] %}value="{{ tracker_metadata['last_seen'][0:4] }}-{{ tracker_metadata['last_seen'][4:6] }}-{{ tracker_metadata['last_seen'][6:8] }}"
{% endif %}>
</div> </div>
</div> </div>
</div> </div>
@ -285,40 +291,52 @@
</div> </div>
</div> </div>
{% if tracker_metadata['items'] %} {% if tracker_metadata['objs'] %}
<div class="mt-4"> <div class="mt-4">
<table class="table table-bordered table-hover" id="myTable_"> <table id="myTable_" class="table table-striped border-primary">
<thead class="thead-dark"> <thead class="bg-dark text-white">
<tr> <tr>
<th>Date</th> <th>Type</th>
<th>Item Id</th> <th></th>
</tr> <th>Id</th>
</thead> <th>Tags</th>
<tbody> <th></th>
</tr>
</thead>
<tbody style="font-size: 15px;">
{% for object in tracker_metadata['objs'] %}
<tr class="border-color: blue;">
<td>
{% with style=object['icon']['style'], icon=object['icon']['icon'] , color=object['icon']['color'] %}
{% include 'objects/obj_svg_block.html' %}
{% endwith %}
{{ object['type']}}
</td>
<td>
{% if object['subtype'] %}
{{ object['subtype']}}
{% endif %}
</td>
<td>
<a href="{{ object['link'] }}">
{{ object['id']}}
</a>
</td>
<td>
{% for tag in object['tags'] %}
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }}</span>
{% endfor %}
</td>
<td class="text-right">
{# <a href="{{ url_for('investigations_b.unregister_investigation') }}?uuid={{ tracker_metadata['uuid']}}&type={{ object['type'] }}&subtype={{ object['subtype']}}&id={{ object['id']}}">#}
{# <button type="button" class="btn btn-danger"><i class="fas fa-trash-alt"></i></button>#}
{# </a>#}
</td>
</tr>
{% endfor %}
</tbody>
</table>
{% for item in tracker_metadata['items'] %}
<tr>
<td>
{{ item['date'][0:4] }}/{{ item['date'][4:6] }}/{{ item['date'][6:8] }}
</td>
<td>
<a class="text-secondary" target="_blank"
href="{{ url_for('objects_item.showItem') }}?id={{ item['id'] }}">
<div style="line-height:0.9;">{{ item['id'] }}</div>
</a>
<div class="mb-2">
{% for tag in item['tags'] %}
<a href="{{ url_for('tags_ui.get_obj_by_tags') }}?object_type=item&ltags={{ tag }}">
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }}</span>
</a>
{% endfor %}
</div>
</td>
</tr>
{% endfor %}
</tbody>
</table>
</div> </div>
{% endif %} {% endif %}
@ -331,7 +349,7 @@
$('#div_edit_mails').hide(); $('#div_edit_mails').hide();
$('#div_edit_tags').hide(); $('#div_edit_tags').hide();
$('#div_edit_description').hide(); $('#div_edit_description').hide();
$("#page-Decoded").addClass("active"); $("#page-Tracker").addClass("active");
$('#date-range-from').dateRangePicker({ $('#date-range-from').dateRangePicker({
separator: ' to ', separator: ' to ',

64
var/www/modules/restApi/Flask_restApi.py
View File

@ -22,8 +22,6 @@ from lib.objects import Items
from lib import Tag from lib import Tag
from lib import Tracker from lib import Tracker
from packages import Term # TODO REMOVE ME
from packages import Import_helper from packages import Import_helper
from importer.FeederImporter import api_add_json_feeder_to_queue from importer.FeederImporter import api_add_json_feeder_to_queue
@ -324,9 +322,10 @@ def get_all_tags():
res = {'tags': Tag.get_all_tags()} res = {'tags': Tag.get_all_tags()}
return Response(json.dumps(res, indent=2, sort_keys=True), mimetype='application/json'), 200 return Response(json.dumps(res, indent=2, sort_keys=True), mimetype='application/json'), 200
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # TODO
# # # # # # # # # # # # # # TRACKER # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # TRACKER # # # # # # # # # # # # # # # # # TODO
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # TODO
'''
@restApi.route("api/v1/add/tracker", methods=['POST']) @restApi.route("api/v1/add/tracker", methods=['POST'])
@token_required('analyst') @token_required('analyst')
def add_tracker_term(): def add_tracker_term():
@ -371,8 +370,21 @@ def get_tracker_metadata_api():
data = request.get_json() data = request.get_json()
tracker_uuid = data.get('tracker_uuid', None) tracker_uuid = data.get('tracker_uuid', None)
req_data = {'tracker_uuid': tracker_uuid} req_data = {'tracker_uuid': tracker_uuid}
tracker_uuid = request_dict.get('tracker_uuid', None)
if not request_dict:
return {'status': 'error', 'reason': 'Malformed JSON'}, 400
if not tracker_uuid:
return {'status': 'error', 'reason': 'Mandatory parameter(s) not provided'}, 400
if not is_valid_uuid_v4(tracker_uuid):
return {"status": "error", "reason": "Invalid Tracker UUID"}, 400
if not r_serv_tracker.exists(f'tracker:{tracker_uuid}'):
return {'status': 'error', 'reason': 'Tracker not found'}, 404
res = Tracker.get_tracker_metadata_api(req_data) res = Tracker.get_tracker_metadata_api(req_data)
return Response(json.dumps(res[0], indent=2, sort_keys=False), mimetype='application/json'), res[1] return Response(json.dumps(res[0], indent=2, sort_keys=False), mimetype='application/json'), res[1]
'''
''' '''
@ -530,27 +542,27 @@ def get_domain_status_minimal():
res[0]['domain'] = domain res[0]['domain'] = domain
return create_json_response(res[0], res[1]) return create_json_response(res[0], res[1])
@restApi.route("api/v1/get/crawled/domain/list", methods=['POST']) # @restApi.route("api/v1/get/crawled/domain/list", methods=['POST'])
@token_required('analyst') # @token_required('analyst')
def get_crawled_domain_list(): # def get_crawled_domain_list():
data = request.get_json() # data = request.get_json()
res = get_mandatory_fields(data, ['date_from', 'date_to']) # res = get_mandatory_fields(data, ['date_from', 'date_to'])
if res: # if res:
return create_json_response(res[0], res[1]) # return create_json_response(res[0], res[1])
#
date_from = data.get('date_from', None) # date_from = data.get('date_from', None)
date_to = data.get('date_to', None) # date_to = data.get('date_to', None)
domain_type = data.get('domain_type', None) # domain_type = data.get('domain_type', None)
domain_status = 'UP' # domain_status = 'UP'
# TODO TO MIGRATE # # TODO TO MIGRATE
raise Exception('TO MIGRATE') # raise Exception('TO MIGRATE')
# res = Domain.api_get_domains_by_status_daterange(date_from, date_to, domain_type) # # res = Domain.api_get_domains_by_status_daterange(date_from, date_to, domain_type)
dict_res = res[0] # dict_res = res[0]
dict_res['date_from'] = date_from # dict_res['date_from'] = date_from
dict_res['date_to'] = date_to # dict_res['date_to'] = date_to
dict_res['domain_status'] = domain_status # dict_res['domain_status'] = domain_status
dict_res['domain_type'] = domain_type # dict_res['domain_type'] = domain_type
return create_json_response(dict_res, res[1]) # return create_json_response(dict_res, res[1])
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# # # # # # # # # # # # # IMPORT # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # IMPORT # # # # # # # # # # # # # # # # # #

14
var/www/templates/hunter/menu_sidebar.html
View File

@ -8,43 +8,43 @@
<nav class="navbar navbar-expand navbar-light bg-light flex-md-column flex-row align-items-start py-2" id="nav_menu"> <nav class="navbar navbar-expand navbar-light bg-light flex-md-column flex-row align-items-start py-2" id="nav_menu">
<h5 class="d-flex text-muted w-100" id="nav_title_trackers"> <h5 class="d-flex text-muted w-100" id="nav_title_trackers">
<span>Trackers </span> <span>Trackers </span>
<a class="ml-auto" href="{{url_for('hunter.add_tracked_menu')}}"> <a class="ml-auto" href="{{url_for('hunters.add_tracked_menu')}}">
<i class="fas fa-plus-circle ml-auto"></i> <i class="fas fa-plus-circle ml-auto"></i>
</a> </a>
</h5> </h5>
<ul class="nav flex-md-column flex-row navbar-nav justify-content-between w-100"> <!--nav-pills--> <ul class="nav flex-md-column flex-row navbar-nav justify-content-between w-100"> <!--nav-pills-->
<li class="nav-item"> <li class="nav-item">
<a class="nav-link" href="{{url_for('hunter.tracked_menu')}}" id="nav_tracker_"> <a class="nav-link" href="{{url_for('hunters.tracked_menu')}}" id="nav_tracker_">
<i class="fas fa-ruler-combined"></i> <i class="fas fa-ruler-combined"></i>
<span>All Trackers</span> <span>All Trackers</span>
</a> </a>
</li> </li>
<li class="nav-item"> <li class="nav-item">
<a class="nav-link" href="{{url_for('hunter.tracked_menu_word')}}" id="nav_tracker_word"> <a class="nav-link" href="{{url_for('hunters.tracked_menu_word')}}" id="nav_tracker_word">
<i class="fas fa-font"></i> <i class="fas fa-font"></i>
<span>Words</span> <span>Words</span>
</a> </a>
</li> </li>
<li class="nav-item"> <li class="nav-item">
<a class="nav-link" href="{{url_for('hunter.tracked_menu_set')}}" id="nav_tracker_set"> <a class="nav-link" href="{{url_for('hunters.tracked_menu_set')}}" id="nav_tracker_set">
<i class="fas fa-layer-group"></i> <i class="fas fa-layer-group"></i>
<span>Set</span> <span>Set</span>
</a> </a>
</li> </li>
<li class="nav-item"> <li class="nav-item">
<a class="nav-link" href="{{url_for('hunter.tracked_menu_regex')}}" id="nav_tracker_regex"> <a class="nav-link" href="{{url_for('hunters.tracked_menu_regex')}}" id="nav_tracker_regex">
<i class="fas fa-drafting-compass"></i> <i class="fas fa-drafting-compass"></i>
<span>Regex</span> <span>Regex</span>
</a> </a>
</li> </li>
<li class="nav-item"> <li class="nav-item">
<a class="nav-link" href="{{url_for('hunter.tracked_menu_yara')}}" id="nav_tracker_yara"> <a class="nav-link" href="{{url_for('hunters.tracked_menu_yara')}}" id="nav_tracker_yara">
<span class="bg-danger text-white font-weight-bold" style="font-size: 120%">&nbsp;{ </span> <span class="bg-danger text-white font-weight-bold" style="font-size: 120%">&nbsp;{ </span>
<span>&nbsp;YARA</span> <span>&nbsp;YARA</span>
</a> </a>
</li> </li>
<li class="nav-item"> <li class="nav-item">
<a class="nav-link" href="{{url_for('hunter.tracked_menu_typosquatting')}}" id="nav_tracker_typosquatting"> <a class="nav-link" href="{{url_for('hunters.tracked_menu_typosquatting')}}" id="nav_tracker_typosquatting">
<i class="fa fa-clone"></i> <i class="fa fa-clone"></i>
<span>Typo-squatting</span> <span>Typo-squatting</span>
</a> </a>

396
var/www/templates/hunter/tracker_add.html Normal file
View File

@ -0,0 +1,396 @@
<!DOCTYPE html>
<html>
<head>
<title>AIL-Framework</title>
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png')}}">
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/daterangepicker.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/tags.css') }}" rel="stylesheet">
<!-- JS -->
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/tags.js') }}"></script>
</head>
<body>
{% include 'nav_bar.html' %}
<div class="container-fluid">
<div class="row">
{% include 'hunter/menu_sidebar.html' %}
<div class="col-12 col-lg-10" id="core_content">
<div class="card my-3">
<div class="card-header bg-dark text-white">
<h5 class="card-title">{%if dict_tracker%}Edit a{%else%}Create a new{%endif%} Tracker</h5>
</div>
<div class="card-body">
<form action="{{ url_for('hunters.add_tracked_menu') }}" method='post'>
{%if dict_tracker%}
<input id="tracker_uuid" name="tracker_uuid" class="form-control" type="text" value="{{dict_tracker['uuid']}}" hidden>
{%endif%}
<div class="row">
<div class="col-12 col-xl-9">
<div class="input-group mb-2 mr-sm-2">
<div class="input-group-prepend">
<div class="input-group-text bg-secondary text-white"><i class="fas fa-at"></i></div>
</div>
<input id="mails" name="mails" class="form-control" placeholder="E-Mails Notification (optional, space separated)" type="text" {%if dict_tracker%}{%if dict_tracker['mails']%}value="{{dict_tracker['mails']}}"{%endif%}{%endif%}>
</div>
<div class="input-group mb-2 mr-sm-2">
<div class="input-group-prepend">
<div class="input-group-text bg-info text-white"><i class="fas fa-anchor"></i></div>
</div>
<input id="webhook" name="webhook" class="form-control" placeholder="Webhook URL" type="text" {%if dict_tracker%}{%if dict_tracker['webhook']%}value="{{dict_tracker['webhook']}}"{%endif%}{%endif%}>
</div>
<div class="input-group mb-2 mr-sm-2">
<div class="input-group-prepend">
<div class="input-group-text bg-info text-white"><i class="fas fa-pencil-alt"></i></div>
</div>
<input id="description" name="description" class="form-control" placeholder="Tracker Description (optional)" type="text" {%if dict_tracker%}{%if dict_tracker['description']%}value="{{dict_tracker['description']}}"{%endif%}{%endif%}>
</div>
<div class="card my-4">
<div class="card-header bg-info text-white">
<b>Objects to Track:</b>
</div>
<div class="card-body">
{# <div class="custom-control custom-switch mt-1">#}
{# <input class="custom-control-input" type="checkbox" name="cve_obj" id="cve_obj" checked="">#}
{# <label class="custom-control-label" for="cve_obj"><i class="fas fa-bug"></i>&nbsp;CVE</label>#}
{# </div>#}
{# <div class="custom-control custom-switch mt-1">#}
{# <input class="custom-control-input" type="checkbox" name="crypto_obj" id="crypto_obj" checked="">#}
{# <label class="custom-control-label" for="crypto_obj"><i class="fas fa-coins"></i>&nbsp;Cryptocurrency</label>#}
{# </div>#}
<div class="custom-control custom-switch mt-1">
<input class="custom-control-input" type="checkbox" name="decoded_obj" id="decoded_obj" checked="">
<label class="custom-control-label" for="decoded_obj"><i class="fas fa-lock-open"></i>&nbsp;Decoded</label>
</div>
{# <div class="custom-control custom-switch mt-1">#}
{# <input class="custom-control-input" type="checkbox" name="domain_obj" id="domain_obj" checked="">#}
{# <label class="custom-control-label" for="domain_obj"><i class="fas fa-spider"></i>&nbsp;Domain</label>#}
{# </div>#}
<div class="custom-control custom-switch mt-1">
<input class="custom-control-input" type="checkbox" name="item_obj" id="item_obj" checked="">
<label class="custom-control-label" for="item_obj"><i class="fas fa-file"></i>&nbsp;Item</label>
</div>
<div class="card border-dark mb-4" id="sources_item_div">
<div class="card-body">
<h5>Filter Item by sources</h5>
<div class="input-group mb-2 mr-sm-2">
<div class="input-group-prepend">
<div class="input-group-text bg-dark text-white"><i class="fas fa-folder"></i></div>
</div>
<input id="sources_item" class="form-control" type="text" name="sources_item" placeholder="Item Sources to track (ALL IF EMPTY)" autocomplete="off">
</div>
</div>
</div>
<div class="custom-control custom-switch mt-1">
<input class="custom-control-input" type="checkbox" name="pgp_obj" id="pgp_obj" checked="">
<label class="custom-control-label" for="pgp_obj"><i class="fas fa-key"></i>&nbsp;PGP</label>
</div>
<div class="card border-dark mb-4" id="sources_pgp_div">
<div class="card-body">
<h6>Filter PGP by subtype:</h6>
<div class="custom-control custom-switch mt-1">
<input class="custom-control-input" type="checkbox" name="filter_pgp_name" id="filter_pgp_name" checked="">
<label class="custom-control-label" for="filter_pgp_name">
<svg height="26" width="26">
<g class="nodes">
<circle cx="13" cy="13" r="13" fill="#44AA99"></circle>
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fas" font-size="16px">&#xf507;</text>
</g>
</svg>
name
</label>
</div>
<div class="custom-control custom-switch mt-1">
<input class="custom-control-input" type="checkbox" name="filter_pgp_mail" id="filter_pgp_mail" checked="">
<label class="custom-control-label" for="filter_pgp_mail">
<svg height="26" width="26">
<g class="nodes">
<circle cx="13" cy="13" r="13" fill="#44AA99"></circle>
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="fas" font-size="16px">&#xf1fa;</text>
</g>
</svg>
mail
</label>
</div>
</div>
</div>
{# <div class="custom-control custom-switch mt-1">#}
{# <input class="custom-control-input" type="checkbox" name="level" id="screenshot_obj" checked="">#}
{# <label class="custom-control-label" for="screenshot_obj"><i class="fas fa-image"></i>&nbsp;Screenshot</label>#}
{# </div>#}
{# <div class="custom-control custom-switch mt-1">#}
{# <input class="custom-control-input" type="checkbox" name="level" id="username_obj" checked="">#}
{# <label class="custom-control-label" for="username_obj"><i class="fas fa-user"></i>&nbsp;Username</label>#}
{# </div>#}
{# <li class="list-group-item text-left">#}
{# <div class="form-check">#}
{# <input class="form-check-input" type="checkbox" value="True" id="CveCheck" name="CveCheck" checked="">#}
{# <label class="form-check-label" for="CveCheck"><i class="fas fa-bug"></i> Cve</label>#}
{# </div>#}
{# <div class="form-check">#}
{# <input class="form-check-input" type="checkbox" value="True" id="CryptocurrencyCheck" name="CryptocurrencyCheck" checked="">#}
{# <label class="form-check-label" for="CryptocurrencyCheck"><i class="fas fa-coins"></i> Cryptocurrency</label>#}
{# </div>#}
{# <div class="form-check">#}
{# <input class="form-check-input" type="checkbox" value="True" id="DecodedCheck" name="DecodedCheck" checked="">#}
{# <label class="form-check-label" for="DecodedCheck"><i class="fas fa-lock-open"></i> Decoded</label>#}
{# </div>#}
{# <div class="form-check">#}
{# <input class="form-check-input" type="checkbox" value="True" id="ScreenshotCheck" name="ScreenshotCheck" checked="">#}
{# <label class="form-check-label" for="ScreenshotCheck"><i class="fas fa-image"></i> Screenshot</label>#}
{# </div>#}
{# <div class="form-check">#}
{# <input class="form-check-input" type="checkbox" value="True" id="PgpCheck" name="PgpCheck" checked="">#}
{# <label class="form-check-label" for="PgpCheck"><i class="fas fa-key"></i> PGP</label>#}
{# </div>#}
{# <div class="form-check">#}
{# <input class="form-check-input" type="checkbox" value="True" id="UsernameCheck" name="UsernameCheck" checked="">#}
{# <label class="form-check-label" for="UsernameCheck"><i class="fas fa-user"></i> Username</label>#}
{# </div>#}
{# <div class="form-check">#}
{# <input class="form-check-input" type="checkbox" value="True" id="DomainCheck" name="DomainCheck" checked="">#}
{# <label class="form-check-label" for="DomainCheck"><i class="fas fa-spider"></i> Domain</label>#}
{# </div>#}
{# <div class="form-check">#}
{# <input class="form-check-input" type="checkbox" value="True" id="ItemCheck" name="ItemCheck" checked="">#}
{# <label class="form-check-label" for="ItemCheck"><i class="fas fa-user"></i> Item</label>#}
{# </div>#}
{##}
{# </li>#}
</div>
</div>
<div class="card my-4">
<div class="card-header bg-secondary text-white">
<b>Tags</b>
</div>
<div class="card-body">
<div class="input-group mb-2 mr-sm-2">
<div class="input-group-prepend">
<div class="input-group-text bg-danger text-white"><i class="fas fa-tag"></i></div>
</div>
<input id="tags" name="tags" class="form-control" placeholder="Custom Tags (optional, space separated)" type="text" {%if dict_tracker%}{%if dict_tracker['tags']%}value="{{dict_tracker['tags']}}"{%endif%}{%endif%}>
</div>
{% include 'tags/block_tags_selector.html' %}
</div>
</div>
</div>
<div class="col-12 col-xl-3">
<div class="custom-control custom-switch mt-1">
<input class="custom-control-input" type="checkbox" name="level" id="id_level" {%if dict_tracker%}{%if dict_tracker['level']==1%}checked{%endif%}{%else%}checked{%endif%}>
<label class="custom-control-label" for="id_level">
<i class="fas fa-users"></i>&nbsp;Show tracker to all Users
</label>
</div>
</div>
</div>
<hr>
<h4>Tracker Type:</h4>
<select id="tracker_type" name="tracker_type" class="custom-select w-25 mb-3">
<option disabled selected value> -- Select a tracker type -- </option>
<option value="word">Word</option>
<option value="set">Set</option>
<option value="regex">Regex</option>
<option value="yara">YARA rule</option>
<option value="typosquatting">Typo-squatting</option>
</select>
<p id="tracker_desc">Terms to track (space separated)</p>
<div class="row" id="simple_input">
<div class="col-12 col-lg-10">
<input id="tracker" name="tracker" class="form-control" placeholder="Terms to track (space separated)" type="text" {%if dict_tracker%}{%if dict_tracker['tracker']!='yara'%}value="{{dict_tracker['tracker']}}"{%endif%}{%endif%}>
</div>
<div class="col-12 col-lg-2">
<input type="number" id="nb_word" name="nb_word" name="quantity" min="1" placeholder="Nb of keywords" {%if dict_tracker%}{%if dict_tracker['nb_words']%}value="{{dict_tracker['nb_words']}}"{%endif%}{%endif%}>
</div>
</div>
<div class="mb-2" id="yara_rule">
<div class="" id="yara_default_rules">
<h6>Default YARA rules:</h6>
<select class="custom-select w-100 mb-3" id="yara_default_rule" name="yara_default_rule" onchange="get_default_rule_content(this);">
<option selected>Select a default rule</option>
{% for yara_types in all_yara_files %}
{% for yara_file_name in all_yara_files[yara_types] %}
<option value="{{yara_types}}/{{yara_file_name}}">{{yara_types}} - {{yara_file_name}}</option>
{% endfor %}
{% endfor %}
</select>
<pre class="border bg-light" id="default_yara_rule_content"></pre>
</div>
<hr>
<h6>Custom YARA rules:</h6>
<div class="row" id="textarea">
<textarea class="form-control mx-3" id="text_input" name="yara_custom_rule" placeholder="Enter your own YARA rule" rows="5">{%if dict_tracker%}{%if dict_tracker['type']=='yara' and dict_tracker['content']%}{{dict_tracker['content']}}{%endif%}{%endif%}</textarea>
</div>
</div>
<br>
<button class="btn btn-success mt-2">
<i class="fas fa-plus"></i> {%if dict_tracker%}Edit{%else%}Create{%endif%} Tracker
</button>
</form>
</div>
</div>
</div>
</div>
</div>
</body>
<script>
var ltags;
var chart = {};
$(document).ready(function(){
$("#page-Tracker").addClass("active");
$("#nav_manual_crawler").addClass("active");
$("#tracker_desc").hide();
$("#tracker").hide();
$("#nb_word").hide();
$("#yara_rule").hide();
sources_item = $('#sources_item').tagSuggest({
data: {{all_sources|safe}},
{%if dict_tracker%}{%if dict_tracker['item_sources']%}value: {{dict_tracker['item_sources']|safe}},{%endif%}{%endif%}
sortOrder: 'name',
maxDropHeight: 200,
name: 'sources_item',
emptyText: 'Item Sources to track (ALL IF EMPTY)',
});
$('#tracker_type').on('change', function() {
var tracker_type = this.value;
if (tracker_type=="word") {
$("#tracker_desc").text("Token to track. You need to use a regex if you want to use one of the following special characters [<>~!?@#$%^&*|()_-+={}\":;,.\'\n\r\t]/\\ ");
$("#tracker_desc").show();
$("#tracker").show();
$("#nb_word").hide();
$("#yara_rule").hide();
} else if (tracker_type=="set") {
$("#tracker_desc").text("Set of Terms to track (space separated). This tracker is used to check if an item contain one or more terms specified in a set. If an item contain NB unique terms (by default NB of unique keywords = 1), this tracker is triggered. You need to use a regex if you want to use one of the following special characters [<>~!?@#$%^&*|()_-+={}\":;,.\'\n\r\t]/\\ ");
$("#tracker_desc").show();
$("#tracker").show();
$("#nb_word").show();
$("#yara_rule").hide();
} else if (tracker_type=="regex") {
$("#tracker_desc").text("Enter a valid Python regex");
$("#tracker_desc").show();
$("#tracker").show();
$("#nb_word").hide();
$("#yara_rule").hide();
} else if (tracker_type=="yara") {
$("#tracker_desc").text("Select a default yara rule or create your own rule:");
$("#tracker_desc").show();
$("#tracker").hide();
$("#nb_word").hide();
$("#yara_rule").show();
} else if (tracker_type=="typosquatting") {
$("#tracker_desc").text("Generation of variation for domain name. Only one domain name at a time.");
$("#tracker_desc").show();
$("#tracker").show();
$("#nb_word").hide();
$("#yara_rule").hide();
}
});
{%if dict_tracker%}
$('#tracker_type').val('{{dict_tracker['type']}}').change();
{%if dict_tracker['type']=='yara' and dict_tracker['yara_file']%}
$('#yara_default_rule').val('{{dict_tracker['yara_file']}}').change();
{%endif%}
{%endif%}
$('#item_obj').on("change", function () {
item_source_input_controller();
});
$('#pgp_obj').on("change", function () {
pgp_source_input_controller();
});
});
function toggle_sidebar(){
if($('#nav_menu').is(':visible')){
$('#nav_menu').hide();
$('#side_menu').removeClass('border-right')
$('#side_menu').removeClass('col-lg-2')
$('#core_content').removeClass('col-lg-10')
}else{
$('#nav_menu').show();
$('#side_menu').addClass('border-right')
$('#side_menu').addClass('col-lg-2')
$('#core_content').addClass('col-lg-10')
}
}
function item_source_input_controller() {
if($('#item_obj').is(':checked')){
$("#sources_item_div").show();
}else{
$("#sources_item_div").hide();
}
}
function pgp_source_input_controller() {
if($('#pgp_obj').is(':checked')){
$("#sources_pgp_div").show();
}else{
$("#sources_pgp_div").hide();
}
}
function get_default_rule_content(selector){
var yara_name = selector.value
if (yara_name === "Select a default rule") {
jQuery("#default_yara_rule_content").text("")
} else {
$.getJSON("{{ url_for('hunter.get_default_yara_rule_content') }}?rule_name=" + yara_name,
function(data) {
jQuery("#default_yara_rule_content").text(data['content'])
});
}
}
</script>

18
var/www/modules/hunter/templates/trackersManagement.html → var/www/templates/hunter/trackersManagement.html
View File

@ -71,11 +71,11 @@
<td> <td>
<span> <span>
<a target="_blank" href="{{ url_for('hunter.show_tracker') }}?uuid={{ dict_uuid['uuid'] }}"> <a target="_blank" href="{{ url_for('hunter.show_tracker') }}?uuid={{ dict_uuid['uuid'] }}">
{% if dict_uuid['tracker'] %} {% if dict_uuid['tracked'] %}
{% if dict_uuid['tracker']|length > 256 %} {% if dict_uuid['tracked']|length > 256 %}
{{ dict_uuid['tracker'][0:256] }}... {{ dict_uuid['tracked'][0:256] }}...
{% else %} {% else %}
{{ dict_uuid['tracker'] }} {{ dict_uuid['tracked'] }}
{% endif %} {% endif %}
{% endif %} {% endif %}
</a> </a>
@ -136,11 +136,11 @@
<td> <td>
<span> <span>
<a target="_blank" href="{{ url_for('hunter.show_tracker') }}?uuid={{ dict_uuid['uuid'] }}"> <a target="_blank" href="{{ url_for('hunter.show_tracker') }}?uuid={{ dict_uuid['uuid'] }}">
{% if dict_uuid['tracker'] %} {% if dict_uuid['tracked'] %}
{% if dict_uuid['tracker']|length > 256 %} {% if dict_uuid['tracked']|length > 256 %}
{{ dict_uuid['tracker'][0:256] }}... {{ dict_uuid['tracked'][0:256] }}...
{% else %} {% else %}
{{ dict_uuid['tracker'] }} {{ dict_uuid['tracked'] }}
{% endif %} {% endif %}
{% endif %} {% endif %}
</a> </a>
@ -178,7 +178,7 @@
</div> </div>
<a class="btn btn-info my-4" href="{{ url_for('hunter.add_tracked_menu') }}"> <a class="btn btn-info my-4" href="{{ url_for('hunters.add_tracked_menu') }}">
<i class="fas fa-plus-circle ml-auto"></i> <i class="fas fa-plus-circle ml-auto"></i>
Create New Tracker Create New Tracker
</a> </a>

174
var/www/templates/hunter/trackers_dashboard.html Normal file
View File

@ -0,0 +1,174 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Trackers - AIL</title>
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
<script src="{{ url_for('static', filename='js/jquery.js') }}"></script>
<script src="{{ url_for('static', filename='js/bootstrap4.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/d3.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/d3/sparklines.js') }}"></script>
</head>
<body>
{% include 'nav_bar.html' %}
<div class="container-fluid">
<div class="row">
{% include 'hunter/menu_sidebar.html' %}
<div class="col-12 col-lg-10" id="core_content">
<div class="d-flex justify-content-around my-2">
<a class="btn btn-info border-secondary" style="width: 20rem;" href="{{ url_for('hunters.tracked_menu') }}">
<div class="card-body">
<h5 class="card-title"><i class="fas fa-ruler-combined"></i>
All Trackers&nbsp;&nbsp;&nbsp;&nbsp;<span class="badge badge-light">{{ stats['all'] }}</span>
</h5>
</div>
</a>
<a class="btn btn-info border-secondary" style="width: 20rem;" href="{{ url_for('hunters.tracked_menu_word') }}">
<div class="card-body">
<h5 class="card-title"><i class="fas fa-font"></i>
Words&nbsp;&nbsp;&nbsp;&nbsp;<span class="badge badge-light">{{ stats['word'] }}</span>
</h5>
</div>
</a>
<a class="btn btn-info border-secondary" style="width: 20rem;" href="{{ url_for('hunters.tracked_menu_set') }}">
<div class="card-body">
<h5 class="card-title"><i class="fas fa-layer-group"></i>
Set&nbsp;&nbsp;&nbsp;&nbsp;<span class="badge badge-light">{{ stats['set'] }}</span>
</h5>
</div>
</a>
</div>
<div class="d-flex justify-content-around my-2">
<a class="btn btn-info border-secondary" style="width: 20rem;" href="{{ url_for('hunters.tracked_menu_regex') }}">
<div class="card-body">
<h5 class="card-title">
<i class="fas fa-drafting-compass"></i>
Regex&nbsp;&nbsp;&nbsp;&nbsp;
<span class="badge badge-light">{{ stats['regex'] }}</span>
</h5>
</div>
</a>
<a class="btn btn-info border-secondary" style="width: 20rem;" href="{{ url_for('hunters.tracked_menu_typosquatting') }}">
<div class="card-body">
<h5 class="card-title">
<i class="fa fa-clone"></i>
Typo Squatting&nbsp;&nbsp;&nbsp;&nbsp;
<span class="badge badge-light">{{ stats['typosquatting'] }}</span>
</h5>
</div>
</a>
<a class="btn btn-info border-secondary" style="width: 20rem;" href="{{ url_for('hunters.tracked_menu_yara') }}">
<div class="card-body">
<h5 class="card-title">
<span class="bg-danger text-white font-weight-bold" style="font-size: 120%">&nbsp;{ </span>
Yara&nbsp;&nbsp;&nbsp;&nbsp;
<span class="badge badge-light">{{ stats['yara'] }}</span>
</h5>
</div>
</a>
</div>
{# TODO Buttons#}
<div class="card my-3">
<div class="card-header bg-dark text-white">
<h5 class="card-title"><b>Trackers Last Matches</b></h5>
</div>
<div class="card-body">
<table id="table_global_trackers" class="table table-striped table-borderless table-condensed">
<tbody style="font-size: 15px;">
{% for meta in trackers %}
<tr>
<td>{{ meta['type'] }}</td>
<td>
<span>
<a target="_blank" href="{{ url_for('hunter.show_tracker') }}?uuid={{ meta['uuid'] }}">
{% if meta['tracked'] %}
{% if meta['tracked']|length > 256 %}
{{ meta['tracked'][0:256] }}...
{% else %}
{{ meta['tracked'] }}
{% endif %}
{% endif %}
</a>
</span>
</td>
<td>
{% for tag in meta['tags'] %}
<a href="{{ url_for('tags_ui.get_obj_by_tags') }}?object_type=item&ltags={{ tag }}">
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span>
</a>
{% endfor %}
</td>
<td>
{% if meta['first_seen'] %}
{{ meta['first_seen'][0:4] }}/{{ meta['first_seen'][4:6] }}/{{ meta['first_seen'][6:8] }}
{% endif %}
</td>
<td>
{{ meta['timestamp'] }}
</td>
{# <td id="sparklines_{{ dict_uuid['uuid'] }}" style="text-align:center;"></td>#}
</tr>
{% endfor %}
</tbody>
</table>
</div>
</div>
<a class="btn btn-primary my-4" href="{{ url_for('hunters.add_tracked_menu') }}">
<i class="fas fa-plus-circle ml-auto"></i>
Create New Tracker
</a>
</div>
</div>
</div>
<script>
$(document).ready(function () {
$("#page-Tracker").addClass("active");
$("#nav_tracker_{{tracker_type}}").addClass("active");
{# {% for dict_uuid in global_trackers %}#}
{# sparkline("sparklines_{{ dict_uuid['uuid'] }}", {{ dict_uuid['sparkline'] }}, {height: 40});#}
{# {% endfor %}#}
});
function toggle_sidebar() {
if ($('#nav_menu').is(':visible')) {
$('#nav_menu').hide();
$('#side_menu').removeClass('border-right')
$('#side_menu').removeClass('col-lg-2')
$('#core_content').removeClass('col-lg-10')
} else {
$('#nav_menu').show();
$('#side_menu').addClass('border-right')
$('#side_menu').addClass('col-lg-2')
$('#core_content').addClass('col-lg-10')
}
}
</script>
</body>
</html>

2
var/www/templates/nav_bar.html
View File

@ -19,7 +19,7 @@
<a class="nav-link" id="navbar-tags" href="{{ url_for('tags_ui.tags_search_items') }}" aria-disabled="true"><i class="fas fa-tag"></i> Tags</a> <a class="nav-link" id="navbar-tags" href="{{ url_for('tags_ui.tags_search_items') }}" aria-disabled="true"><i class="fas fa-tag"></i> Tags</a>
</li> </li>
<li class="nav-item mr-3"> <li class="nav-item mr-3">
<a class="nav-link" id="page-Tracker" href="{{ url_for('hunter.tracked_menu') }}" aria-disabled="true"><i class="fas fa-crosshairs"></i> Leaks Hunter</a> <a class="nav-link" id="page-Tracker" href="{{ url_for('hunters.trackers_dashboard') }}" aria-disabled="true"><i class="fas fa-crosshairs"></i> Leaks Hunter</a>
</li> </li>
<li class="nav-item mr-3"> <li class="nav-item mr-3">
<a class="nav-link" id="page-Crawler" href="{{ url_for('crawler_splash.crawlers_dashboard') }}" tabindex="-1" aria-disabled="true"><i class="fas fa-spider"></i> Crawlers</a> <a class="nav-link" id="page-Crawler" href="{{ url_for('crawler_splash.crawlers_dashboard') }}" tabindex="-1" aria-disabled="true"><i class="fas fa-spider"></i> Crawlers</a>