CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

chg: [trackers] refactor trackers: track objects + filter by object types/sources/subtypes

pull/594/head
Terrtia 2023-05-04 16:35:56 +02:00
parent e363dcda62
commit 0daf5bad44
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
41 changed files with 2549 additions and 2124 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
bin/DB_KVROCKS_MIGRATION.py
View File

@ -233,17 +233,61 @@ def ail_2_ail_migration():
# item in queue
ail_2_ail.set_last_updated_sync_config()
###############################
# #
# TRACKER MIGRATION #
# #
###############################
def get_tracker_level(tracker_uuid):
level = r_serv_tracker.hget(f'tracker:{tracker_uuid}', 'level')
if not level:
level = 0
return int(level)
def get_tracker_metadata(tracker_uuid):
meta = {'uuid': tracker_uuid,
'tracked': r_serv_tracker.hget('tracker:{tracker_uuid}', 'tracked'),
'type': r_serv_tracker.hget('tracker:{tracker_uuid}', 'type'),
'date': r_serv_tracker.hget(f'tracker:{tracker_uuid}', 'date'),
'first_seen': r_serv_tracker.hget(f'tracker:{tracker_uuid}', 'first_seen'),
'last_seen': r_serv_tracker.hget(f'tracker:{tracker_uuid}', 'last_seen'),
'user_id': r_serv_tracker.hget('tracker:{tracker_uuid}', 'user_id'),
'level': get_tracker_level(tracker_uuid),
'mails': list(r_serv_tracker.smembers('tracker:mail:{tracker_uuid}')),
'sources': list(r_serv_tracker.smembers(f'tracker:sources:{tracker_uuid}')),
'tags': list(r_serv_tracker.smembers(f'tracker:tags:{tracker_uuid}')),
'description': r_serv_tracker.hget(f'tracker:{tracker_uuid}', 'description'),
'webhook': r_serv_tracker.hget(f'tracker:{tracker_uuid}', 'webhook')}
return meta
def get_tracker_items_by_daterange(tracker_uuid, date_from, date_to):
all_item_id = set()
if date_from and date_to:
l_date_match = r_serv_tracker.zrange(f'tracker:stat:{tracker_uuid}', 0, -1, withscores=True)
if l_date_match:
dict_date_match = dict(l_date_match)
for date_day in Date.substract_date(date_from, date_to):
if date_day in dict_date_match:
all_item_id |= r_serv_tracker.smembers(f'tracker:item:{tracker_uuid}:{date_day}')
return all_item_id
# trackers + retro_hunts
def trackers_migration():
print('TRACKERS MIGRATION...')
for tracker_uuid in old_Tracker.get_all_tracker_uuid():
meta = old_Tracker.get_tracker_metadata(tracker_uuid, user_id=True, description=True, level=True, tags=True, mails=True, sources=True, sparkline=False, webhook=True)
Tracker._re_create_tracker(meta['tracker'], meta['type'], meta['user_id'], meta['level'], meta['tags'], meta['mails'], meta['description'], meta['webhook'], 0, meta['uuid'], meta['sources'], meta['first_seen'], meta['last_seen'])
meta = get_tracker_metadata(tracker_uuid)
Tracker._re_create_tracker(meta['type'], meta['uuid'], meta['tracked'], meta['user_id'], meta['level'],
tags=meta['tags'], mails=meta['mails'], description=meta['description'],
webhook=meta['webhook'], sources=meta['sources'],
first_seen=meta['first_seen'], last_seen=meta['last_seen'])
tracker = Tracker.Tracker(tracker_uuid)
# object migration # # TODO: in background
for item_id in old_Tracker.get_tracker_items_by_daterange(tracker_uuid, meta['first_seen'], meta['last_seen']):
print(item_id)
Tracker.add_tracked_item(tracker_uuid, item_id)
item_date = get_item_date(item_id)
tracker.add('item', '', item_id, date=item_date)
print('RETRO HUNT MIGRATION...')
@ -929,13 +973,13 @@ if __name__ == '__main__':
# user_migration()
#tags_migration()
# items_migration()
crawler_migration()
# crawler_migration()
# domain_migration() # TO TEST ###########################
# decodeds_migration()
# screenshots_migration()
# subtypes_obj_migration()
# ail_2_ail_migration()
# trackers_migration()
trackers_migration()
# investigations_migration()
## statistics_migration()

16
bin/core/DbCleaner.py
View File

@ -14,23 +14,9 @@ sys.path.append(os.environ['AIL_BIN'])
##################################
# Import Project packages
##################################
from packages import Date
from packages import Term
from pubsublogger import publisher
def clean_term_db_stat_token():
all_stat_date = Term.get_all_token_stat_history()
list_date_to_keep = Date.get_date_range(31)
for date in all_stat_date:
if date not in list_date_to_keep:
# remove history
Term.delete_token_statistics_by_date(date)
print('Term Stats Cleaned')
if __name__ == "__main__":
publisher.port = 6380
@ -46,7 +32,7 @@ if __name__ == "__main__":
while True:
if daily_cleaner:
clean_term_db_stat_token()
daily_cleaner = False
else:
sys.exit(0)

2
bin/exporter/MailExporter.py
View File

@ -105,7 +105,7 @@ class MailExporterTracker(MailExporter):
def export(self, tracker, obj): # TODO match
tracker_type = tracker.get_type()
tracker_name = tracker.get_tracker()
tracker_name = tracker.get_tracked()
subject = f'AIL Framework Tracker: {tracker_name}' # TODO custom subject
body = f"AIL Framework, New occurrence for {tracker_type} tracker: {tracker_name}\n"
body += f'Item: {obj.id}\nurl:{obj.get_link()}'

2
bin/exporter/WebHookExporter.py
View File

@ -53,7 +53,7 @@ class WebHookExporterTracker(WebHookExporter):
data = {'trackerId': tracker.get_uuid(),
'trackerType': tracker.get_type(),
'tags': tracker.get_tags(),
'tracker': tracker.get_tracker(),
'tracker': tracker.get_tracked(),
# object
'itemId': obj.get_id(),
'itemURL': obj.get_link()}

8
bin/lib/Investigations.py
View File

@ -186,9 +186,9 @@ class Investigation(object):
def set_threat_level(self, threat_level):
try:
threat_level = int(threat_level)
except:
except TypeError:
raise UpdateInvestigationError('threat_level Not an integer')
if threat_level >= 1 and threat_level <= 4:
if 1 <= threat_level <= 4:
r_tracking.hset(f'investigations:data:{self.uuid}', 'threat_level', threat_level)
else:
raise UpdateInvestigationError(f'Invalid threat_level: {threat_level}')
@ -196,9 +196,9 @@ class Investigation(object):
def set_analysis(self, analysis):
try:
analysis = int(analysis)
except:
except TypeError:
raise UpdateInvestigationError('analysis Not an integer')
if analysis >= 0 and analysis <= 2:
if 0 <= analysis <= 2:
r_tracking.hset(f'investigations:data:{self.uuid}', 'analysis', analysis)
else:
raise UpdateInvestigationError(f'Invalid analysis: {analysis}')

1746
bin/lib/Tracker.py
View File

File diff suppressed because it is too large Load Diff

3
bin/lib/ail_core.py
View File

@ -40,6 +40,9 @@ def get_object_all_subtypes(obj_type):
return ['telegram', 'twitter', 'jabber']
return []
def get_objects_tracked():
return ['decoded', 'item', 'pgp']
def get_all_objects_with_subtypes_tuple():
str_objs = []
for obj_type in get_all_objects():

23
bin/lib/module_extractor.py
View File

@ -95,17 +95,18 @@ def convert_byte_offset_to_string(b_content, offset):
def get_tracker_match(obj_id, content):
extracted = []
extracted_yara = []
trackers = Tracker.get_obj_all_trackers('item', '', obj_id)
trackers = Tracker.get_obj_trackers('item', '', obj_id)
for tracker_uuid in trackers:
tracker_type = Tracker.get_tracker_type(tracker_uuid)
tracker = Tracker.Tracker(tracker_uuid)
tracker_type = tracker.get_type()
# print(tracker_type)
tracker = Tracker.get_tracker_by_uuid(tracker_uuid)
tracked = tracker.get_tracked()
if tracker_type == 'regex': # TODO Improve word detection -> word delimiter
regex_match = regex_helper.regex_finditer(r_key, tracker, obj_id, content)
regex_match = regex_helper.regex_finditer(r_key, tracked, obj_id, content)
for match in regex_match:
extracted.append([int(match[0]), int(match[1]), match[2], f'tracker:{tracker_uuid}'])
extracted.append([int(match[0]), int(match[1]), match[2], f'tracker:{tracker.uuid}'])
elif tracker_type == 'yara':
rule = Tracker.get_yara_rule_by_uuid(tracker_uuid)
rule = tracker.get_rule()
rule.match(data=content.encode(), callback=_get_yara_match,
which_callbacks=yara.CALLBACK_MATCHES, timeout=30)
yara_match = r_cache.smembers(f'extractor:yara:match:{r_key}')
@ -113,20 +114,20 @@ def get_tracker_match(obj_id, content):
extracted = []
for match in yara_match:
start, end, value = match.split(':', 2)
extracted_yara.append([int(start), int(end), value, f'tracker:{tracker_uuid}'])
extracted_yara.append([int(start), int(end), value, f'tracker:{tracker.uuid}'])
elif tracker_type == 'word' or tracker_type == 'set':
if tracker_type == 'set':
tracker = tracker.rsplit(';', 1)[0]
words = tracker.split(',')
tracked = tracked.rsplit(';', 1)[0]
words = tracked.split(',')
else:
words = [tracker]
words = [tracked]
for word in words:
regex = _get_word_regex(word)
regex_match = regex_helper.regex_finditer(r_key, regex, obj_id, content)
# print(regex_match)
for match in regex_match:
extracted.append([int(match[0]), int(match[1]), match[2], f'tracker:{tracker_uuid}'])
extracted.append([int(match[0]), int(match[1]), match[2], f'tracker:{tracker.uuid}'])
# Convert byte offset to string offset
if extracted_yara:

73
bin/lib/objects/CryptoCurrencies.py
View File

@ -2,12 +2,13 @@
# -*-coding:UTF-8 -*
import os
import re
import sys
from flask import url_for
from hashlib import sha256
from pymisp import MISPObject, MISPAttribute
from pymisp import MISPObject
sys.path.append(os.environ['AIL_BIN'])
##################################
@ -175,31 +176,57 @@ def get_all_cryptocurrencies():
cryptos[subtype] = get_all_cryptocurrencies_by_subtype(subtype)
return cryptos
def get_all_cryptocurrencies_by_subtype(subtype):
return get_all_id('cryptocurrency', subtype)
def sanitize_cryptocurrency_name_to_search(name_to_search, subtype): # TODO FILTER NAME + Key + mail
if subtype == '':
pass
elif subtype == 'name':
pass
elif subtype == 'mail':
pass
return name_to_search
# TODO save object
def import_misp_object(misp_obj):
"""
:type misp_obj: MISPObject
"""
obj_id = None
obj_subtype = None
for attribute in misp_obj.attributes:
if attribute.object_relation == 'address': # TODO: handle xmr address field
obj_id = attribute.value
elif attribute.object_relation == 'symbol':
obj_subtype = get_subtype_by_symbol(attribute.value)
if obj_id and obj_subtype:
obj = CryptoCurrency(obj_id, obj_subtype)
first_seen, last_seen = obj.get_misp_object_first_last_seen(misp_obj)
tags = obj.get_misp_object_tags(misp_obj)
# for tag in tags:
# obj.add_tag()
def search_cryptocurrency_by_name(name_to_search, subtype, r_pos=False):
cryptocurrencies = {}
# for subtype in subtypes:
r_name = sanitize_cryptocurrency_name_to_search(name_to_search, subtype)
if not name_to_search or isinstance(r_name, dict):
# break
return cryptocurrencies
r_name = re.compile(r_name)
for crypto_name in get_all_cryptocurrencies_by_subtype(subtype):
res = re.search(r_name, crypto_name)
if res:
cryptocurrencies[crypto_name] = {}
if r_pos:
cryptocurrencies[crypto_name]['hl-start'] = res.start()
cryptocurrencies[crypto_name]['hl-end'] = res.end()
return cryptocurrencies
if __name__ == '__main__':
res = get_all_cryptocurrencies()
print(res)
# # TODO save object
# def import_misp_object(misp_obj):
# """
# :type misp_obj: MISPObject
# """
# obj_id = None
# obj_subtype = None
# for attribute in misp_obj.attributes:
# if attribute.object_relation == 'address': # TODO: handle xmr address field
# obj_id = attribute.value
# elif attribute.object_relation == 'symbol':
# obj_subtype = get_subtype_by_symbol(attribute.value)
# if obj_id and obj_subtype:
# obj = CryptoCurrency(obj_id, obj_subtype)
# first_seen, last_seen = obj.get_misp_object_first_last_seen(misp_obj)
# tags = obj.get_misp_object_tags(misp_obj)
# # for tag in tags:
# # obj.add_tag()
# if __name__ == '__main__':
# name_to_search = '3c'
# subtype = 'bitcoin'
# print(search_cryptocurrency_by_name(name_to_search, subtype))

23
bin/lib/objects/Cves.py
View File

@ -2,9 +2,11 @@
# -*-coding:UTF-8 -*
import os
import re
import sys
from flask import url_for
from pymisp import MISPObject
import requests
@ -110,6 +112,25 @@ def get_cves_meta(cves_id, options=set()):
dict_cve[cve_id] = cve.get_meta(options=options)
return dict_cve
def sanitize_cve_name_to_search(name_to_search): # TODO FILTER NAME
return name_to_search
def search_cves_by_name(name_to_search, r_pos=False):
cves = {}
# for subtype in subtypes:
r_name = sanitize_cve_name_to_search(name_to_search)
if not name_to_search or isinstance(r_name, dict):
return cves
r_name = re.compile(r_name)
for cve_name in get_all_cves():
res = re.search(r_name, cve_name)
if res:
cves[cve_name] = {}
if r_pos:
cves[cve_name]['hl-start'] = res.start()
cves[cve_name]['hl-end'] = res.end()
return cves
def api_get_cves_range_by_daterange(date_from, date_to):
cves = []
for date in Date.substract_date(date_from, date_to):
@ -133,3 +154,5 @@ def get_cve_graphline(cve_id):
# if __name__ == '__main__':
# name_to_search = '98'
# print(search_cves_by_name(name_to_search))

36
bin/lib/objects/Decodeds.py
View File

@ -2,6 +2,7 @@
# -*-coding:UTF-8 -*
import os
import re
import sys
import magic
import requests
@ -9,6 +10,7 @@ import zipfile
from flask import url_for
from io import BytesIO
from pymisp import MISPObject
sys.path.append(os.environ['AIL_BIN'])
@ -114,11 +116,15 @@ class Decoded(AbstractDaterangeObject):
def get_filepath(self, mimetype=None):
return os.path.join(os.environ['AIL_HOME'], self.get_rel_path(mimetype=mimetype))
def get_content(self, mimetype=None):
def get_content(self, mimetype=None, r_str=False):
filepath = self.get_filepath(mimetype=mimetype)
with open(filepath, 'rb') as f:
file_content = BytesIO(f.read())
return file_content
if r_str:
with open(filepath, 'r') as f:
content = f.read()
else:
with open(filepath, 'rb') as f:
content = BytesIO(f.read())
return content
def get_zip_content(self):
# mimetype = self.get_estimated_type()
@ -347,6 +353,25 @@ def sanitise_mimetype(mimetype):
else:
return None
def sanitize_decoded_name_to_search(name_to_search): # TODO FILTER NAME
return name_to_search
def search_decodeds_by_name(name_to_search, r_pos=False):
decodeds = {}
# for subtype in subtypes:
r_name = sanitize_decoded_name_to_search(name_to_search)
if not name_to_search or isinstance(r_name, dict):
return decodeds
r_name = re.compile(r_name)
for decoded_name in get_all_decodeds():
res = re.search(r_name, decoded_name)
if res:
decodeds[decoded_name] = {}
if r_pos:
decodeds[decoded_name]['hl-start'] = res.start()
decodeds[decoded_name]['hl-end'] = res.end()
return decodeds
############################################################################
def sanityze_decoder_names(decoder_name):
@ -512,4 +537,7 @@ def get_all_decodeds_files():
decodeds.append(file)
return decodeds
# if __name__ == '__main__':
# name_to_search = '4d36'
# print(search_decodeds_by_name(name_to_search))

3
bin/lib/objects/Domains.py
View File

@ -96,6 +96,9 @@ class Domain(AbstractObject):
elif int(last_check) < date:
self._set_last_check(date)
def get_content(self):
return self.id
def get_last_origin(self, obj=False):
origin = {'item': r_crawler.hget(f'domain:meta:{self.id}', 'last_origin')}
if obj and origin['item']:

2
bin/lib/objects/Items.py
View File

@ -79,7 +79,7 @@ class Item(AbstractObject):
else:
return filename
def get_content(self, binary=False):
def get_content(self, r_str=True, binary=False):
"""
Returns Item content
"""

31
bin/lib/objects/Pgps.py
View File

@ -2,6 +2,7 @@
# -*-coding:UTF-8 -*
import os
import re
import sys
from flask import url_for
@ -100,5 +101,35 @@ def get_all_pgps():
def get_all_pgps_by_subtype(subtype):
return get_all_id('pgp', subtype)
# TODO FILTER NAME + Key + mail
def sanitize_pgp_name_to_search(name_to_search, subtype): # TODO FILTER NAME + Key + mail
if subtype == 'key':
pass
elif subtype == 'name':
pass
elif subtype == 'mail':
pass
return name_to_search
def search_pgps_by_name(name_to_search, subtype, r_pos=False):
pgps = {}
# for subtype in subtypes:
r_name = sanitize_pgp_name_to_search(name_to_search, subtype)
if not name_to_search or isinstance(r_name, dict):
# break
return pgps
r_name = re.compile(r_name)
for pgp_name in get_all_pgps_by_subtype(subtype):
res = re.search(r_name, pgp_name)
if res:
pgps[pgp_name] = {}
if r_pos:
pgps[pgp_name]['hl-start'] = res.start()
pgps[pgp_name]['hl-end'] = res.end()
return pgps
# if __name__ == '__main__':
# name_to_search = 'ex'
# subtype = 'name'
# print(search_pgps_by_name(name_to_search, subtype))

28
bin/lib/objects/Screenshots.py
View File

@ -3,6 +3,7 @@
import base64
import os
import re
import sys
from hashlib import sha256
@ -71,6 +72,9 @@ class Screenshot(AbstractObject):
file_content = BytesIO(f.read())
return file_content
def get_content(self):
return self.get_file_content()
def get_misp_object(self):
obj_attrs = []
obj = MISPObject('file')
@ -129,4 +133,26 @@ def create_screenshot(content, size_limit=5000000, b64=True, force=False):
return screenshot
return None
#if __name__ == '__main__':
def sanitize_screenshot_name_to_search(name_to_search): # TODO FILTER NAME
return name_to_search
def search_screenshots_by_name(name_to_search, r_pos=False):
screenshots = {}
# for subtype in subtypes:
r_name = sanitize_screenshot_name_to_search(name_to_search)
if not name_to_search or isinstance(r_name, dict):
return screenshots
r_name = re.compile(r_name)
for screenshot_name in get_all_screenshots():
res = re.search(r_name, screenshot_name)
if res:
screenshots[screenshot_name] = {}
if r_pos:
screenshots[screenshot_name]['hl-start'] = res.start()
screenshots[screenshot_name]['hl-end'] = res.end()
return screenshots
# if __name__ == '__main__':
# name_to_search = '29ba'
# print(search_screenshots_by_name(name_to_search))

41
bin/lib/objects/Usernames.py
View File

@ -3,13 +3,11 @@
import os
import sys
import redis
import re
from flask import url_for
from pymisp import MISPObject
# sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
sys.path.append(os.environ['AIL_BIN'])
##################################
# Import Project packages
@ -61,7 +59,7 @@ class Username(AbstractSubtypeObject):
else:
style = 'fas'
icon = '\uf007'
return {'style': style, 'icon': icon, 'color': '#4dffff', 'radius':5}
return {'style': style, 'icon': icon, 'color': '#4dffff', 'radius': 5}
def get_meta(self, options=set()):
meta = self._get_meta(options=options)
@ -74,15 +72,15 @@ class Username(AbstractSubtypeObject):
obj_attrs = []
if self.subtype == 'telegram':
obj = MISPObject('telegram-account', standalone=True)
obj_attrs.append( obj.add_attribute('username', value=self.id) )
obj_attrs.append(obj.add_attribute('username', value=self.id))
elif self.subtype == 'twitter':
obj = MISPObject('twitter-account', standalone=True)
obj_attrs.append( obj.add_attribute('name', value=self.id) )
obj_attrs.append(obj.add_attribute('name', value=self.id))
else:
obj = MISPObject('user-account', standalone=True)
obj_attrs.append( obj.add_attribute('username', value=self.id) )
obj_attrs.append(obj.add_attribute('username', value=self.id))
obj.first_seen = self.get_first_seen()
obj.last_seen = self.get_last_seen()
@ -107,9 +105,30 @@ def get_all_usernames():
def get_all_usernames_by_subtype(subtype):
return get_all_id('username', subtype)
# TODO FILTER NAME + Key + mail
def sanitize_username_name_to_search(name_to_search, subtype): # TODO FILTER NAME
return name_to_search
def search_usernames_by_name(name_to_search, subtype, r_pos=False):
usernames = {}
# for subtype in subtypes:
r_name = sanitize_username_name_to_search(name_to_search, subtype)
if not name_to_search or isinstance(r_name, dict):
# break
return usernames
r_name = re.compile(r_name)
for user_name in get_all_usernames_by_subtype(subtype):
res = re.search(r_name, user_name)
if res:
usernames[user_name] = {}
if r_pos:
usernames[user_name]['hl-start'] = res.start()
usernames[user_name]['hl-end'] = res.end()
return usernames
if __name__ == '__main__':
obj = Username('ninechantw', 'telegram')
print(obj.get_misp_object().to_json())
# if __name__ == '__main__':
# name_to_search = 'co'
# subtype = 'telegram'
# print(search_usernames_by_name(name_to_search, subtype))

3
bin/lib/objects/abstract_daterange_object.py
View File

@ -108,6 +108,9 @@ class AbstractDaterangeObject(AbstractObject, ABC):
sparkline.append(self.get_nb_seen_by_date(date))
return sparkline
def get_content(self):
return self.id
def _add_create(self):
r_object.sadd(f'{self.type}:all', self.id)

20
bin/lib/objects/abstract_object.py
View File

@ -19,9 +19,9 @@ sys.path.append(os.environ['AIL_BIN'])
##################################
from lib import Tag
from lib import Duplicate
from lib.correlations_engine import get_nb_correlations, get_correlations, add_obj_correlation, delete_obj_correlation, exists_obj_correlation, is_obj_correlated, get_nb_correlation_by_correl_type
from lib.correlations_engine import get_nb_correlations, get_correlations, add_obj_correlation, delete_obj_correlation, delete_obj_correlations, exists_obj_correlation, is_obj_correlated, get_nb_correlation_by_correl_type
from lib.Investigations import is_object_investigated, get_obj_investigations, delete_obj_investigations
from lib.Tracker import is_obj_tracked, get_obj_all_trackers, delete_obj_trackers
from lib.Tracker import is_obj_tracked, get_obj_trackers, delete_obj_trackers
class AbstractObject(ABC):
@ -85,6 +85,13 @@ class AbstractObject(ABC):
#- Tags -#
@abstractmethod
def get_content(self):
"""
Get Object Content
"""
pass
## Duplicates ##
def get_duplicates(self):
return Duplicate.get_obj_duplicates(self.type, self.get_subtype(r_str=True), self.id)
@ -125,7 +132,7 @@ class AbstractObject(ABC):
return is_obj_tracked(self.type, self.subtype, self.id)
def get_trackers(self):
return get_obj_all_trackers(self.type, self.subtype, self.id)
return get_obj_trackers(self.type, self.subtype, self.id)
def delete_trackers(self):
return delete_obj_trackers(self.type, self.subtype, self.id)
@ -134,13 +141,14 @@ class AbstractObject(ABC):
def _delete(self):
# DELETE TAGS
Tag.delete_obj_all_tags(self.id, self.type) # ########### # TODO: # TODO: # FIXME:
Tag.delete_object_tags(self.type, self.get_subtype(r_str=True), self.id)
# remove from tracker
self.delete_trackers()
# remove from retro hunt currently item only TODO
# remove from investigations
self.delete_investigations()
# # TODO: remove from correlation
# Delete Correlations
delete_obj_correlations(self.type, self.get_subtype(r_str=True), self.id)
@abstractmethod
def delete(self):

3
bin/lib/objects/abstract_subtype_object.py
View File

@ -116,6 +116,9 @@ class AbstractSubtypeObject(AbstractObject, ABC):
if date > last_seen:
self.set_last_seen(date)
def get_content(self):
return self.id
def get_sparkline(self):
sparkline = []
for date in Date.get_previous_date_list(6):

38
bin/lib/objects/ail_objects.py
View File

@ -138,9 +138,14 @@ def get_object_meta(obj_type, subtype, id, options=set(), flask_context=False):
def get_objects_meta(objs, options=set(), flask_context=False):
metas = []
for obj_dict in objs:
metas.append(get_object_meta(obj_dict['type'], obj_dict['subtype'], obj_dict['id'], options=options,
flask_context=flask_context))
for obj in objs:
if isinstance(obj, dict):
obj_type = obj['type']
subtype = obj['subtype']
obj_id = obj['id']
else:
obj_type, subtype, obj_id = obj.split(':', 2)
metas.append(get_object_meta(obj_type, subtype, obj_id, options=options, flask_context=flask_context))
return metas
@ -162,7 +167,25 @@ def get_object_card_meta(obj_type, subtype, id, related_btc=False):
return meta
def get_ui_obj_tag_table_keys(obj_type):
#### OBJ FILTERS ####
def is_filtered(obj, filters):
if 'mimetypes' in filters:
mimetype = obj.get_mimetype()
if mimetype not in filters['mimetypes']:
return True
if 'sources' in filters:
obj_source = obj.get_source()
if obj_source not in filters['sources']:
return True
if 'subtypes' in filters:
subtype = obj.get_subtype(r_str=True)
if subtype not in filters['subtypes']:
return True
return False
def get_ui_obj_tag_table_keys(obj_type): # TODO REMOVE ME
"""
Warning: use only in flask (dynamic templates)
"""
@ -170,6 +193,7 @@ def get_ui_obj_tag_table_keys(obj_type):
return ['id', 'first_seen', 'last_check', 'status'] # # TODO: add root screenshot
# # # # MISP OBJECTS # # # #
# # TODO: CHECK IF object already have an UUID
@ -265,6 +289,7 @@ def api_sanitize_object_type(obj_type):
if not is_valid_object_type(obj_type):
return {'status': 'error', 'reason': 'Incorrect object type'}, 400
#### CORRELATION ####
def get_obj_correlations(obj_type, subtype, obj_id):
obj = get_object(obj_type, subtype, obj_id)
@ -292,13 +317,14 @@ def get_obj_correlations_objs(obj_type, subtype, obj_id, filter_types=[], lvl=0,
def obj_correlations_objs_add_tags(obj_type, subtype, obj_id, tags, filter_types=[], lvl=0, nb_max=300):
objs = get_obj_correlations_objs(obj_type, subtype, obj_id, filter_types=filter_types, lvl=lvl, nb_max=nb_max)
# print(objs)
for obj_tuple in objs:
obj1_type, subtype1, id1 = obj_tuple
add_obj_tags(obj1_type, subtype1, id1, tags)
return objs
################################################################################
################################################################################
################################################################################ TODO
################################################################################
def delete_obj_correlations(obj_type, subtype, obj_id):
@ -357,7 +383,7 @@ def get_correlations_graph_node(obj_type, subtype, obj_id, filter_types=[], max_
"links": create_correlation_graph_links(links)}
###############
# --- CORRELATION --- #
# if __name__ == '__main__':

18
bin/modules/Decoder.py
View File

@ -23,6 +23,9 @@ from modules.abstract_module import AbstractModule
from lib.ConfigLoader import ConfigLoader
from lib.objects.Items import Item
from lib.objects.Decodeds import Decoded
from trackers.Tracker_Term import Tracker_Term
from trackers.Tracker_Regex import Tracker_Regex
from trackers.Tracker_Yara import Tracker_Yara
config_loader = ConfigLoader()
hex_max_execution_time = config_loader.get_config_int("Decoder", "max_execution_time_hexadecimal")
@ -76,6 +79,10 @@ class Decoder(AbstractModule):
# Waiting time in seconds between to message processed
self.pending_seconds = 1
self.tracker_term = Tracker_Term(queue=False)
self.tracker_regex = Tracker_Regex(queue=False)
self.tracker_yara = Tracker_Yara(queue=False)
# Send module state to logs
self.redis_logger.info(f'Module {self.module_name} initialized')
@ -84,6 +91,7 @@ class Decoder(AbstractModule):
item = Item(message)
content = item.get_content()
date = item.get_date()
new_decodeds = []
for decoder in self.decoder_order:
find = False
@ -96,8 +104,8 @@ class Decoder(AbstractModule):
encodeds = set(encodeds)
for encoded in encodeds:
find = False
if len(encoded) >= decoder['encoded_min_size']:
find = True
decoded_file = self.decoder_function[dname](encoded)
sha1_string = sha1(decoded_file).hexdigest()
@ -109,6 +117,7 @@ class Decoder(AbstractModule):
print(sha1_string, item.id)
raise Exception(f'Invalid mimetype: {decoded.id} {item.id}')
decoded.save_file(decoded_file, mimetype)
new_decodeds.append(decoded.id)
else:
mimetype = decoded.get_mimetype()
decoded.add(dname, date, item.id, mimetype=mimetype)
@ -125,6 +134,13 @@ class Decoder(AbstractModule):
msg = f'infoleak:automatic-detection="{dname}";{item.id}'
self.add_message_to_queue(msg, 'Tags')
####################
# TRACKERS DECODED
for decoded_id in new_decodeds:
self.tracker_term.compute(decoded_id, obj_type='decoded')
self.tracker_regex.compute(decoded_id, obj_type='decoded')
self.tracker_yara.compute(decoded_id, obj_type='decoded')
if __name__ == '__main__':
module = Decoder()

12
bin/modules/PgpDump.py
View File

@ -216,16 +216,16 @@ class PgpDump(AbstractModule):
pgp = Pgps.Pgp(name, 'name')
pgp.add(date, self.item_id)
print(f' name: {name}')
self.tracker_term.compute(self.item_id, item_content=name)
self.tracker_regex.compute(self.item_id, content=name)
self.tracker_yara.compute(self.item_id, item_content=name)
self.tracker_term.compute(name, obj_type='pgp', subtype='name')
self.tracker_regex.compute(name, obj_type='pgp', subtype='name')
self.tracker_yara.compute(name, obj_type='pgp', subtype='name')
for mail in self.mails:
pgp = Pgps.Pgp(mail, 'mail')
pgp.add(date, self.item_id)
print(f' mail: {mail}')
self.tracker_term.compute(self.item_id, item_content=mail)
self.tracker_regex.compute(self.item_id, content=mail)
self.tracker_yara.compute(self.item_id, item_content=mail)
self.tracker_term.compute(mail, obj_type='pgp', subtype='mail')
self.tracker_regex.compute(mail, obj_type='pgp', subtype='mail')
self.tracker_yara.compute(mail, obj_type='pgp', subtype='mail')
# Keys extracted from PGP PRIVATE KEY BLOCK
for key in self.private_keys:

532
bin/packages/Term.py
View File

@ -1,532 +0,0 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
import os
import re
import sys
import time
import uuid
import datetime
from collections import defaultdict
from flask import escape
from nltk.tokenize import RegexpTokenizer
from textblob import TextBlob
sys.path.append(os.environ['AIL_BIN'])
##################################
# Import Project packages
##################################
from lib import ConfigLoader
from lib import Tracker
from packages import Date
from lib.objects import Items
config_loader = ConfigLoader.ConfigLoader()
r_serv_term = config_loader.get_db_conn("Kvrocks_Trackers")
config_loader = None
email_regex = r'[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,6}'
email_regex = re.compile(email_regex)
special_characters = set('[<>~!?@#$%^&*|()_-+={}":;,.\'\n\r\t]/\\')
special_characters.add('\\s')
# NLTK tokenizer
tokenizer = RegexpTokenizer('[\&\~\:\;\,\.\(\)\{\}\|\[\]\\\\/\-/\=\'\"\%\$\?\@\+\#\_\^\<\>\!\*\n\r\t\s]+',
gaps=True, discard_empty=True)
def is_valid_uuid_v4(UUID):
if not UUID:
return False
UUID = UUID.replace('-', '')
try:
uuid_test = uuid.UUID(hex=UUID, version=4)
return uuid_test.hex == UUID
except:
return False
# # TODO: use new package => duplicate fct
def is_in_role(user_id, role):
if r_serv_term.sismember('user_role:{}'.format(role), user_id):
return True
else:
return False
def check_term_uuid_valid_access(term_uuid, user_id):
if not is_valid_uuid_v4(term_uuid):
return {"status": "error", "reason": "Invalid uuid"}, 400
level = r_serv_term.hget('tracker:{}'.format(term_uuid), 'level')
if not level:
return {"status": "error", "reason": "Unknown uuid"}, 404
if level == 0:
if r_serv_term.hget('tracker:{}'.format(term_uuid), 'user_id') != user_id:
if not is_in_role(user_id, 'admin'):
return {"status": "error", "reason": "Unknown uuid"}, 404
return None
def is_valid_mail(email):
result = email_regex.match(email)
if result:
return True
else:
return False
def verify_mail_list(mail_list):
for mail in mail_list:
if not is_valid_mail(mail):
return {'status': 'error', 'reason': 'Invalid email', 'value': mail}, 400
return None
def is_valid_regex(term_regex):
try:
re.compile(term_regex)
return True
except:
return False
def get_text_word_frequency(item_content, filtering=True):
item_content = item_content.lower()
words_dict = defaultdict(int)
if filtering:
blob = TextBlob(item_content, tokenizer=tokenizer)
else:
blob = TextBlob(item_content)
for word in blob.tokens:
words_dict[word] += 1
return words_dict
# # TODO: create all tracked words
def get_tracked_words_list():
return list(r_serv_term.smembers('all:tracker:word'))
def get_set_tracked_words_list():
set_list = r_serv_term.smembers('all:tracker:set')
all_set_list = []
for elem in set_list:
res = elem.split(';')
num_words = int(res[1])
ter_set = res[0].split(',')
all_set_list.append((ter_set, num_words, elem))
return all_set_list
def get_regex_tracked_words_dict():
regex_list = r_serv_term.smembers('all:tracker:regex')
dict_tracked_regex = {}
for regex in regex_list:
dict_tracked_regex[regex] = re.compile(regex)
return dict_tracked_regex
def get_tracked_term_list_item(term_uuid, date_from, date_to):
all_item_id = []
if date_from and date_to:
for date in r_serv_term.zrangebyscore('tracker:stat:{}'.format(term_uuid), int(date_from), int(date_to)):
all_item_id = all_item_id + list(r_serv_term.smembers('tracker:item:{}:{}'.format(term_uuid, date)))
return all_item_id
def is_term_tracked_in_global_level(term, term_type):
res = r_serv_term.smembers('all:tracker_uuid:{}:{}'.format(term_type, term))
if res:
for elem_uuid in res:
if r_serv_term.hget('tracker:{}'.format(elem_uuid), 'level') == '1':
return True
return False
def is_term_tracked_in_user_level(term, term_type, user_id):
res = r_serv_term.smembers('user:tracker:{}'.format(user_id))
if res:
for elem_uuid in res:
if r_serv_term.hget('tracker:{}'.format(elem_uuid), 'tracked') == term:
if r_serv_term.hget('tracker:{}'.format(elem_uuid), 'type') == term_type:
return True
return False
def parse_json_term_to_add(dict_input, user_id):
term = dict_input.get('term', None)
if not term:
return {"status": "error", "reason": "Term not provided"}, 400
term_type = dict_input.get('type', None)
if not term_type:
return {"status": "error", "reason": "Term type not provided"}, 400
nb_words = dict_input.get('nb_words', 1)
description = dict_input.get('description', '')
description = escape(description)
webhook = dict_input.get('webhook', '')
webhook = escape(webhook)
res = parse_tracked_term_to_add(term, term_type, nb_words=nb_words)
if res[1] != 200:
return res
term = res[0]['term']
term_type = res[0]['type']
tags = dict_input.get('tags', [])
mails = dict_input.get('mails', [])
res = verify_mail_list(mails)
if res:
return res
level = dict_input.get('level', 1)
try:
level = int(level)
if level not in range(0, 1):
level = 1
except:
level = 1
# check if term already tracked in global
if level == 1:
if is_term_tracked_in_global_level(term, term_type):
return {"status": "error", "reason": "Term already tracked"}, 409
else:
if is_term_tracked_in_user_level(term, term_type, user_id):
return {"status": "error", "reason": "Term already tracked"}, 409
term_uuid = add_tracked_term(term, term_type, user_id, level, tags, mails, description, webhook)
return {'term': term, 'type': term_type, 'uuid': term_uuid}, 200
def parse_tracked_term_to_add(term, term_type, nb_words=1):
if term_type == 'regex':
if not is_valid_regex(term):
return {"status": "error", "reason": "Invalid regex"}, 400
elif term_type == 'word' or term_type == 'set':
# force lowercase
term = term.lower()
word_set = set(term)
set_inter = word_set.intersection(special_characters)
if set_inter:
return {"status": "error", "reason": "special character not allowed", "message": "Please use a regex or remove all special characters"}, 400
words = term.split()
# not a word
if term_type == 'word' and len(words) > 1:
term_type = 'set'
# output format: term1,term2,term3;2
if term_type == 'set':
try:
nb_words = int(nb_words)
except:
nb_words = 1
if nb_words == 0:
nb_words = 1
words_set = set(words)
words_set = sorted(words_set)
if nb_words > len(words_set):
nb_words = len(words_set)
term = ",".join(words_set)
term = "{};{}".format(term, nb_words)
elif term_type == 'yara_custom':
if not Tracker.is_valid_yara_rule(term):
return {"status": "error", "reason": "Invalid custom Yara Rule"}, 400
elif term_type == 'yara_default':
if not Tracker.is_valid_default_yara_rule(term):
return {"status": "error", "reason": "The Yara Rule doesn't exist"}, 400
else:
return {"status": "error", "reason": "Incorrect type"}, 400
return {"status": "success", "term": term, "type": term_type}, 200
def add_tracked_term(term, term_type, user_id, level, tags, mails, description, webhook, dashboard=0):
term_uuid = str(uuid.uuid4())
# YARA
if term_type == 'yara_custom' or term_type == 'yara_default':
term = Tracker.save_yara_rule(term_type, term, tracker_uuid=term_uuid)
term_type = 'yara'
# create metadata
r_serv_term.hset('tracker:{}'.format(term_uuid), 'tracked', term) # # TODO: use hash
r_serv_term.hset('tracker:{}'.format(term_uuid), 'type', term_type)
r_serv_term.hset('tracker:{}'.format(term_uuid), 'date', datetime.date.today().strftime("%Y%m%d"))
r_serv_term.hset('tracker:{}'.format(term_uuid), 'user_id', user_id)
r_serv_term.hset('tracker:{}'.format(term_uuid), 'level', level)
r_serv_term.hset('tracker:{}'.format(term_uuid), 'dashboard', dashboard)
if description:
r_serv_term.hset('tracker:{}'.format(term_uuid), 'description', description)
if webhook:
r_serv_term.hset('tracker:{}'.format(term_uuid), 'webhook', webhook)
# create all term set
r_serv_term.sadd('all:tracker:{}'.format(term_type), term)
# create term - uuid map
r_serv_term.sadd('all:tracker_uuid:{}:{}'.format(term_type, term), term_uuid)
# add display level set
if level == 0: # user only
r_serv_term.sadd('user:tracker:{}'.format(user_id), term_uuid)
r_serv_term.sadd('user:tracker:{}:{}'.format(user_id, term_type), term_uuid)
elif level == 1: # global
r_serv_term.sadd('global:tracker', term_uuid)
r_serv_term.sadd('global:tracker:{}'.format(term_type), term_uuid)
# create term tags list
for tag in tags:
r_serv_term.sadd('tracker:tags:{}'.format(term_uuid), escape(tag))
# create term tags mail notification list
for mail in mails:
r_serv_term.sadd('tracker:mail:{}'.format(term_uuid), escape(mail))
# toggle refresh module tracker list/set
r_serv_term.set('tracker:refresh:{}'.format(term_type), time.time())
return term_uuid
def parse_tracked_term_to_delete(dict_input, user_id):
term_uuid = dict_input.get("uuid", None)
res = check_term_uuid_valid_access(term_uuid, user_id)
if res:
return res
delete_term(term_uuid)
return {"uuid": term_uuid}, 200
# # TODO: MOVE IN TRACKER
def delete_term(term_uuid):
term = r_serv_term.hget('tracker:{}'.format(term_uuid), 'tracked')
term_type = r_serv_term.hget('tracker:{}'.format(term_uuid), 'type')
level = r_serv_term.hget('tracker:{}'.format(term_uuid), 'level')
r_serv_term.srem('all:tracker_uuid:{}:{}'.format(term_type, term), term_uuid)
r_serv_term.srem(f'trackers:all', term_uuid)
r_serv_term.srem(f'trackers:all:{term_type}', term_uuid)
# Term not tracked by other users
if not r_serv_term.exists('all:tracker_uuid:{}:{}'.format(term_type, term)):
r_serv_term.srem('all:tracker:{}'.format(term_type), term)
# toggle refresh module tracker list/set
r_serv_term.set('tracker:refresh:{}'.format(term_type), time.time())
if level == '0': # user only
user_id = term_type = r_serv_term.hget('tracker:{}'.format(term_uuid), 'user_id')
r_serv_term.srem('user:tracker:{}'.format(user_id), term_uuid)
r_serv_term.srem('user:tracker:{}:{}'.format(user_id, term_type), term_uuid)
elif level == '1': # global
r_serv_term.srem('global:tracker', term_uuid)
r_serv_term.srem('global:tracker:{}'.format(term_type), term_uuid)
# delete metatadata
r_serv_term.delete('tracker:{}'.format(term_uuid))
# remove tags
r_serv_term.delete('tracker:tags:{}'.format(term_uuid))
# remove mails
r_serv_term.delete('tracker:mail:{}'.format(term_uuid))
# remove sources
r_serv_term.delete('tracker:sources:{}'.format(term_uuid))
# remove item set
all_item_date = r_serv_term.zrange(f'tracker:stat:{term_uuid}', 0, -1, withscores=True)
if all_item_date:
all_item_date = dict(all_item_date)
for date in all_item_date:
for item_id in r_serv_term.smembers(f'tracker:item:{term_uuid}:{date}'):
r_serv_term.srem(f'obj:trackers:item:{item_id}', term_uuid)
r_serv_term.delete(f'tracker:item:{term_uuid}:{date}')
r_serv_term.delete('tracker:stat:{}'.format(term_uuid))
if term_type == 'yara':
# delete custom rule
if not Tracker.is_default_yara_rule(term):
filepath = Tracker.get_yara_rule_file_by_tracker_name(term)
if filepath:
os.remove(filepath)
def replace_tracker_description(term_uuid, description):
description = escape(description)
r_serv_term.hset('tracker:{}'.format(term_uuid), 'description', description)
def replace_tracked_term_tags(term_uuid, tags):
r_serv_term.delete('tracker:tags:{}'.format(term_uuid))
for tag in tags:
tag = escape(tag)
r_serv_term.sadd('tracker:tags:{}'.format(term_uuid), tag)
def replace_tracked_term_mails(term_uuid, mails):
res = verify_mail_list(mails)
if res:
return res
else:
r_serv_term.delete('tracker:mail:{}'.format(term_uuid))
for mail in mails:
mail = escape(mail)
r_serv_term.sadd('tracker:mail:{}'.format(term_uuid), mail)
def get_term_uuid_list(term, term_type):
return list(r_serv_term.smembers('all:tracker_uuid:{}:{}'.format(term_type, term)))
def get_term_tags(term_uuid):
return list(r_serv_term.smembers('tracker:tags:{}'.format(term_uuid)))
def get_term_mails(term_uuid):
return list(r_serv_term.smembers('tracker:mail:{}'.format(term_uuid)))
def get_term_webhook(term_uuid):
return r_serv_term.hget('tracker:{}'.format(term_uuid), "webhook")
def add_tracked_item(term_uuid, item_id, item_date):
# track item
r_serv_term.sadd('tracker:item:{}:{}'.format(term_uuid, item_date), item_id)
# track nb item by date
r_serv_term.zadd('tracker:stat:{}'.format(term_uuid), {item_date: item_date})
def create_token_statistics(item_date, word, nb):
r_serv_term.zincrby('stat_token_per_item_by_day:{}'.format(item_date), 1, word)
r_serv_term.zincrby('stat_token_total_by_day:{}'.format(item_date), nb, word)
r_serv_term.sadd('stat_token_history', item_date)
def delete_token_statistics_by_date(item_date):
r_serv_term.delete('stat_token_per_item_by_day:{}'.format(item_date))
r_serv_term.delete('stat_token_total_by_day:{}'.format(item_date))
r_serv_term.srem('stat_token_history', item_date)
def get_all_token_stat_history():
return r_serv_term.smembers('stat_token_history')
def get_tracked_term_last_updated_by_type(term_type):
epoch_update = r_serv_term.get('tracker:refresh:{}'.format(term_type))
if not epoch_update:
epoch_update = 0
return float(epoch_update)
def parse_get_tracker_term_item(dict_input, user_id):
term_uuid = dict_input.get('uuid', None)
res = check_term_uuid_valid_access(term_uuid, user_id)
if res:
return res
date_from = dict_input.get('date_from', None)
date_to = dict_input.get('date_to', None)
if date_from is None:
date_from = get_tracked_term_first_seen(term_uuid)
if date_from:
date_from = date_from[0]
if date_to is None:
date_to = date_from
if date_from > date_to:
date_from = date_to
all_item_id = Tracker.get_tracker_items_by_daterange(term_uuid, date_from, date_to)
all_item_id = Items.get_item_list_desc(all_item_id)
res_dict = {'uuid': term_uuid, 'date_from': date_from, 'date_to': date_to, 'items': all_item_id}
return res_dict, 200
def get_tracked_term_first_seen(term_uuid):
return Tracker.get_tracker_first_seen(term_uuid)
def get_tracked_term_last_seen(term_uuid):
return Tracker.get_tracker_last_seen(term_uuid)
def get_term_metedata(term_uuid, user_id=False, description=False, level=False, tags=False, mails=False, sparkline=False):
dict_uuid = {}
dict_uuid['term'] = r_serv_term.hget('tracker:{}'.format(term_uuid), 'tracked')
dict_uuid['type'] = r_serv_term.hget('tracker:{}'.format(term_uuid), 'type')
dict_uuid['date'] = r_serv_term.hget('tracker:{}'.format(term_uuid), 'date')
dict_uuid['first_seen'] = get_tracked_term_first_seen(term_uuid)
dict_uuid['last_seen'] = get_tracked_term_last_seen(term_uuid)
if user_id:
dict_uuid['user_id'] = r_serv_term.hget('tracker:{}'.format(term_uuid), 'user_id')
if level:
dict_uuid['level'] = r_serv_term.hget('tracker:{}'.format(term_uuid), 'level')
if mails:
dict_uuid['mails'] = get_list_trackeed_term_mails(term_uuid)
if tags:
dict_uuid['tags'] = get_list_trackeed_term_tags(term_uuid)
if sparkline:
dict_uuid['sparkline'] = get_tracked_term_sparkline(term_uuid)
if description:
dict_uuid['description'] = r_serv_term.hget('tracker:{}'.format(term_uuid), 'description')
dict_uuid['uuid'] = term_uuid
return dict_uuid
def get_tracked_term_sparkline(tracker_uuid, num_day=6):
date_range_sparkline = Date.get_date_range(num_day)
sparklines_value = []
for date_day in date_range_sparkline:
nb_seen_this_day = r_serv_term.scard('tracker:item:{}:{}'.format(tracker_uuid, date_day))
if nb_seen_this_day is None:
nb_seen_this_day = 0
sparklines_value.append(int(nb_seen_this_day))
return sparklines_value
def get_list_tracked_term_stats_by_day(list_tracker_uuid, num_day=31, date_from=None, date_to=None):
if date_from and date_to:
date_range = Date.substract_date(date_from, date_to)
else:
date_range = Date.get_date_range(num_day)
list_tracker_stats = []
for tracker_uuid in list_tracker_uuid:
dict_tracker_data = []
tracker = r_serv_term.hget('tracker:{}'.format(tracker_uuid), 'tracked')
for date_day in date_range:
nb_seen_this_day = r_serv_term.scard('tracker:item:{}:{}'.format(tracker_uuid, date_day))
if nb_seen_this_day is None:
nb_seen_this_day = 0
dict_tracker_data.append({"date": date_day, "value": int(nb_seen_this_day)})
list_tracker_stats.append({"name": tracker, "Data": dict_tracker_data})
return list_tracker_stats
def get_list_trackeed_term_tags(term_uuid):
res = r_serv_term.smembers('tracker:tags:{}'.format(term_uuid))
if res:
return list(res)
else:
return []
def get_list_trackeed_term_mails(term_uuid):
res = r_serv_term.smembers('tracker:mail:{}'.format(term_uuid))
if res:
return list(res)
else:
return []
def get_user_tracked_term_uuid(user_id, filter_type=None):
if filter_type:
return list(r_serv_term.smembers('user:tracker:{}:{}'.format(user_id, filter_type)))
else:
return list(r_serv_term.smembers('user:tracker:{}'.format(user_id)))
def get_global_tracked_term_uuid(filter_type=None):
if filter_type:
return list(r_serv_term.smembers('global:tracker:{}'.format(filter_type)))
else:
return list(r_serv_term.smembers('global:tracker'))
def get_all_user_tracked_terms(user_id, filter_type=None):
all_user_term = []
all_user_term_uuid = get_user_tracked_term_uuid(user_id, filter_type=filter_type)
for term_uuid in all_user_term_uuid:
all_user_term.append(get_term_metedata(term_uuid, tags=True, mails=True, sparkline=True))
return all_user_term
def get_all_global_tracked_terms(filter_type=None):
all_user_term = []
all_user_term_uuid = get_global_tracked_term_uuid(filter_type=filter_type)
for term_uuid in all_user_term_uuid:
all_user_term.append(get_term_metedata(term_uuid, user_id=True, tags=True, mails=True, sparkline=True))
return all_user_term

2
bin/packages/lib_words.py
View File

@ -3,8 +3,6 @@
import os
import string
from pubsublogger import publisher
import calendar
from datetime import date
from dateutil.rrule import rrule, DAILY

30
bin/trackers/Retro_Hunt.py
View File

@ -61,15 +61,17 @@ class Retro_Hunt(AbstractModule):
self.progress = 0
# First launch
# restart
rule = Tracker.get_retro_hunt_task_rule(task_uuid, r_compile=True)
retro_hunt = Tracker.RetroHunt(task_uuid) # TODO SELF
timeout = Tracker.get_retro_hunt_task_timeout(task_uuid)
rule = retro_hunt.get_rule(r_compile=True)
timeout = retro_hunt.get_timeout()
self.redis_logger.debug(f'{self.module_name}, Retro Hunt rule {task_uuid} timeout {timeout}')
sources = Tracker.get_retro_hunt_task_sources(task_uuid, r_sort=True)
sources = retro_hunt.get_sources(r_sort=True)
self.date_from = Tracker.get_retro_hunt_task_date_from(task_uuid)
self.date_to = Tracker.get_retro_hunt_task_date_to(task_uuid)
self.tags = Tracker.get_retro_hunt_task_tags(task_uuid)
self.date_from = retro_hunt.get_date_from()
self.date_to = retro_hunt.get_date_to()
self.tags = retro_hunt.get_tags()
curr_date = Tracker.get_retro_hunt_task_current_date(task_uuid)
self.nb_src_done = Tracker.get_retro_hunt_task_nb_src_done(task_uuid, sources=sources)
self.update_progress(sources, curr_date)
@ -106,11 +108,10 @@ class Retro_Hunt(AbstractModule):
# PAUSE
self.update_progress(sources, curr_date)
if Tracker.check_retro_hunt_pause(task_uuid):
if retro_hunt.to_pause():
Tracker.set_retro_hunt_last_analyzed(task_uuid, id)
# self.update_progress(sources, curr_date, save_db=True)
Tracker.pause_retro_hunt_task(task_uuid)
Tracker.clear_retro_hunt_task_cache(task_uuid)
retro_hunt.pause()
return None
self.nb_src_done += 1
@ -120,9 +121,7 @@ class Retro_Hunt(AbstractModule):
self.update_progress(sources, curr_date)
Tracker.set_retro_hunt_task_state(task_uuid, 'completed')
Tracker.set_retro_hunt_nb_match(task_uuid)
Tracker.clear_retro_hunt_task_cache(task_uuid)
retro_hunt.complete()
print(f'Retro Hunt {task_uuid} completed')
self.redis_logger.warning(f'{self.module_name}, Retro Hunt {task_uuid} completed')
@ -130,10 +129,11 @@ class Retro_Hunt(AbstractModule):
# # TODO: stop
def update_progress(self, sources, curr_date, save_db=False):
progress = Tracker.compute_retro_hunt_task_progress(self.task_uuid, date_from=self.date_from, date_to=self.date_to,
sources=sources, curr_date=curr_date, nb_src_done=self.nb_src_done)
retro_hunt = Tracker.RetroHunt(retro_hubt) # TODO USE SELF
progress = retro_hunt.compute_progress(date_from=self.date_from, date_to=self.date_to,
sources=sources, curr_date=curr_date, nb_src_done=self.nb_src_done)
if self.progress != progress:
Tracker.set_cache_retro_hunt_task_progress(self.task_uuid, progress)
retro_hunt.set_progress(progress)
self.progress = progress
# if save_db:
# Tracker.set_retro_hunt_task_progress(task_uuid, progress)

70
bin/trackers/Tracker_Regex.py
View File

@ -17,9 +17,8 @@ sys.path.append(os.environ['AIL_BIN'])
# Import Project packages
##################################
from modules.abstract_module import AbstractModule
from lib.objects.Items import Item
from lib.objects import ail_objects
from lib.ConfigLoader import ConfigLoader
from packages import Term
from lib import Tracker
from exporter.MailExporter import MailExporterTracker
@ -39,7 +38,7 @@ class Tracker_Regex(AbstractModule):
self.max_execution_time = config_loader.get_config_int(self.module_name, "max_execution_time")
# refresh Tracked Regex
self.dict_regex_tracked = Term.get_regex_tracked_words_dict()
self.tracked_regexs = Tracker.get_tracked_regexs()
self.last_refresh = time.time()
# Exporter
@ -48,56 +47,65 @@ class Tracker_Regex(AbstractModule):
self.redis_logger.info(f"Module: {self.module_name} Launched")
def compute(self, item_id, content=None):
def compute(self, obj_id, obj_type='item', subtype=''):
# refresh Tracked regex
if self.last_refresh < Tracker.get_tracker_last_updated_by_type('regex'):
self.dict_regex_tracked = Term.get_regex_tracked_words_dict()
self.tracked_regexs = Tracker.get_tracked_regexs()
self.last_refresh = time.time()
self.redis_logger.debug('Tracked regex refreshed')
print('Tracked regex refreshed')
item = Item(item_id)
item_id = item.get_id()
if not content:
content = item.get_content()
obj = ail_objects.get_object(obj_type, subtype, obj_id)
obj_id = obj.get_id()
obj_type = obj.get_type()
for regex in self.dict_regex_tracked:
matched = self.regex_findall(self.dict_regex_tracked[regex], item_id, content)
# Object Filter
if obj_type not in self.tracked_regexs:
return None
content = obj.get_content(r_str=True)
for dict_regex in self.tracked_regexs[obj_type]:
matched = self.regex_findall(dict_regex['regex'], obj_id, content)
if matched:
self.new_tracker_found(regex, 'regex', item)
self.new_tracker_found(dict_regex['tracked'], 'regex', obj)
def new_tracker_found(self, tracker_name, tracker_type, item):
uuid_list = Tracker.get_tracker_uuid_list(tracker_name, tracker_type)
item_id = item.get_id()
# date = item.get_date()
item_source = item.get_source()
for tracker_uuid in uuid_list:
def new_tracker_found(self, tracker_name, tracker_type, obj):
obj_id = obj.get_id()
for tracker_uuid in Tracker.get_trackers_by_tracked_obj_type(tracker_type, obj.get_type(), tracker_name):
tracker = Tracker.Tracker(tracker_uuid)
# Source Filtering
tracker_sources = tracker.get_sources()
if tracker_sources and item_source not in tracker_sources:
# Filter Object
filters = tracker.get_filters()
if ail_objects.is_filtered(obj, filters):
continue
print(f'new tracked regex found: {tracker_name} in {item_id}')
self.redis_logger.warning(f'new tracked regex found: {tracker_name} in {item_id}')
# TODO
Tracker.add_tracked_item(tracker_uuid, item_id)
print(f'new tracked regex found: {tracker_name} in {obj_id}')
self.redis_logger.warning(f'new tracked regex found: {tracker_name} in {obj_id}')
if obj.get_type() == 'item':
date = obj.get_date()
else:
date = None
tracker.add(obj.get_type(), obj.get_subtype(r_str=True), obj_id, date=date)
for tag in tracker.get_tags():
msg = f'{tag};{item_id}'
self.add_message_to_queue(msg, 'Tags')
if obj.get_type() == 'item':
msg = f'{tag};{obj_id}'
self.add_message_to_queue(msg, 'Tags')
else:
obj.add_tag(tag)
if tracker.mail_export():
# TODO add matches + custom subjects
self.exporters['mail'].export(tracker, item)
self.exporters['mail'].export(tracker, obj)
if tracker.webhook_export():
self.exporters['webhook'].export(tracker, item)
self.exporters['webhook'].export(tracker, obj)
if __name__ == "__main__":
module = Tracker_Regex()
module.run()
# module.compute('submitted/2023/05/02/submitted_b1e518f1-703b-40f6-8238-d1c22888197e.gz')

92
bin/trackers/Tracker_Term.py
View File

@ -21,8 +21,7 @@ sys.path.append(os.environ['AIL_BIN'])
##################################
from modules.abstract_module import AbstractModule
from lib.ConfigLoader import ConfigLoader
from lib.objects.Items import Item
from packages import Term
from lib.objects import ail_objects
from lib import Tracker
from exporter.MailExporter import MailExporterTracker
@ -54,9 +53,9 @@ class Tracker_Term(AbstractModule):
self.max_execution_time = config_loader.get_config_int('Tracker_Term', "max_execution_time")
# loads tracked words
self.list_tracked_words = Term.get_tracked_words_list()
self.tracked_words = Tracker.get_tracked_words()
self.last_refresh_word = time.time()
self.set_tracked_words_list = Term.get_set_tracked_words_list()
self.tracked_sets = Tracker.get_tracked_sets()
self.last_refresh_set = time.time()
# Exporter
@ -65,93 +64,94 @@ class Tracker_Term(AbstractModule):
self.redis_logger.info(f"Module: {self.module_name} Launched")
def compute(self, item_id, item_content=None):
def compute(self, obj_id, obj_type='item', subtype=''):
# refresh Tracked term
if self.last_refresh_word < Term.get_tracked_term_last_updated_by_type('word'):
self.list_tracked_words = Term.get_tracked_words_list()
if self.last_refresh_word < Tracker.get_tracker_last_updated_by_type('word'):
self.tracked_words = Tracker.get_tracked_words()
self.last_refresh_word = time.time()
self.redis_logger.debug('Tracked word refreshed')
print('Tracked word refreshed')
if self.last_refresh_set < Term.get_tracked_term_last_updated_by_type('set'):
self.set_tracked_words_list = Term.get_set_tracked_words_list()
if self.last_refresh_set < Tracker.get_tracker_last_updated_by_type('set'):
self.tracked_sets = Tracker.get_tracked_sets()
self.last_refresh_set = time.time()
self.redis_logger.debug('Tracked set refreshed')
print('Tracked set refreshed')
# Cast message as Item
item = Item(item_id)
if not item_content:
item_content = item.get_content()
obj = ail_objects.get_object(obj_type, subtype, obj_id)
obj_type = obj.get_type()
# Object Filter
if obj_type not in self.tracked_words and obj_type not in self.tracked_sets:
return None
content = obj.get_content(r_str=True)
signal.alarm(self.max_execution_time)
dict_words_freq = None
try:
dict_words_freq = Term.get_text_word_frequency(item_content)
dict_words_freq = Tracker.get_text_word_frequency(content)
except TimeoutException:
self.redis_logger.warning(f"{item.get_id()} processing timeout")
self.redis_logger.warning(f"{obj.get_id()} processing timeout")
else:
signal.alarm(0)
if dict_words_freq:
# create token statistics
# for word in dict_words_freq:
# Term.create_token_statistics(item_date, word, dict_words_freq[word])
# check solo words
# ###### # TODO: check if source needed #######
for word in self.list_tracked_words:
for word in self.tracked_words[obj_type]:
if word in dict_words_freq:
self.new_term_found(word, 'word', item)
self.new_tracker_found(word, 'word', obj)
# check words set
for elem in self.set_tracked_words_list:
list_words = elem[0]
nb_words_threshold = elem[1]
word_set = elem[2]
for tracked_set in self.tracked_sets[obj_type]:
nb_uniq_word = 0
for word in list_words:
for word in tracked_set['words']:
if word in dict_words_freq:
nb_uniq_word += 1
if nb_uniq_word >= nb_words_threshold:
self.new_term_found(word_set, 'set', item)
if nb_uniq_word >= tracked_set['nb']:
self.new_tracker_found(tracked_set['tracked'], 'set', obj)
def new_term_found(self, tracker_name, tracker_type, item):
uuid_list = Tracker.get_tracker_uuid_list(tracker_name, tracker_type)
def new_tracker_found(self, tracker_name, tracker_type, obj): # TODO FILTER
obj_id = obj.get_id()
item_id = item.get_id()
item_source = item.get_source()
for tracker_uuid in uuid_list:
for tracker_uuid in Tracker.get_trackers_by_tracked_obj_type(tracker_type, obj.get_type(), tracker_name):
tracker = Tracker.Tracker(tracker_uuid)
# Source Filtering
tracker_sources = tracker.get_sources()
if tracker_sources and item_source not in tracker_sources:
# Filter Object
filters = tracker.get_filters()
if ail_objects.is_filtered(obj, filters):
continue
print(f'new tracked term found: {tracker_name} in {item_id}')
self.redis_logger.warning(f'new tracked term found: {tracker_name} in {item_id}')
# TODO
Tracker.add_tracked_item(tracker_uuid, item_id)
print(f'new tracked term found: {tracker_name} in {obj_id}')
self.redis_logger.warning(f'new tracked term found: {tracker_name} in {obj_id}')
if obj.get_type() == 'item':
date = obj.get_date()
else:
date = None
tracker.add(obj.get_type(), obj.get_subtype(), obj_id, date=date)
# Tags
for tag in tracker.get_tags():
msg = f'{tag};{item_id}'
self.add_message_to_queue(msg, 'Tags')
if obj.get_type() == 'item':
msg = f'{tag};{obj_id}'
self.add_message_to_queue(msg, 'Tags')
else:
obj.add_tag(tag)
# Mail
if tracker.mail_export():
# TODO add matches + custom subjects
self.exporters['mail'].export(tracker, item)
self.exporters['mail'].export(tracker, obj)
# Webhook
if tracker.webhook_export():
self.exporters['webhook'].export(tracker, item)
self.exporters['webhook'].export(tracker, obj)
if __name__ == '__main__':
module = Tracker_Term()
module.run()
# module.compute('submitted/2023/05/02/submitted_b1e518f1-703b-40f6-8238-d1c22888197e.gz')

68
bin/trackers/Tracker_Typo_Squatting.py
View File

@ -19,7 +19,7 @@ sys.path.append(os.environ['AIL_BIN'])
# Import Project packages
##################################
from modules.abstract_module import AbstractModule
from lib.objects.Items import Item
from lib.objects import ail_objects
from lib import Tracker
from exporter.MailExporter import MailExporterTracker
@ -36,8 +36,8 @@ class Tracker_Typo_Squatting(AbstractModule):
self.pending_seconds = 5
# Refresh typo squatting
self.typosquat_tracked_words_list = Tracker.get_typosquatting_tracked_words_list()
self.last_refresh_typosquat = time.time()
self.tracked_typosquattings = Tracker.get_tracked_typosquatting()
self.last_refresh_typosquatting = time.time()
# Exporter
self.exporters = {'mail': MailExporterTracker(),
@ -45,50 +45,62 @@ class Tracker_Typo_Squatting(AbstractModule):
self.redis_logger.info(f"Module: {self.module_name} Launched")
def compute(self, message):
def compute(self, message, obj_type='item', subtype=''):
# refresh Tracked typo
if self.last_refresh_typosquat < Tracker.get_tracker_last_updated_by_type('typosquatting'):
self.typosquat_tracked_words_list = Tracker.get_typosquatting_tracked_words_list()
self.last_refresh_typosquat = time.time()
if self.last_refresh_typosquatting < Tracker.get_tracker_last_updated_by_type('typosquatting'):
self.tracked_typosquattings = Tracker.get_tracked_typosquatting()
self.last_refresh_typosquatting = time.time()
self.redis_logger.debug('Tracked typosquatting refreshed')
print('Tracked typosquatting refreshed')
host, item_id = message.split()
host, obj_id = message.split()
obj = ail_objects.get_object(obj_type, subtype, obj_id)
obj_type = obj.get_type()
# Cast message as Item
for tracker in self.typosquat_tracked_words_list:
if host in self.typosquat_tracked_words_list[tracker]:
item = Item(item_id)
self.new_tracker_found(tracker, 'typosquatting', item)
# Object Filter
if obj_type not in self.tracked_typosquattings:
return None
def new_tracker_found(self, tracker, tracker_type, item):
item_id = item.get_id()
item_source = item.get_source()
print(f'new tracked typosquatting found: {tracker} in {item_id}')
self.redis_logger.warning(f'tracker typosquatting: {tracker} in {item_id}')
for typo in self.tracked_typosquattings[obj_type]:
if host in typo['domains']:
self.new_tracker_found(typo['tracked'], 'typosquatting', obj)
for tracker_uuid in Tracker.get_tracker_uuid_list(tracker, tracker_type):
def new_tracker_found(self, tracked, tracker_type, obj):
obj_id = obj.get_id()
for tracker_uuid in Tracker.get_trackers_by_tracked_obj_type(tracker_type, obj.get_type(), tracked):
tracker = Tracker.Tracker(tracker_uuid)
# Source Filtering
tracker_sources = tracker.get_sources()
if tracker_sources and item_source not in tracker_sources:
# Filter Object
filters = tracker.get_filters()
if ail_objects.is_filtered(obj, filters):
continue
Tracker.add_tracked_item(tracker_uuid, item_id)
print(f'new tracked typosquatting found: {tracked} in {obj_id}')
self.redis_logger.warning(f'tracker typosquatting: {tracked} in {obj_id}')
if obj.get_type() == 'item':
date = obj.get_date()
else:
date = None
tracker.add(obj.get_type(), obj.get_subtype(r_str=True), obj_id, date=date)
# Tags
for tag in tracker.get_tags():
msg = f'{tag};{item_id}'
self.add_message_to_queue(msg, 'Tags')
if obj.get_type() == 'item':
msg = f'{tag};{obj_id}'
self.add_message_to_queue(msg, 'Tags')
else:
obj.add_tag(tag)
if tracker.mail_export():
self.exporters['mail'].export(tracker, item)
self.exporters['mail'].export(tracker, obj)
if tracker.webhook_export():
self.exporters['webhook'].export(tracker, item)
self.exporters['webhook'].export(tracker, obj)
if __name__ == '__main__':
module = Tracker_Typo_Squatting()
module.run()
#module.compute('g00gle.com tests/2020/01/01/test.gz')
# module.compute('g00gle.com tests/2020/01/01/test.gz')

84
bin/trackers/Tracker_Yara.py
View File

@ -19,7 +19,7 @@ sys.path.append(os.environ['AIL_BIN'])
# Import Project packages
##################################
from modules.abstract_module import AbstractModule
from lib.objects.Items import Item
from lib.objects import ail_objects
from lib import Tracker
from exporter.MailExporter import MailExporterTracker
@ -35,10 +35,10 @@ class Tracker_Yara(AbstractModule):
self.pending_seconds = 5
# Load Yara rules
self.rules = Tracker.reload_yara_rules()
self.rules = Tracker.get_tracked_yara_rules()
self.last_refresh = time.time()
self.item = None
self.obj = None
# Exporter
self.exporters = {'mail': MailExporterTracker(),
@ -46,58 +46,67 @@ class Tracker_Yara(AbstractModule):
self.redis_logger.info(f"Module: {self.module_name} Launched")
def compute(self, item_id, item_content=None):
def compute(self, obj_id, obj_type='item', subtype=''):
# refresh YARA list
if self.last_refresh < Tracker.get_tracker_last_updated_by_type('yara'):
self.rules = Tracker.reload_yara_rules()
self.rules = Tracker.get_tracked_yara_rules()
self.last_refresh = time.time()
self.redis_logger.debug('Tracked set refreshed')
print('Tracked set refreshed')
self.item = Item(item_id)
if not item_content:
item_content = self.item.get_content()
self.obj = ail_objects.get_object(obj_type, subtype, obj_id)
obj_type = self.obj.get_type()
# Object Filter
if obj_type not in self.rules:
return None
content = self.obj.get_content(r_str=True)
try:
yara_match = self.rules.match(data=item_content, callback=self.yara_rules_match,
which_callbacks=yara.CALLBACK_MATCHES, timeout=60)
yara_match = self.rules[obj_type].match(data=content, callback=self.yara_rules_match,
which_callbacks=yara.CALLBACK_MATCHES, timeout=60)
if yara_match:
self.redis_logger.warning(f'tracker yara: new match {self.item.get_id()}: {yara_match}')
print(f'{self.item.get_id()}: {yara_match}')
self.redis_logger.warning(f'tracker yara: new match {self.obj.get_id()}: {yara_match}')
print(f'{self.obj.get_id()}: {yara_match}')
except yara.TimeoutError:
print(f'{self.item.get_id()}: yara scanning timed out')
self.redis_logger.info(f'{self.item.get_id()}: yara scanning timed out')
print(f'{self.obj.get_id()}: yara scanning timed out')
self.redis_logger.info(f'{self.obj.get_id()}: yara scanning timed out')
def yara_rules_match(self, data):
tracker_uuid = data['namespace']
item_id = self.item.get_id()
tracker_name = data['namespace']
obj_id = self.obj.get_id()
for tracker_uuid in Tracker.get_trackers_by_tracked_obj_type('yara', self.obj.get_type(), tracker_name):
tracker = Tracker.Tracker(tracker_uuid)
item = Item(item_id)
item_source = self.item.get_source()
# Filter Object
filters = tracker.get_filters()
if ail_objects.is_filtered(self.obj, filters):
continue
tracker = Tracker.Tracker(tracker_uuid)
if self.obj.get_type() == 'item':
date = self.obj.get_date()
else:
date = None
# Source Filtering
tracker_sources = tracker.get_sources()
if tracker_sources and item_source not in tracker_sources:
print(f'Source Filtering: {data["rule"]}')
return yara.CALLBACK_CONTINUE
tracker.add(self.obj.get_type(), self.obj.get_subtype(r_str=True), obj_id, date=date)
Tracker.add_tracked_item(tracker_uuid, item_id) # TODO
# Tags
for tag in tracker.get_tags():
if self.obj.get_type() == 'item':
msg = f'{tag};{obj_id}'
self.add_message_to_queue(msg, 'Tags')
else:
self.obj.add_tag(tag)
# Tags
for tag in tracker.get_tags():
msg = f'{tag};{item_id}'
self.add_message_to_queue(msg, 'Tags')
# Mails
if tracker.mail_export():
# TODO add matches + custom subjects
self.exporters['mail'].export(tracker, self.obj)
# Mails
if tracker.mail_export():
# TODO add matches + custom subjects
self.exporters['mail'].export(tracker, item)
# Webhook
if tracker.webhook_export():
self.exporters['webhook'].export(tracker, item)
# Webhook
if tracker.webhook_export():
self.exporters['webhook'].export(tracker, self.obj)
return yara.CALLBACK_CONTINUE
@ -105,3 +114,4 @@ class Tracker_Yara(AbstractModule):
if __name__ == '__main__':
module = Tracker_Yara()
module.run()
# module.compute('archive/gist.github.com/2023/04/13/chipzoller_d8d6d2d737d02ad4fe9d30a897170761.gz')

137
update/v2.2/Update.py
View File

@ -39,74 +39,75 @@ if __name__ == '__main__':
r_serv_term_stats.flushdb()
# convert all regex:
all_regex = r_serv_termfreq.smembers('TrackedRegexSet')
for regex in all_regex:
tags = list(r_serv_termfreq.smembers('TrackedNotificationTags_{}'.format(regex)))
mails = list(r_serv_termfreq.smembers('TrackedNotificationEmails_{}'.format(regex)))
new_term = regex[1:-1]
res = Term.parse_json_term_to_add({"term": new_term, "type": 'regex', "tags": tags, "mails": mails, "level": 1},
'admin@admin.test')
if res[1] == 200:
term_uuid = res[0]['uuid']
list_items = r_serv_termfreq.smembers('regex_{}'.format(regex))
for paste_item in list_items:
item_id = get_item_id(paste_item)
item_date = get_item_date(item_id)
Term.add_tracked_item(term_uuid, item_id, item_date)
# Invalid Tracker => remove it
else:
print('Invalid Regex Removed: {}'.format(regex))
print(res[0])
# allow reprocess
r_serv_termfreq.srem('TrackedRegexSet', regex)
all_tokens = r_serv_termfreq.smembers('TrackedSetTermSet')
for token in all_tokens:
tags = list(r_serv_termfreq.smembers('TrackedNotificationTags_{}'.format(token)))
mails = list(r_serv_termfreq.smembers('TrackedNotificationEmails_{}'.format(token)))
res = Term.parse_json_term_to_add({"term": token, "type": 'word', "tags": tags, "mails": mails, "level": 1}, 'admin@admin.test')
if res[1] == 200:
term_uuid = res[0]['uuid']
list_items = r_serv_termfreq.smembers('tracked_{}'.format(token))
for paste_item in list_items:
item_id = get_item_id(paste_item)
item_date = get_item_date(item_id)
Term.add_tracked_item(term_uuid, item_id, item_date)
# Invalid Tracker => remove it
else:
print('Invalid Token Removed: {}'.format(token))
print(res[0])
# allow reprocess
r_serv_termfreq.srem('TrackedSetTermSet', token)
all_set = r_serv_termfreq.smembers('TrackedSetSet')
for curr_set in all_set:
tags = list(r_serv_termfreq.smembers('TrackedNotificationTags_{}'.format(curr_set)))
mails = list(r_serv_termfreq.smembers('TrackedNotificationEmails_{}'.format(curr_set)))
to_remove = ',{}'.format(curr_set.split(',')[-1])
new_set = rreplace(curr_set, to_remove, '', 1)
new_set = new_set[2:]
new_set = new_set.replace(',', '')
res = Term.parse_json_term_to_add({"term": new_set, "type": 'set', "nb_words": 1, "tags": tags, "mails": mails, "level": 1}, 'admin@admin.test')
if res[1] == 200:
term_uuid = res[0]['uuid']
list_items = r_serv_termfreq.smembers('tracked_{}'.format(curr_set))
for paste_item in list_items:
item_id = get_item_id(paste_item)
item_date = get_item_date(item_id)
Term.add_tracked_item(term_uuid, item_id, item_date)
# Invalid Tracker => remove it
else:
print('Invalid Set Removed: {}'.format(curr_set))
print(res[0])
# allow reprocess
r_serv_termfreq.srem('TrackedSetSet', curr_set)
# Disabled. Checkout the v2.2 branch if you need it
# # convert all regex:
# all_regex = r_serv_termfreq.smembers('TrackedRegexSet')
# for regex in all_regex:
# tags = list(r_serv_termfreq.smembers('TrackedNotificationTags_{}'.format(regex)))
# mails = list(r_serv_termfreq.smembers('TrackedNotificationEmails_{}'.format(regex)))
#
# new_term = regex[1:-1]
# res = Term.parse_json_term_to_add({"term": new_term, "type": 'regex', "tags": tags, "mails": mails, "level": 1},
# 'admin@admin.test')
# if res[1] == 200:
# term_uuid = res[0]['uuid']
# list_items = r_serv_termfreq.smembers('regex_{}'.format(regex))
# for paste_item in list_items:
# item_id = get_item_id(paste_item)
# item_date = get_item_date(item_id)
# Term.add_tracked_item(term_uuid, item_id, item_date)
#
# # Invalid Tracker => remove it
# else:
# print('Invalid Regex Removed: {}'.format(regex))
# print(res[0])
# # allow reprocess
# r_serv_termfreq.srem('TrackedRegexSet', regex)
#
# all_tokens = r_serv_termfreq.smembers('TrackedSetTermSet')
# for token in all_tokens:
# tags = list(r_serv_termfreq.smembers('TrackedNotificationTags_{}'.format(token)))
# mails = list(r_serv_termfreq.smembers('TrackedNotificationEmails_{}'.format(token)))
#
# res = Term.parse_json_term_to_add({"term": token, "type": 'word', "tags": tags, "mails": mails, "level": 1}, 'admin@admin.test')
# if res[1] == 200:
# term_uuid = res[0]['uuid']
# list_items = r_serv_termfreq.smembers('tracked_{}'.format(token))
# for paste_item in list_items:
# item_id = get_item_id(paste_item)
# item_date = get_item_date(item_id)
# Term.add_tracked_item(term_uuid, item_id, item_date)
# # Invalid Tracker => remove it
# else:
# print('Invalid Token Removed: {}'.format(token))
# print(res[0])
# # allow reprocess
# r_serv_termfreq.srem('TrackedSetTermSet', token)
#
# all_set = r_serv_termfreq.smembers('TrackedSetSet')
# for curr_set in all_set:
# tags = list(r_serv_termfreq.smembers('TrackedNotificationTags_{}'.format(curr_set)))
# mails = list(r_serv_termfreq.smembers('TrackedNotificationEmails_{}'.format(curr_set)))
#
# to_remove = ',{}'.format(curr_set.split(',')[-1])
# new_set = rreplace(curr_set, to_remove, '', 1)
# new_set = new_set[2:]
# new_set = new_set.replace(',', '')
#
# res = Term.parse_json_term_to_add({"term": new_set, "type": 'set', "nb_words": 1, "tags": tags, "mails": mails, "level": 1}, 'admin@admin.test')
# if res[1] == 200:
# term_uuid = res[0]['uuid']
# list_items = r_serv_termfreq.smembers('tracked_{}'.format(curr_set))
# for paste_item in list_items:
# item_id = get_item_id(paste_item)
# item_date = get_item_date(item_id)
# Term.add_tracked_item(term_uuid, item_id, item_date)
# # Invalid Tracker => remove it
# else:
# print('Invalid Set Removed: {}'.format(curr_set))
# print(res[0])
# # allow reprocess
# r_serv_termfreq.srem('TrackedSetSet', curr_set)
r_serv_termfreq.flushdb()

2
update/v3.7/Update.py
View File

@ -24,7 +24,7 @@ class Updater(AIL_Updater):
print('Fixing Tracker_uuid list ...')
Tracker.fix_all_tracker_uuid_list()
nb = 0
for tracker_uuid in Tracker.get_all_tracker_uuid():
for tracker_uuid in Tracker.get_trackers():
self.r_serv.sadd('trackers_update_v3.7', tracker_uuid)
nb += 1

209
var/www/blueprints/hunters.py
View File

@ -1,15 +1,15 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
'''
"""
Blueprint Flask: crawler splash endpoints: dashboard, onion crawler ...
'''
"""
import os
import sys
import json
from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response, make_response
from flask import render_template, jsonify, request, Blueprint, redirect, url_for, Response
from flask_login import login_required, current_user, login_user, logout_user
sys.path.append('modules')
@ -22,8 +22,10 @@ sys.path.append(os.environ['AIL_BIN'])
##################################
# Import Project packages
##################################
from lib import ail_core
from lib import item_basic
from lib import Tracker
from lib import Tag
bootstrap_label = Flask_config.bootstrap_label
@ -34,7 +36,6 @@ hunters = Blueprint('hunters', __name__, template_folder=os.path.join(os.environ
# ============ VARIABLES ============
# ============ FUNCTIONS ============
def api_validator(api_response):
if api_response:
@ -45,11 +46,203 @@ def create_json_response(data, status_code):
# ============= ROUTES ==============
##################
# TRACKERS #
##################
@hunters.route('/trackers', methods=['GET'])
@login_required
@login_read_only
def trackers_dashboard():
user_id = current_user.get_id() # TODO
trackers = Tracker.get_trackers_dashboard()
stats = Tracker.get_trackers_stats(user_id)
return render_template("trackers_dashboard.html", trackers=trackers, stats=stats, bootstrap_label=bootstrap_label)
@hunters.route("/trackers/all")
@login_required
@login_read_only
def tracked_menu():
user_id = current_user.get_id()
user_trackers = Tracker.get_user_trackers_meta(user_id)
global_trackers = Tracker.get_global_trackers_meta()
return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers, bootstrap_label=bootstrap_label)
@hunters.route("/trackers/word")
@login_required
@login_read_only
def tracked_menu_word():
tracker_type = 'word'
user_id = current_user.get_id()
user_trackers = Tracker.get_user_trackers_meta(user_id, tracker_type='word')
global_trackers = Tracker.get_global_trackers_meta(tracker_type='word')
return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers, bootstrap_label=bootstrap_label, tracker_type=tracker_type)
@hunters.route("/trackers/set")
@login_required
@login_read_only
def tracked_menu_set():
tracker_type = 'set'
user_id = current_user.get_id()
user_trackers = Tracker.get_user_trackers_meta(user_id, tracker_type=tracker_type)
global_trackers = Tracker.get_global_trackers_meta(tracker_type=tracker_type)
return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers, bootstrap_label=bootstrap_label, tracker_type=tracker_type)
@hunters.route("/trackers/regex")
@login_required
@login_read_only
def tracked_menu_regex():
tracker_type = 'regex'
user_id = current_user.get_id()
user_trackers = Tracker.get_user_trackers_meta(user_id, tracker_type=tracker_type)
global_trackers = Tracker.get_global_trackers_meta(tracker_type=tracker_type)
return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers, bootstrap_label=bootstrap_label, tracker_type=tracker_type)
@hunters.route("/trackers/yara")
@login_required
@login_read_only
def tracked_menu_yara():
tracker_type = 'yara'
user_id = current_user.get_id()
user_trackers = Tracker.get_user_trackers_meta(user_id, tracker_type=tracker_type)
global_trackers = Tracker.get_global_trackers_meta(tracker_type=tracker_type)
return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers, bootstrap_label=bootstrap_label, tracker_type=tracker_type)
@hunters.route("/trackers/typosquatting")
@login_required
@login_read_only
def tracked_menu_typosquatting():
tracker_type = 'typosquatting'
user_id = current_user.get_id()
user_trackers = Tracker.get_user_trackers_meta(user_id, tracker_type=tracker_type)
global_trackers = Tracker.get_global_trackers_meta(tracker_type=tracker_type)
return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers,
bootstrap_label=bootstrap_label, tracker_type=tracker_type)
@hunters.route("/tracker/add", methods=['GET', 'POST'])
@login_required
@login_analyst
def add_tracked_menu():
if request.method == 'POST':
to_track = request.form.get("tracker")
tracker_uuid = request.form.get("tracker_uuid")
tracker_type = request.form.get("tracker_type")
nb_words = request.form.get("nb_word", 1)
description = request.form.get("description", '')
webhook = request.form.get("webhook", '')
level = request.form.get("level", 0)
mails = request.form.get("mails", [])
# TAGS
tags = request.form.get("tags", [])
taxonomies_tags = request.form.get('taxonomies_tags')
if taxonomies_tags:
try:
taxonomies_tags = json.loads(taxonomies_tags)
except:
taxonomies_tags = []
else:
taxonomies_tags = []
galaxies_tags = request.form.get('galaxies_tags')
if galaxies_tags:
try:
galaxies_tags = json.loads(galaxies_tags)
except:
galaxies_tags = []
# custom tags
if tags:
tags = tags.split()
else:
tags = []
tags = tags + taxonomies_tags + galaxies_tags
# YARA #
if tracker_type == 'yara':
yara_default_rule = request.form.get("yara_default_rule")
yara_custom_rule = request.form.get("yara_custom_rule")
if yara_custom_rule:
to_track = yara_custom_rule
tracker_type = 'yara_custom'
else:
to_track = yara_default_rule
tracker_type = 'yara_default'
if level == 'on':
level = 1
else:
level = 0
if mails:
mails = mails.split()
else:
tags = []
# FILTERS
filters = {}
for obj_type in Tracker.get_objects_tracked():
new_filter = request.form.get(f'{obj_type}_obj')
if new_filter == 'on':
filters[obj_type] = {}
# Mimetypes
mimetypes = request.form.get(f'mimetypes_{obj_type}', [])
if mimetypes:
mimetypes = json.loads(mimetypes)
filters[obj_type]['mimetypes'] = mimetypes
# Sources
sources = request.form.get(f'sources_{obj_type}', [])
if sources:
sources = json.loads(sources)
filters[obj_type]['sources'] = sources
# Subtypes
for obj_subtype in ail_core.get_object_all_subtypes(obj_type):
subtype = request.form.get(f'filter_{obj_type}_{obj_subtype}')
if subtype == 'on':
if 'subtypes' not in filters[obj_type]:
filters[obj_type]['subtypes'] = []
filters[obj_type]['subtypes'].append(obj_subtype)
input_dict = {"tracked": to_track, "type": tracker_type,
"tags": tags, "mails": mails, "filters": filters,
"level": level, "description": description, "webhook": webhook}
if tracker_type == 'set':
try:
input_dict['nb_words'] = int(nb_words)
except TypeError:
input_dict['nb_words'] = 1
user_id = current_user.get_id()
res = Tracker.api_add_tracker(input_dict, user_id)
if res[1] == 200:
return redirect(url_for('hunters.trackers_dashboard'))
else:
return create_json_response(res[0], res[1])
else:
return render_template("tracker_add.html",
all_sources=item_basic.get_all_items_sources(r_list=True),
tags_selector_data=Tag.get_tags_selector_data(),
all_yara_files=Tracker.get_all_default_yara_files())
@hunters.route('/tracker/delete', methods=['GET'])
@login_required
@login_analyst
def tracker_delete():
user_id = current_user.get_id()
tracker_uuid = request.args.get('uuid')
res = Tracker.api_delete_tracker({'uuid': tracker_uuid}, user_id)
if res[1] != 200:
return create_json_response(res[0], res[1])
else:
return redirect(url_for('hunter.tracked_menu'))
####################
# RETRO HUNT #
####################
@hunters.route('/retro_hunt/tasks', methods=['GET'])
@login_required
@login_read_only
def retro_hunt_all_tasks():
retro_hunts = Tracker.get_all_retro_hunt_tasks_with_metadata()
retro_hunts = Tracker.get_retro_hunt_tasks_metas()
return render_template("retro_hunt_tasks.html", retro_hunts=retro_hunts, bootstrap_label=bootstrap_label)
@hunters.route('/retro_hunt/task/show', methods=['GET'])
@ -69,8 +262,8 @@ def retro_hunt_show_task():
if res:
return create_json_response(res[0], res[1])
dict_task = Tracker.get_retro_hunt_task_metadata(task_uuid, date=True, progress=True, creator=True,
sources=True, tags=True, description=True)
retro_hunt = Tracker.RetroHunt(task_uuid)
dict_task = retro_hunt.get_meta(options={'creator', 'date', 'description', 'progress', 'sources', 'tags'})
rule_content = Tracker.get_yara_rule_content(dict_task['rule'])
if date_from:
@ -177,7 +370,7 @@ def retro_hunt_delete_task():
#### JSON ####
@hunters.route("/tracker/get_json_retro_hunt_nb_items_by_date", methods=['GET'])
@hunters.route("/retro_hunt/nb_items/date/json", methods=['GET'])
@login_required
@login_read_only
def get_json_retro_hunt_nb_items_by_date():

208
var/www/modules/hunter/Flask_hunter.py
View File

@ -17,10 +17,11 @@ sys.path.append(os.environ['AIL_BIN'])
##################################
# Import Project packages
##################################
from lib.objects import ail_objects
from lib import item_basic
from lib import Tracker
from lib import Tag
from packages import Term
from packages import Date
# ============ VARIABLES ============
@ -34,142 +35,11 @@ hunter = Blueprint('hunter', __name__, template_folder='templates')
# ============ FUNCTIONS ============
def create_json_response(data, status_code):
return Response(json.dumps(data, indent=2, sort_keys=True), mimetype='application/json'), status_code
# ============ ROUTES ============
@hunter.route("/trackers")
@login_required
@login_read_only
def tracked_menu():
user_id = current_user.get_id()
user_trackers = Tracker.get_user_trackers_metadata(user_id)
global_trackers = Tracker.get_global_trackers_metadata()
return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers, bootstrap_label=bootstrap_label)
@hunter.route("/trackers/word")
@login_required
@login_read_only
def tracked_menu_word():
tracker_type = 'word'
user_id = current_user.get_id()
user_trackers = Tracker.get_user_trackers_metadata(user_id, tracker_type='word')
global_trackers = Tracker.get_global_trackers_metadata(tracker_type='word')
return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers, bootstrap_label=bootstrap_label, tracker_type=tracker_type)
@hunter.route("/trackers/set")
@login_required
@login_read_only
def tracked_menu_set():
tracker_type = 'set'
user_id = current_user.get_id()
user_trackers = Tracker.get_user_trackers_metadata(user_id, tracker_type=tracker_type)
global_trackers = Tracker.get_global_trackers_metadata(tracker_type=tracker_type)
return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers, bootstrap_label=bootstrap_label, tracker_type=tracker_type)
@hunter.route("/trackers/regex")
@login_required
@login_read_only
def tracked_menu_regex():
tracker_type = 'regex'
user_id = current_user.get_id()
user_trackers = Tracker.get_user_trackers_metadata(user_id, tracker_type=tracker_type)
global_trackers = Tracker.get_global_trackers_metadata(tracker_type=tracker_type)
return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers, bootstrap_label=bootstrap_label, tracker_type=tracker_type)
@hunter.route("/trackers/yara")
@login_required
@login_read_only
def tracked_menu_yara():
tracker_type = 'yara'
user_id = current_user.get_id()
user_trackers = Tracker.get_user_trackers_metadata(user_id, tracker_type=tracker_type)
global_trackers = Tracker.get_global_trackers_metadata(tracker_type=tracker_type)
return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers, bootstrap_label=bootstrap_label, tracker_type=tracker_type)
@hunter.route("/trackers/typosquatting")
@login_required
@login_read_only
def tracked_menu_typosquatting():
tracker_type = 'typosquatting'
user_id = current_user.get_id()
user_trackers = Tracker.get_user_trackers_metadata(user_id, tracker_type=tracker_type)
global_trackers = Tracker.get_global_trackers_metadata(tracker_type=tracker_type)
return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers,
bootstrap_label=bootstrap_label, tracker_type=tracker_type)
@hunter.route("/tracker/add", methods=['GET', 'POST'])
@login_required
@login_analyst
def add_tracked_menu():
if request.method == 'POST':
tracker = request.form.get("tracker")
tracker_uuid = request.form.get("tracker_uuid")
tracker_type = request.form.get("tracker_type")
nb_words = request.form.get("nb_word", 1)
description = request.form.get("description", '')
webhook = request.form.get("webhook", '')
level = request.form.get("level", 0)
mails = request.form.get("mails", [])
sources = request.form.get("sources", [])
tags = request.form.get("tags", [])
taxonomies_tags = request.form.get('taxonomies_tags')
if taxonomies_tags:
try:
taxonomies_tags = json.loads(taxonomies_tags)
except Exception:
taxonomies_tags = []
else:
taxonomies_tags = []
galaxies_tags = request.form.get('galaxies_tags')
if galaxies_tags:
try:
galaxies_tags = json.loads(galaxies_tags)
except Exception:
galaxies_tags = []
# YARA #
if tracker_type == 'yara':
yara_default_rule = request.form.get("yara_default_rule")
yara_custom_rule = request.form.get("yara_custom_rule")
if yara_custom_rule:
tracker = yara_custom_rule
tracker_type='yara_custom'
else:
tracker = yara_default_rule
tracker_type='yara_default'
if level == 'on':
level = 1
if mails:
mails = mails.split()
if tags:
tags = tags.split()
if sources:
sources = json.loads(sources)
input_dict = {"tracker": tracker, "type": tracker_type, "nb_words": nb_words,
"tags": tags, "mails": mails, "sources": sources,
"level": level, "description": description, "webhook": webhook}
user_id = current_user.get_id()
# edit tracker
if tracker_uuid:
input_dict['uuid'] = tracker_uuid
res = Tracker.api_add_tracker(input_dict, user_id)
if res[1] == 200:
if 'uuid' in res[0]:
return redirect(url_for('hunter.show_tracker', uuid=res[0]['uuid']))
else:
return redirect(url_for('hunter.tracked_menu'))
else:
## TODO: use modal
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
else:
return render_template("edit_tracker.html",
all_sources=item_basic.get_all_items_sources(r_list=True),
tags_selector_data=Tag.get_tags_selector_data(),
all_yara_files=Tracker.get_all_default_yara_files())
@hunter.route("/tracker/edit", methods=['GET', 'POST'])
@login_required
@login_analyst
@ -181,7 +51,8 @@ def edit_tracked_menu():
if res[1] != 200: # invalid access
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
dict_tracker = Tracker.get_tracker_metadata(tracker_uuid, user_id=True, level=True, description=True, tags=True, mails=True, sources=True, webhook=True)
tracker = Tracker.Tracker(tracker_uuid)
dict_tracker = tracker.get_meta(options={'description', 'level', 'mails', 'sources', 'tags', 'user', 'webhook'})
dict_tracker['tags'] = ' '.join(dict_tracker['tags'])
dict_tracker['mails'] = ' '.join(dict_tracker['mails'])
@ -205,7 +76,7 @@ def edit_tracked_menu():
# word
# set of word + nb words
# regex
# yara custum
# yara custom
# yara default ???? => allow edit ?
#### EDIT SHow Trackers ??????????????????????????????????????????????????
@ -228,34 +99,36 @@ def show_tracker():
if date_to:
date_to = date_to.replace('-', '')
tracker_metadata = Tracker.get_tracker_metadata(tracker_uuid, user_id=True, level=True, description=True, tags=True, mails=True, sources=True, sparkline=True, webhook=True)
tracker = Tracker.Tracker(tracker_uuid)
meta = tracker.get_meta(options={'description', 'level', 'mails', 'filters', 'sparkline', 'tags',
'user', 'webhook'})
if tracker_metadata['type'] == 'yara':
yara_rule_content = Tracker.get_yara_rule_content(tracker_metadata['tracker'])
if meta['type'] == 'yara':
yara_rule_content = Tracker.get_yara_rule_content(meta['tracked'])
else:
yara_rule_content = None
if tracker_metadata['type'] == 'typosquatting':
typo_squatting = list(Tracker.get_tracker_typosquatting_domains(tracker_uuid))
typo_squatting.sort()
if meta['type'] == 'typosquatting':
typo_squatting = Tracker.get_tracked_typosquatting_domains(meta['tracked'])
sorted(typo_squatting)
else:
typo_squatting = None
typo_squatting = set()
if date_from:
res = Term.parse_get_tracker_term_item({'uuid': tracker_uuid, 'date_from': date_from, 'date_to': date_to}, user_id)
if res[1] != 200:
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
tracker_metadata['items'] = res[0]['items']
tracker_metadata['date_from'] = res[0]['date_from']
tracker_metadata['date_to'] = res[0]['date_to']
date_from, date_to = Date.sanitise_daterange(meta['first_seen'], meta['last_seen'])
objs = tracker.get_objs_by_daterange(date_from, date_to)
meta['objs'] = ail_objects.get_objects_meta(objs, flask_context=True)
else:
tracker_metadata['items'] = []
tracker_metadata['date_from'] = ''
tracker_metadata['date_to'] = ''
date_from = ''
date_to = ''
meta['objs'] = []
tracker_metadata['sources'] = sorted(tracker_metadata['sources'])
meta['date_from'] = date_from
meta['date_to'] = date_to
print(meta['filters'])
meta['item_sources'] = sorted(meta['filters'].get('item', {}).get('sources', []))
return render_template("showTracker.html", tracker_metadata=tracker_metadata,
return render_template("showTracker.html", tracker_metadata=meta,
yara_rule_content=yara_rule_content,
typo_squatting=typo_squatting,
bootstrap_label=bootstrap_label)
@ -309,21 +182,16 @@ def update_tracker_mails():
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
return redirect(url_for('hunter.show_tracker', uuid=term_uuid))
@hunter.route("/tracker/delete", methods=['GET'])
@login_required
@login_analyst
def delete_tracker():
user_id = current_user.get_id()
term_uuid = request.args.get('uuid')
res = Term.parse_tracked_term_to_delete({'uuid': term_uuid}, user_id)
if res[1] !=200:
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
return redirect(url_for('hunter.tracked_menu'))
@hunter.route("/tracker/get_json_tracker_stats", methods=['GET'])
@login_required
@login_read_only
def get_json_tracker_stats():
user_id = current_user.get_id()
tracker_uuid = request.args.get('uuid')
res = Tracker.api_check_tracker_acl(tracker_uuid, user_id)
if res:
return create_json_response(res[0], res[1])
date_from = request.args.get('date_from')
date_to = request.args.get('date_to')
@ -331,13 +199,10 @@ def get_json_tracker_stats():
date_from = date_from.replace('-', '')
if date_to:
date_to = date_to.replace('-', '')
tracker_uuid = request.args.get('uuid')
if date_from and date_to:
res = Term.get_list_tracked_term_stats_by_day([tracker_uuid], date_from=date_from, date_to=date_to)
res = Tracker.get_trackers_graph_by_day([tracker_uuid], date_from=date_from, date_to=date_to)
else:
res = Term.get_list_tracked_term_stats_by_day([tracker_uuid])
res = Tracker.get_trackers_graph_by_day([tracker_uuid])
return jsonify(res)
@hunter.route("/tracker/yara/default_rule/content", methods=['GET'])
@ -348,5 +213,6 @@ def get_default_yara_rule_content():
res = Tracker.api_get_default_rule_content(default_yara_rule)
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
# ========= REGISTRATION =========
app.register_blueprint(hunter, url_prefix=baseUrl)

260
var/www/modules/hunter/templates/edit_tracker.html
View File

@ -1,260 +0,0 @@
<!DOCTYPE html>
<html>
<head>
<title>AIL-Framework</title>
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png')}}">
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/daterangepicker.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/tags.css') }}" rel="stylesheet">
<!-- JS -->
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/tags.js') }}"></script>
</head>
<body>
{% include 'nav_bar.html' %}
<div class="container-fluid">
<div class="row">
{% include 'hunter/menu_sidebar.html' %}
<div class="col-12 col-lg-10" id="core_content">
<div class="card my-3">
<div class="card-header bg-dark text-white">
<h5 class="card-title">{%if dict_tracker%}Edit a{%else%}Create a new{%endif%} Tracker</h5>
</div>
<div class="card-body">
<form action="{{ url_for('hunter.add_tracked_menu') }}" method='post'>
{%if dict_tracker%}
<input id="tracker_uuid" name="tracker_uuid" class="form-control" type="text" value="{{dict_tracker['uuid']}}" hidden>
{%endif%}
<div class="row">
<div class="col-12 col-xl-9">
<div class="input-group mb-2 mr-sm-2">
<div class="input-group-prepend">
<div class="input-group-text bg-secondary text-white"><i class="fas fa-at"></i></div>
</div>
<input id="mails" name="mails" class="form-control" placeholder="E-Mails Notification (optional, space separated)" type="text" {%if dict_tracker%}{%if dict_tracker['mails']%}value="{{dict_tracker['mails']}}"{%endif%}{%endif%}>
</div>
<div class="input-group mb-2 mr-sm-2">
<div class="input-group-prepend">
<div class="input-group-text bg-info text-white"><i class="fas fa-anchor"></i></div>
</div>
<input id="webhook" name="webhook" class="form-control" placeholder="Webhook URL" type="text" {%if dict_tracker%}{%if dict_tracker['webhook']%}value="{{dict_tracker['webhook']}}"{%endif%}{%endif%}>
</div>
<div class="input-group mb-2 mr-sm-2">
<div class="input-group-prepend">
<div class="input-group-text bg-info text-white"><i class="fas fa-pencil-alt"></i></div>
</div>
<input id="description" name="description" class="form-control" placeholder="Tracker Description (optional)" type="text" {%if dict_tracker%}{%if dict_tracker['description']%}value="{{dict_tracker['description']}}"{%endif%}{%endif%}>
</div>
<div class="input-group mb-2 mr-sm-2">
<div class="input-group-prepend">
<div class="input-group-text bg-dark text-white"><i class="fas fa-folder"></i></div>
</div>
<input id="sources" class="form-control" type="text" name="sources" placeholder="Sources to track (ALL IF EMPTY)" autocomplete="off">
</div>
<div class="card my-4">
<div class="card-header bg-secondary text-white">
<b>Tags</b>
</div>
<div class="card-body">
<div class="input-group mb-2 mr-sm-2">
<div class="input-group-prepend">
<div class="input-group-text bg-danger text-white"><i class="fas fa-tag"></i></div>
</div>
<input id="tags" name="tags" class="form-control" placeholder="Custom Tags (optional, space separated)" type="text" {%if dict_tracker%}{%if dict_tracker['tags']%}value="{{dict_tracker['tags']}}"{%endif%}{%endif%}>
</div>
{% include 'tags/block_tags_selector.html' %}
</div>
</div>
</div>
<div class="col-12 col-xl-3">
<div class="custom-control custom-switch mt-1">
<input class="custom-control-input" type="checkbox" name="level" id="id_level" {%if dict_tracker%}{%if dict_tracker['level']==1%}checked{%endif%}{%else%}checked{%endif%}>
<label class="custom-control-label" for="id_level">
<i class="fas fa-users"></i>&nbsp;Show tracker to all Users
</label>
</div>
</div>
</div>
<hr>
<h4>Tracker Type:</h4>
<select id="tracker_type" name="tracker_type" class="custom-select w-25 mb-3">
<option disabled selected value> -- Select a tracker type -- </option>
<option value="word">Word</option>
<option value="set">Set</option>
<option value="regex">Regex</option>
<option value="yara">YARA rule</option>
<option value="typosquatting">Typo-squatting</option>
</select>
<p id="tracker_desc">Terms to track (space separated)</p>
<div class="row" id="simple_input">
<div class="col-12 col-lg-10">
<input id="tracker" name="tracker" class="form-control" placeholder="Terms to track (space separated)" type="text" {%if dict_tracker%}{%if dict_tracker['tracker']!='yara'%}value="{{dict_tracker['tracker']}}"{%endif%}{%endif%}>
</div>
<div class="col-12 col-lg-2">
<input type="number" id="nb_word" name="nb_word" name="quantity" min="1" placeholder="Nb of keywords" {%if dict_tracker%}{%if dict_tracker['nb_words']%}value="{{dict_tracker['nb_words']}}"{%endif%}{%endif%}>
</div>
</div>
<div class="mb-2" id="yara_rule">
<div class="" id="yara_default_rules">
<h6>Default YARA rules:</h6>
<select class="custom-select w-100 mb-3" id="yara_default_rule" name="yara_default_rule" onchange="get_default_rule_content(this);">
<option selected>Select a default rule</option>
{% for yara_types in all_yara_files %}
{% for yara_file_name in all_yara_files[yara_types] %}
<option value="{{yara_types}}/{{yara_file_name}}">{{yara_types}} - {{yara_file_name}}</option>
{% endfor %}
{% endfor %}
</select>
<pre class="border bg-light" id="default_yara_rule_content"></pre>
</div>
<hr>
<h6>Custom YARA rules:</h6>
<div class="row" id="textarea">
<textarea class="form-control mx-3" id="text_input" name="yara_custom_rule" placeholder="Enter your own YARA rule" rows="5">{%if dict_tracker%}{%if dict_tracker['type']=='yara' and dict_tracker['content']%}{{dict_tracker['content']}}{%endif%}{%endif%}</textarea>
</div>
</div>
<br>
<button class="btn btn-success mt-2">
<i class="fas fa-plus"></i> {%if dict_tracker%}Edit{%else%}Create{%endif%} Tracker
</button>
</form>
</div>
</div>
</div>
</div>
</div>
</body>
<script>
var ltags;
var chart = {};
$(document).ready(function(){
$("#page-Tracker").addClass("active");
$("#nav_manual_crawler").addClass("active");
$("#tracker_desc").hide();
$("#tracker").hide();
$("#nb_word").hide();
$("#yara_rule").hide();
sources = $('#sources').tagSuggest({
data: {{all_sources|safe}},
{%if dict_tracker%}{%if dict_tracker['sources']%}value: {{dict_tracker['sources']|safe}},{%endif%}{%endif%}
sortOrder: 'name',
maxDropHeight: 200,
name: 'sources',
emptyText: 'Sources to track (ALL IF EMPTY)',
});
$('#tracker_type').on('change', function() {
var tracker_type = this.value;
if (tracker_type=="word") {
$("#tracker_desc").text("Token to track. You need to use a regex if you want to use one of the following special characters [<>~!?@#$%^&*|()_-+={}\":;,.\'\n\r\t]/\\ ");
$("#tracker_desc").show();
$("#tracker").show();
$("#nb_word").hide();
$("#yara_rule").hide();
} else if (tracker_type=="set") {
$("#tracker_desc").text("Set of Terms to track (space separated). This tracker is used to check if an item contain one or more terms specified in a set. If an item contain NB unique terms (by default NB of unique keywords = 1), this tracker is triggered. You need to use a regex if you want to use one of the following special characters [<>~!?@#$%^&*|()_-+={}\":;,.\'\n\r\t]/\\ ");
$("#tracker_desc").show();
$("#tracker").show();
$("#nb_word").show();
$("#yara_rule").hide();
} else if (tracker_type=="regex") {
$("#tracker_desc").text("Enter a valid Python regex");
$("#tracker_desc").show();
$("#tracker").show();
$("#nb_word").hide();
$("#yara_rule").hide();
} else if (tracker_type=="yara") {
$("#tracker_desc").text("Select a default yara rule or create your own rule:");
$("#tracker_desc").show();
$("#tracker").hide();
$("#nb_word").hide();
$("#yara_rule").show();
} else if (tracker_type=="typosquatting") {
$("#tracker_desc").text("Generation of variation for domain name. Only one domain name at a time.");
$("#tracker_desc").show();
$("#tracker").show();
$("#nb_word").hide();
$("#yara_rule").hide();
}
});
{%if dict_tracker%}
$('#tracker_type').val('{{dict_tracker['type']}}').change();
{%if dict_tracker['type']=='yara' and dict_tracker['yara_file']%}
$('#yara_default_rule').val('{{dict_tracker['yara_file']}}').change();
{%endif%}
{%endif%}
});
function toggle_sidebar(){
if($('#nav_menu').is(':visible')){
$('#nav_menu').hide();
$('#side_menu').removeClass('border-right')
$('#side_menu').removeClass('col-lg-2')
$('#core_content').removeClass('col-lg-10')
}else{
$('#nav_menu').show();
$('#side_menu').addClass('border-right')
$('#side_menu').addClass('col-lg-2')
$('#core_content').addClass('col-lg-10')
}
}
function get_default_rule_content(selector){
var yara_name = selector.value
if (yara_name === "Select a default rule") {
jQuery("#default_yara_rule_content").text("")
} else {
$.getJSON("{{ url_for('hunter.get_default_yara_rule_content') }}?rule_name=" + yara_name,
function(data) {
jQuery("#default_yara_rule_content").text(data['content'])
});
}
}
</script>

124
var/www/modules/hunter/templates/showTracker.html
View File

@ -72,7 +72,7 @@
<tr>
<th>Type</th>
<th>Tracker</th>
<th>Date added</th>
<th>Created</th>
<th>Access Level</th>
<th>Created by</th>
<th>First seen</th>
@ -94,7 +94,7 @@
{% if tracker_metadata['type'] == 'typosquatting' %}
<td>
<a class="btn btn-primary" data-toggle="collapse" href="#collapseTypo" role="button" aria-expanded="false" aria-controls="collapseTypo">
{{ tracker_metadata['tracker'].split(",")[0] }}
{{ tracker_metadata['tracked'].split(",")[0] }}
</a>
<div class="collapse" id="collapseTypo">
<div class="card card-body">
@ -108,7 +108,7 @@
</div>
</td>
{% else %}
<td>{{ tracker_metadata['tracker'] }}</td>
<td>{{ tracker_metadata['tracked'] }}</td>
{% endif %}
<td>{{ tracker_metadata['date'][0:4] }}/{{ tracker_metadata['date'][4:6] }}/{{ tracker_metadata['date'][6:8] }}</td>
<td>
@ -118,19 +118,15 @@
Global
{% endif %}
</td>
<td>{{ tracker_metadata['user_id'] }}</td>
<td>{{ tracker_metadata['user'] }}</td>
<td>
{% if tracker_metadata['first_seen'] %}
{{ tracker_metadata['first_seen'][0:4] }}/
{{ tracker_metadata['first_seen'][4:6] }}/
{{ tracker_metadata['first_seen'][6:8] }}
{{ tracker_metadata['first_seen'][0:4] }}/{{ tracker_metadata['first_seen'][4:6] }}/{{ tracker_metadata['first_seen'][6:8] }}
{% endif %}
</td>
<td>
{% if tracker_metadata['last_seen'] %}
{{ tracker_metadata['last_seen'][0:4] }}/
{{ tracker_metadata['last_seen'][4:6] }}/
{{ tracker_metadata['last_seen'][6:8] }}
{{ tracker_metadata['last_seen'][0:4] }}/{{ tracker_metadata['last_seen'][4:6] }}/{{ tracker_metadata['last_seen'][6:8] }}
{% endif %}
</td>
{% if tracker_metadata['webhook'] %}
@ -159,14 +155,22 @@
<div id="sparkline"></div>
</div>
</div>
<h6>Sources:</h6>
{% if tracker_metadata['sources'] %}
{% for sources in tracker_metadata['sources'] %}
<span class="badge badge-secondary">{{ sources }}</span><br>
{% endfor %}
<h6>Filters:</h6>
{% if tracker_metadata['filters'] %}
<pre>{{ tracker_metadata['filters'] }}</pre>
{% else %}
<span class="badge badge-secondary">All Souces</span><br>
<span class="badge badge-secondary">No Filters</span><br>
{% endif %}
{# <h6>Sources:</h6>#}
{# {% if tracker_metadata['sources'] %}#}
{# {% for sources in tracker_metadata['sources'] %}#}
{# <span class="badge badge-secondary">{{ sources }}</span><br>#}
{# {% endfor %}#}
{# {% else %}#}
{# <span class="badge badge-secondary">All Sources</span><br>#}
{# {% endif %}#}
</li>
</ul>
@ -232,7 +236,7 @@
</div>
<div class="d-flex flex-row-reverse">
<a href="{{ url_for('hunter.delete_tracker') }}?uuid={{ tracker_metadata['uuid'] }}"
<a href="{{ url_for('hunters.tracker_delete') }}?uuid={{ tracker_metadata['uuid'] }}"
style="font-size: 15px">
<button class='btn btn-danger'><i class="fas fa-trash-alt"></i></button>
</a>
@ -262,8 +266,9 @@
class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
<input class="form-control" id="date-range-from-input" placeholder="yyyy-mm-dd"
name="date_from" autocomplete="off"
{% if tracker_metadata['date_from'] %}value="{{ tracker_metadata['date_from'] }}"
{% else %}value="{{ tracker_metadata['first_seen'] }}"{% endif %}>
{% if tracker_metadata['date_from'] %}value="{{ tracker_metadata['date_from'][0:4] }}-{{ tracker_metadata['date_from'][4:6] }}-{{ tracker_metadata['date_from'][6:8] }}"
{% elif tracker_metadata['first_seen'] %}value="{{ tracker_metadata['first_seen'][0:4] }}-{{ tracker_metadata['first_seen'][4:6] }}-{{ tracker_metadata['first_seen'][6:8] }}"
{% endif %}>
</div>
</div>
<div class="col-md-6">
@ -272,8 +277,9 @@
class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
<input class="form-control" id="date-range-to-input" placeholder="yyyy-mm-dd"
name="date_to" autocomplete="off"
{% if tracker_metadata['date_to'] %}value="{{ tracker_metadata['date_to'] }}"
{% else %}value="{{ tracker_metadata['last_seen'] }}"{% endif %}>
{% if tracker_metadata['date_to'] %}value="{{ tracker_metadata['date_to'][0:4] }}-{{ tracker_metadata['date_to'][4:6] }}-{{ tracker_metadata['date_to'][6:8] }}"
{% elif tracker_metadata['last_seen'] %}value="{{ tracker_metadata['last_seen'][0:4] }}-{{ tracker_metadata['last_seen'][4:6] }}-{{ tracker_metadata['last_seen'][6:8] }}"
{% endif %}>
</div>
</div>
</div>
@ -285,40 +291,52 @@
</div>
</div>
{% if tracker_metadata['items'] %}
{% if tracker_metadata['objs'] %}
<div class="mt-4">
<table class="table table-bordered table-hover" id="myTable_">
<thead class="thead-dark">
<table id="myTable_" class="table table-striped border-primary">
<thead class="bg-dark text-white">
<tr>
<th>Date</th>
<th>Item Id</th>
</tr>
</thead>
<tbody>
<th>Type</th>
<th></th>
<th>Id</th>
<th>Tags</th>
<th></th>
</tr>
</thead>
<tbody style="font-size: 15px;">
{% for object in tracker_metadata['objs'] %}
<tr class="border-color: blue;">
<td>
{% with style=object['icon']['style'], icon=object['icon']['icon'] , color=object['icon']['color'] %}
{% include 'objects/obj_svg_block.html' %}
{% endwith %}
{{ object['type']}}
</td>
<td>
{% if object['subtype'] %}
{{ object['subtype']}}
{% endif %}
</td>
<td>
<a href="{{ object['link'] }}">
{{ object['id']}}
</a>
</td>
<td>
{% for tag in object['tags'] %}
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }}</span>
{% endfor %}
</td>
<td class="text-right">
{# <a href="{{ url_for('investigations_b.unregister_investigation') }}?uuid={{ tracker_metadata['uuid']}}&type={{ object['type'] }}&subtype={{ object['subtype']}}&id={{ object['id']}}">#}
{# <button type="button" class="btn btn-danger"><i class="fas fa-trash-alt"></i></button>#}
{# </a>#}
</td>
</tr>
{% endfor %}
</tbody>
</table>
{% for item in tracker_metadata['items'] %}
<tr>
<td>
{{ item['date'][0:4] }}/{{ item['date'][4:6] }}/{{ item['date'][6:8] }}
</td>
<td>
<a class="text-secondary" target="_blank"
href="{{ url_for('objects_item.showItem') }}?id={{ item['id'] }}">
<div style="line-height:0.9;">{{ item['id'] }}</div>
</a>
<div class="mb-2">
{% for tag in item['tags'] %}
<a href="{{ url_for('tags_ui.get_obj_by_tags') }}?object_type=item&ltags={{ tag }}">
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }}</span>
</a>
{% endfor %}
</div>
</td>
</tr>
{% endfor %}
</tbody>
</table>
</div>
{% endif %}
@ -331,7 +349,7 @@
$('#div_edit_mails').hide();
$('#div_edit_tags').hide();
$('#div_edit_description').hide();
$("#page-Decoded").addClass("active");
$("#page-Tracker").addClass("active");
$('#date-range-from').dateRangePicker({
separator: ' to ',

64
var/www/modules/restApi/Flask_restApi.py
View File

@ -22,8 +22,6 @@ from lib.objects import Items
from lib import Tag
from lib import Tracker
from packages import Term # TODO REMOVE ME
from packages import Import_helper
from importer.FeederImporter import api_add_json_feeder_to_queue
@ -324,9 +322,10 @@ def get_all_tags():
res = {'tags': Tag.get_all_tags()}
return Response(json.dumps(res, indent=2, sort_keys=True), mimetype='application/json'), 200
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# # # # # # # # # # # # # # TRACKER # # # # # # # # # # # # # # # # #
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # TODO
# # # # # # # # # # # # # # TRACKER # # # # # # # # # # # # # # # # # TODO
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # TODO
'''
@restApi.route("api/v1/add/tracker", methods=['POST'])
@token_required('analyst')
def add_tracker_term():
@ -371,11 +370,24 @@ def get_tracker_metadata_api():
data = request.get_json()
tracker_uuid = data.get('tracker_uuid', None)
req_data = {'tracker_uuid': tracker_uuid}
tracker_uuid = request_dict.get('tracker_uuid', None)
if not request_dict:
return {'status': 'error', 'reason': 'Malformed JSON'}, 400
if not tracker_uuid:
return {'status': 'error', 'reason': 'Mandatory parameter(s) not provided'}, 400
if not is_valid_uuid_v4(tracker_uuid):
return {"status": "error", "reason": "Invalid Tracker UUID"}, 400
if not r_serv_tracker.exists(f'tracker:{tracker_uuid}'):
return {'status': 'error', 'reason': 'Tracker not found'}, 404
res = Tracker.get_tracker_metadata_api(req_data)
return Response(json.dumps(res[0], indent=2, sort_keys=False), mimetype='application/json'), res[1]
'''
'''
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# # # # # # # # # # # # CRYPTOCURRENCY # # # # # # # # # # # # # #
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
@ -530,27 +542,27 @@ def get_domain_status_minimal():
res[0]['domain'] = domain
return create_json_response(res[0], res[1])
@restApi.route("api/v1/get/crawled/domain/list", methods=['POST'])
@token_required('analyst')
def get_crawled_domain_list():
data = request.get_json()
res = get_mandatory_fields(data, ['date_from', 'date_to'])
if res:
return create_json_response(res[0], res[1])
date_from = data.get('date_from', None)
date_to = data.get('date_to', None)
domain_type = data.get('domain_type', None)
domain_status = 'UP'
# TODO TO MIGRATE
raise Exception('TO MIGRATE')
# res = Domain.api_get_domains_by_status_daterange(date_from, date_to, domain_type)
dict_res = res[0]
dict_res['date_from'] = date_from
dict_res['date_to'] = date_to
dict_res['domain_status'] = domain_status
dict_res['domain_type'] = domain_type
return create_json_response(dict_res, res[1])
# @restApi.route("api/v1/get/crawled/domain/list", methods=['POST'])
# @token_required('analyst')
# def get_crawled_domain_list():
# data = request.get_json()
# res = get_mandatory_fields(data, ['date_from', 'date_to'])
# if res:
# return create_json_response(res[0], res[1])
#
# date_from = data.get('date_from', None)
# date_to = data.get('date_to', None)
# domain_type = data.get('domain_type', None)
# domain_status = 'UP'
# # TODO TO MIGRATE
# raise Exception('TO MIGRATE')
# # res = Domain.api_get_domains_by_status_daterange(date_from, date_to, domain_type)
# dict_res = res[0]
# dict_res['date_from'] = date_from
# dict_res['date_to'] = date_to
# dict_res['domain_status'] = domain_status
# dict_res['domain_type'] = domain_type
# return create_json_response(dict_res, res[1])
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# # # # # # # # # # # # # IMPORT # # # # # # # # # # # # # # # # # #

14
var/www/templates/hunter/menu_sidebar.html
View File

@ -8,43 +8,43 @@
<nav class="navbar navbar-expand navbar-light bg-light flex-md-column flex-row align-items-start py-2" id="nav_menu">
<h5 class="d-flex text-muted w-100" id="nav_title_trackers">
<span>Trackers </span>
<a class="ml-auto" href="{{url_for('hunter.add_tracked_menu')}}">
<a class="ml-auto" href="{{url_for('hunters.add_tracked_menu')}}">
<i class="fas fa-plus-circle ml-auto"></i>
</a>
</h5>
<ul class="nav flex-md-column flex-row navbar-nav justify-content-between w-100"> <!--nav-pills-->
<li class="nav-item">
<a class="nav-link" href="{{url_for('hunter.tracked_menu')}}" id="nav_tracker_">
<a class="nav-link" href="{{url_for('hunters.tracked_menu')}}" id="nav_tracker_">
<i class="fas fa-ruler-combined"></i>
<span>All Trackers</span>
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="{{url_for('hunter.tracked_menu_word')}}" id="nav_tracker_word">
<a class="nav-link" href="{{url_for('hunters.tracked_menu_word')}}" id="nav_tracker_word">
<i class="fas fa-font"></i>
<span>Words</span>
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="{{url_for('hunter.tracked_menu_set')}}" id="nav_tracker_set">
<a class="nav-link" href="{{url_for('hunters.tracked_menu_set')}}" id="nav_tracker_set">
<i class="fas fa-layer-group"></i>
<span>Set</span>
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="{{url_for('hunter.tracked_menu_regex')}}" id="nav_tracker_regex">
<a class="nav-link" href="{{url_for('hunters.tracked_menu_regex')}}" id="nav_tracker_regex">
<i class="fas fa-drafting-compass"></i>
<span>Regex</span>
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="{{url_for('hunter.tracked_menu_yara')}}" id="nav_tracker_yara">
<a class="nav-link" href="{{url_for('hunters.tracked_menu_yara')}}" id="nav_tracker_yara">
<span class="bg-danger text-white font-weight-bold" style="font-size: 120%">&nbsp;{ </span>
<span>&nbsp;YARA</span>
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="{{url_for('hunter.tracked_menu_typosquatting')}}" id="nav_tracker_typosquatting">
<a class="nav-link" href="{{url_for('hunters.tracked_menu_typosquatting')}}" id="nav_tracker_typosquatting">
<i class="fa fa-clone"></i>
<span>Typo-squatting</span>
</a>

396
var/www/templates/hunter/tracker_add.html Normal file
View File

@ -0,0 +1,396 @@
<!DOCTYPE html>
<html>
<head>
<title>AIL-Framework</title>
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png')}}">
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/daterangepicker.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/tags.css') }}" rel="stylesheet">
<!-- JS -->
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/tags.js') }}"></script>
</head>
<body>
{% include 'nav_bar.html' %}
<div class="container-fluid">
<div class="row">
{% include 'hunter/menu_sidebar.html' %}
<div class="col-12 col-lg-10" id="core_content">
<div class="card my-3">
<div class="card-header bg-dark text-white">
<h5 class="card-title">{%if dict_tracker%}Edit a{%else%}Create a new{%endif%} Tracker</h5>
</div>
<div class="card-body">
<form action="{{ url_for('hunters.add_tracked_menu') }}" method='post'>
{%if dict_tracker%}
<input id="tracker_uuid" name="tracker_uuid" class="form-control" type="text" value="{{dict_tracker['uuid']}}" hidden>
{%endif%}
<div class="row">
<div class="col-12 col-xl-9">
<div class="input-group mb-2 mr-sm-2">
<div class="input-group-prepend">
<div class="input-group-text bg-secondary text-white"><i class="fas fa-at"></i></div>
</div>
<input id="mails" name="mails" class="form-control" placeholder="E-Mails Notification (optional, space separated)" type="text" {%if dict_tracker%}{%if dict_tracker['mails']%}value="{{dict_tracker['mails']}}"{%endif%}{%endif%}>
</div>
<div class="input-group mb-2 mr-sm-2">
<div class="input-group-prepend">
<div class="input-group-text bg-info text-white"><i class="fas fa-anchor"></i></div>
</div>
<input id="webhook" name="webhook" class="form-control" placeholder="Webhook URL" type="text" {%if dict_tracker%}{%if dict_tracker['webhook']%}value="{{dict_tracker['webhook']}}"{%endif%}{%endif%}>
</div>
<div class="input-group mb-2 mr-sm-2">
<div class="input-group-prepend">
<div class="input-group-text bg-info text-white"><i class="fas fa-pencil-alt"></i></div>
</div>
<input id="description" name="description" class="form-control" placeholder="Tracker Description (optional)" type="text" {%if dict_tracker%}{%if dict_tracker['description']%}value="{{dict_tracker['description']}}"{%endif%}{%endif%}>
</div>
<div class="card my-4">
<div class="card-header bg-info text-white">
<b>Objects to Track:</b>
</div>
<div class="card-body">
{# <div class="custom-control custom-switch mt-1">#}
{# <input class="custom-control-input" type="checkbox" name="cve_obj" id="cve_obj" checked="">#}
{# <label class="custom-control-label" for="cve_obj"><i class="fas fa-bug"></i>&nbsp;CVE</label>#}
{# </div>#}
{# <div class="custom-control custom-switch mt-1">#}
{# <input class="custom-control-input" type="checkbox" name="crypto_obj" id="crypto_obj" checked="">#}
{# <label class="custom-control-label" for="crypto_obj"><i class="fas fa-coins"></i>&nbsp;Cryptocurrency</label>#}
{# </div>#}
<div class="custom-control custom-switch mt-1">
<input class="custom-control-input" type="checkbox" name="decoded_obj" id="decoded_obj" checked="">
<label class="custom-control-label" for="decoded_obj"><i class="fas fa-lock-open"></i>&nbsp;Decoded</label>
</div>
{# <div class="custom-control custom-switch mt-1">#}
{# <input class="custom-control-input" type="checkbox" name="domain_obj" id="domain_obj" checked="">#}
{# <label class="custom-control-label" for="domain_obj"><i class="fas fa-spider"></i>&nbsp;Domain</label>#}
{# </div>#}
<div class="custom-control custom-switch mt-1">
<input class="custom-control-input" type="checkbox" name="item_obj" id="item_obj" checked="">
<label class="custom-control-label" for="item_obj"><i class="fas fa-file"></i>&nbsp;Item</label>
</div>
<div class="card border-dark mb-4" id="sources_item_div">
<div class="card-body">
<h5>Filter Item by sources</h5>
<div class="input-group mb-2 mr-sm-2">
<div class="input-group-prepend">
<div class="input-group-text bg-dark text-white"><i class="fas fa-folder"></i></div>
</div>
<input id="sources_item" class="form-control" type="text" name="sources_item" placeholder="Item Sources to track (ALL IF EMPTY)" autocomplete="off">
</div>
</div>
</div>
<div class="custom-control custom-switch mt-1">
<input class="custom-control-input" type="checkbox" name="pgp_obj" id="pgp_obj" checked="">
<label class="custom-control-label" for="pgp_obj"><i class="fas fa-key"></i>&nbsp;PGP</label>
</div>
<div class="card border-dark mb-4" id="sources_pgp_div">
<div class="card-body">
<h6>Filter PGP by subtype:</h6>
<div class="custom-control custom-switch mt-1">
<input class="custom-control-input" type="checkbox" name="filter_pgp_name" id="filter_pgp_name" checked="">
<label class="custom-control-label" for="filter_pgp_name">
<svg height="26" width="26">
<g class="nodes">
<circle cx="13" cy="13" r="13" fill="#44AA99"></circle>
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fas" font-size="16px">&#xf507;</text>
</g>
</svg>
name
</label>
</div>
<div class="custom-control custom-switch mt-1">
<input class="custom-control-input" type="checkbox" name="filter_pgp_mail" id="filter_pgp_mail" checked="">
<label class="custom-control-label" for="filter_pgp_mail">
<svg height="26" width="26">
<g class="nodes">
<circle cx="13" cy="13" r="13" fill="#44AA99"></circle>
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="fas" font-size="16px">&#xf1fa;</text>
</g>
</svg>
mail
</label>
</div>
</div>
</div>
{# <div class="custom-control custom-switch mt-1">#}
{# <input class="custom-control-input" type="checkbox" name="level" id="screenshot_obj" checked="">#}
{# <label class="custom-control-label" for="screenshot_obj"><i class="fas fa-image"></i>&nbsp;Screenshot</label>#}
{# </div>#}
{# <div class="custom-control custom-switch mt-1">#}
{# <input class="custom-control-input" type="checkbox" name="level" id="username_obj" checked="">#}
{# <label class="custom-control-label" for="username_obj"><i class="fas fa-user"></i>&nbsp;Username</label>#}
{# </div>#}
{# <li class="list-group-item text-left">#}
{# <div class="form-check">#}
{# <input class="form-check-input" type="checkbox" value="True" id="CveCheck" name="CveCheck" checked="">#}
{# <label class="form-check-label" for="CveCheck"><i class="fas fa-bug"></i> Cve</label>#}
{# </div>#}
{# <div class="form-check">#}
{# <input class="form-check-input" type="checkbox" value="True" id="CryptocurrencyCheck" name="CryptocurrencyCheck" checked="">#}
{# <label class="form-check-label" for="CryptocurrencyCheck"><i class="fas fa-coins"></i> Cryptocurrency</label>#}
{# </div>#}
{# <div class="form-check">#}
{# <input class="form-check-input" type="checkbox" value="True" id="DecodedCheck" name="DecodedCheck" checked="">#}
{# <label class="form-check-label" for="DecodedCheck"><i class="fas fa-lock-open"></i> Decoded</label>#}
{# </div>#}
{# <div class="form-check">#}
{# <input class="form-check-input" type="checkbox" value="True" id="ScreenshotCheck" name="ScreenshotCheck" checked="">#}
{# <label class="form-check-label" for="ScreenshotCheck"><i class="fas fa-image"></i> Screenshot</label>#}
{# </div>#}
{# <div class="form-check">#}
{# <input class="form-check-input" type="checkbox" value="True" id="PgpCheck" name="PgpCheck" checked="">#}
{# <label class="form-check-label" for="PgpCheck"><i class="fas fa-key"></i> PGP</label>#}
{# </div>#}
{# <div class="form-check">#}
{# <input class="form-check-input" type="checkbox" value="True" id="UsernameCheck" name="UsernameCheck" checked="">#}
{# <label class="form-check-label" for="UsernameCheck"><i class="fas fa-user"></i> Username</label>#}
{# </div>#}
{# <div class="form-check">#}
{# <input class="form-check-input" type="checkbox" value="True" id="DomainCheck" name="DomainCheck" checked="">#}
{# <label class="form-check-label" for="DomainCheck"><i class="fas fa-spider"></i> Domain</label>#}
{# </div>#}
{# <div class="form-check">#}
{# <input class="form-check-input" type="checkbox" value="True" id="ItemCheck" name="ItemCheck" checked="">#}
{# <label class="form-check-label" for="ItemCheck"><i class="fas fa-user"></i> Item</label>#}
{# </div>#}
{##}
{# </li>#}
</div>
</div>
<div class="card my-4">
<div class="card-header bg-secondary text-white">
<b>Tags</b>
</div>
<div class="card-body">
<div class="input-group mb-2 mr-sm-2">
<div class="input-group-prepend">
<div class="input-group-text bg-danger text-white"><i class="fas fa-tag"></i></div>
</div>
<input id="tags" name="tags" class="form-control" placeholder="Custom Tags (optional, space separated)" type="text" {%if dict_tracker%}{%if dict_tracker['tags']%}value="{{dict_tracker['tags']}}"{%endif%}{%endif%}>
</div>
{% include 'tags/block_tags_selector.html' %}
</div>
</div>
</div>
<div class="col-12 col-xl-3">
<div class="custom-control custom-switch mt-1">
<input class="custom-control-input" type="checkbox" name="level" id="id_level" {%if dict_tracker%}{%if dict_tracker['level']==1%}checked{%endif%}{%else%}checked{%endif%}>
<label class="custom-control-label" for="id_level">
<i class="fas fa-users"></i>&nbsp;Show tracker to all Users
</label>
</div>
</div>
</div>
<hr>
<h4>Tracker Type:</h4>
<select id="tracker_type" name="tracker_type" class="custom-select w-25 mb-3">
<option disabled selected value> -- Select a tracker type -- </option>
<option value="word">Word</option>
<option value="set">Set</option>
<option value="regex">Regex</option>
<option value="yara">YARA rule</option>
<option value="typosquatting">Typo-squatting</option>
</select>
<p id="tracker_desc">Terms to track (space separated)</p>
<div class="row" id="simple_input">
<div class="col-12 col-lg-10">
<input id="tracker" name="tracker" class="form-control" placeholder="Terms to track (space separated)" type="text" {%if dict_tracker%}{%if dict_tracker['tracker']!='yara'%}value="{{dict_tracker['tracker']}}"{%endif%}{%endif%}>
</div>
<div class="col-12 col-lg-2">
<input type="number" id="nb_word" name="nb_word" name="quantity" min="1" placeholder="Nb of keywords" {%if dict_tracker%}{%if dict_tracker['nb_words']%}value="{{dict_tracker['nb_words']}}"{%endif%}{%endif%}>
</div>
</div>
<div class="mb-2" id="yara_rule">
<div class="" id="yara_default_rules">
<h6>Default YARA rules:</h6>
<select class="custom-select w-100 mb-3" id="yara_default_rule" name="yara_default_rule" onchange="get_default_rule_content(this);">
<option selected>Select a default rule</option>
{% for yara_types in all_yara_files %}
{% for yara_file_name in all_yara_files[yara_types] %}
<option value="{{yara_types}}/{{yara_file_name}}">{{yara_types}} - {{yara_file_name}}</option>
{% endfor %}
{% endfor %}
</select>
<pre class="border bg-light" id="default_yara_rule_content"></pre>
</div>
<hr>
<h6>Custom YARA rules:</h6>
<div class="row" id="textarea">
<textarea class="form-control mx-3" id="text_input" name="yara_custom_rule" placeholder="Enter your own YARA rule" rows="5">{%if dict_tracker%}{%if dict_tracker['type']=='yara' and dict_tracker['content']%}{{dict_tracker['content']}}{%endif%}{%endif%}</textarea>
</div>
</div>
<br>
<button class="btn btn-success mt-2">
<i class="fas fa-plus"></i> {%if dict_tracker%}Edit{%else%}Create{%endif%} Tracker
</button>
</form>
</div>
</div>
</div>
</div>
</div>
</body>
<script>
var ltags;
var chart = {};
$(document).ready(function(){
$("#page-Tracker").addClass("active");
$("#nav_manual_crawler").addClass("active");
$("#tracker_desc").hide();
$("#tracker").hide();
$("#nb_word").hide();
$("#yara_rule").hide();
sources_item = $('#sources_item').tagSuggest({
data: {{all_sources|safe}},
{%if dict_tracker%}{%if dict_tracker['item_sources']%}value: {{dict_tracker['item_sources']|safe}},{%endif%}{%endif%}
sortOrder: 'name',
maxDropHeight: 200,
name: 'sources_item',
emptyText: 'Item Sources to track (ALL IF EMPTY)',
});
$('#tracker_type').on('change', function() {
var tracker_type = this.value;
if (tracker_type=="word") {
$("#tracker_desc").text("Token to track. You need to use a regex if you want to use one of the following special characters [<>~!?@#$%^&*|()_-+={}\":;,.\'\n\r\t]/\\ ");
$("#tracker_desc").show();
$("#tracker").show();
$("#nb_word").hide();
$("#yara_rule").hide();
} else if (tracker_type=="set") {
$("#tracker_desc").text("Set of Terms to track (space separated). This tracker is used to check if an item contain one or more terms specified in a set. If an item contain NB unique terms (by default NB of unique keywords = 1), this tracker is triggered. You need to use a regex if you want to use one of the following special characters [<>~!?@#$%^&*|()_-+={}\":;,.\'\n\r\t]/\\ ");
$("#tracker_desc").show();
$("#tracker").show();
$("#nb_word").show();
$("#yara_rule").hide();
} else if (tracker_type=="regex") {
$("#tracker_desc").text("Enter a valid Python regex");
$("#tracker_desc").show();
$("#tracker").show();
$("#nb_word").hide();
$("#yara_rule").hide();
} else if (tracker_type=="yara") {
$("#tracker_desc").text("Select a default yara rule or create your own rule:");
$("#tracker_desc").show();
$("#tracker").hide();
$("#nb_word").hide();
$("#yara_rule").show();
} else if (tracker_type=="typosquatting") {
$("#tracker_desc").text("Generation of variation for domain name. Only one domain name at a time.");
$("#tracker_desc").show();
$("#tracker").show();
$("#nb_word").hide();
$("#yara_rule").hide();
}
});
{%if dict_tracker%}
$('#tracker_type').val('{{dict_tracker['type']}}').change();
{%if dict_tracker['type']=='yara' and dict_tracker['yara_file']%}
$('#yara_default_rule').val('{{dict_tracker['yara_file']}}').change();
{%endif%}
{%endif%}
$('#item_obj').on("change", function () {
item_source_input_controller();
});
$('#pgp_obj').on("change", function () {
pgp_source_input_controller();
});
});
function toggle_sidebar(){
if($('#nav_menu').is(':visible')){
$('#nav_menu').hide();
$('#side_menu').removeClass('border-right')
$('#side_menu').removeClass('col-lg-2')
$('#core_content').removeClass('col-lg-10')
}else{
$('#nav_menu').show();
$('#side_menu').addClass('border-right')
$('#side_menu').addClass('col-lg-2')
$('#core_content').addClass('col-lg-10')
}
}
function item_source_input_controller() {
if($('#item_obj').is(':checked')){
$("#sources_item_div").show();
}else{
$("#sources_item_div").hide();
}
}
function pgp_source_input_controller() {
if($('#pgp_obj').is(':checked')){
$("#sources_pgp_div").show();
}else{
$("#sources_pgp_div").hide();
}
}
function get_default_rule_content(selector){
var yara_name = selector.value
if (yara_name === "Select a default rule") {
jQuery("#default_yara_rule_content").text("")
} else {
$.getJSON("{{ url_for('hunter.get_default_yara_rule_content') }}?rule_name=" + yara_name,
function(data) {
jQuery("#default_yara_rule_content").text(data['content'])
});
}
}
</script>

18
var/www/modules/hunter/templates/trackersManagement.html → var/www/templates/hunter/trackersManagement.html
View File

@ -71,11 +71,11 @@
<td>
<span>
<a target="_blank" href="{{ url_for('hunter.show_tracker') }}?uuid={{ dict_uuid['uuid'] }}">
{% if dict_uuid['tracker'] %}
{% if dict_uuid['tracker']|length > 256 %}
{{ dict_uuid['tracker'][0:256] }}...
{% if dict_uuid['tracked'] %}
{% if dict_uuid['tracked']|length > 256 %}
{{ dict_uuid['tracked'][0:256] }}...
{% else %}
{{ dict_uuid['tracker'] }}
{{ dict_uuid['tracked'] }}
{% endif %}
{% endif %}
</a>
@ -136,11 +136,11 @@
<td>
<span>
<a target="_blank" href="{{ url_for('hunter.show_tracker') }}?uuid={{ dict_uuid['uuid'] }}">
{% if dict_uuid['tracker'] %}
{% if dict_uuid['tracker']|length > 256 %}
{{ dict_uuid['tracker'][0:256] }}...
{% if dict_uuid['tracked'] %}
{% if dict_uuid['tracked']|length > 256 %}
{{ dict_uuid['tracked'][0:256] }}...
{% else %}
{{ dict_uuid['tracker'] }}
{{ dict_uuid['tracked'] }}
{% endif %}
{% endif %}
</a>
@ -178,7 +178,7 @@
</div>
<a class="btn btn-info my-4" href="{{ url_for('hunter.add_tracked_menu') }}">
<a class="btn btn-info my-4" href="{{ url_for('hunters.add_tracked_menu') }}">
<i class="fas fa-plus-circle ml-auto"></i>
Create New Tracker
</a>

174
var/www/templates/hunter/trackers_dashboard.html Normal file
View File

@ -0,0 +1,174 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Trackers - AIL</title>
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
<script src="{{ url_for('static', filename='js/jquery.js') }}"></script>
<script src="{{ url_for('static', filename='js/bootstrap4.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/d3.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/d3/sparklines.js') }}"></script>
</head>
<body>
{% include 'nav_bar.html' %}
<div class="container-fluid">
<div class="row">
{% include 'hunter/menu_sidebar.html' %}
<div class="col-12 col-lg-10" id="core_content">
<div class="d-flex justify-content-around my-2">
<a class="btn btn-info border-secondary" style="width: 20rem;" href="{{ url_for('hunters.tracked_menu') }}">
<div class="card-body">
<h5 class="card-title"><i class="fas fa-ruler-combined"></i>
All Trackers&nbsp;&nbsp;&nbsp;&nbsp;<span class="badge badge-light">{{ stats['all'] }}</span>
</h5>
</div>
</a>
<a class="btn btn-info border-secondary" style="width: 20rem;" href="{{ url_for('hunters.tracked_menu_word') }}">
<div class="card-body">
<h5 class="card-title"><i class="fas fa-font"></i>
Words&nbsp;&nbsp;&nbsp;&nbsp;<span class="badge badge-light">{{ stats['word'] }}</span>
</h5>
</div>
</a>
<a class="btn btn-info border-secondary" style="width: 20rem;" href="{{ url_for('hunters.tracked_menu_set') }}">
<div class="card-body">
<h5 class="card-title"><i class="fas fa-layer-group"></i>
Set&nbsp;&nbsp;&nbsp;&nbsp;<span class="badge badge-light">{{ stats['set'] }}</span>
</h5>
</div>
</a>
</div>
<div class="d-flex justify-content-around my-2">
<a class="btn btn-info border-secondary" style="width: 20rem;" href="{{ url_for('hunters.tracked_menu_regex') }}">
<div class="card-body">
<h5 class="card-title">
<i class="fas fa-drafting-compass"></i>
Regex&nbsp;&nbsp;&nbsp;&nbsp;
<span class="badge badge-light">{{ stats['regex'] }}</span>
</h5>
</div>
</a>
<a class="btn btn-info border-secondary" style="width: 20rem;" href="{{ url_for('hunters.tracked_menu_typosquatting') }}">
<div class="card-body">
<h5 class="card-title">
<i class="fa fa-clone"></i>
Typo Squatting&nbsp;&nbsp;&nbsp;&nbsp;
<span class="badge badge-light">{{ stats['typosquatting'] }}</span>
</h5>
</div>
</a>
<a class="btn btn-info border-secondary" style="width: 20rem;" href="{{ url_for('hunters.tracked_menu_yara') }}">
<div class="card-body">
<h5 class="card-title">
<span class="bg-danger text-white font-weight-bold" style="font-size: 120%">&nbsp;{ </span>
Yara&nbsp;&nbsp;&nbsp;&nbsp;
<span class="badge badge-light">{{ stats['yara'] }}</span>
</h5>
</div>
</a>
</div>
{# TODO Buttons#}
<div class="card my-3">
<div class="card-header bg-dark text-white">
<h5 class="card-title"><b>Trackers Last Matches</b></h5>
</div>
<div class="card-body">
<table id="table_global_trackers" class="table table-striped table-borderless table-condensed">
<tbody style="font-size: 15px;">
{% for meta in trackers %}
<tr>
<td>{{ meta['type'] }}</td>
<td>
<span>
<a target="_blank" href="{{ url_for('hunter.show_tracker') }}?uuid={{ meta['uuid'] }}">
{% if meta['tracked'] %}
{% if meta['tracked']|length > 256 %}
{{ meta['tracked'][0:256] }}...
{% else %}
{{ meta['tracked'] }}
{% endif %}
{% endif %}
</a>
</span>
</td>
<td>
{% for tag in meta['tags'] %}
<a href="{{ url_for('tags_ui.get_obj_by_tags') }}?object_type=item&ltags={{ tag }}">
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span>
</a>
{% endfor %}
</td>
<td>
{% if meta['first_seen'] %}
{{ meta['first_seen'][0:4] }}/{{ meta['first_seen'][4:6] }}/{{ meta['first_seen'][6:8] }}
{% endif %}
</td>
<td>
{{ meta['timestamp'] }}
</td>
{# <td id="sparklines_{{ dict_uuid['uuid'] }}" style="text-align:center;"></td>#}
</tr>
{% endfor %}
</tbody>
</table>
</div>
</div>
<a class="btn btn-primary my-4" href="{{ url_for('hunters.add_tracked_menu') }}">
<i class="fas fa-plus-circle ml-auto"></i>
Create New Tracker
</a>
</div>
</div>
</div>
<script>
$(document).ready(function () {
$("#page-Tracker").addClass("active");
$("#nav_tracker_{{tracker_type}}").addClass("active");
{# {% for dict_uuid in global_trackers %}#}
{# sparkline("sparklines_{{ dict_uuid['uuid'] }}", {{ dict_uuid['sparkline'] }}, {height: 40});#}
{# {% endfor %}#}
});
function toggle_sidebar() {
if ($('#nav_menu').is(':visible')) {
$('#nav_menu').hide();
$('#side_menu').removeClass('border-right')
$('#side_menu').removeClass('col-lg-2')
$('#core_content').removeClass('col-lg-10')
} else {
$('#nav_menu').show();
$('#side_menu').addClass('border-right')
$('#side_menu').addClass('col-lg-2')
$('#core_content').addClass('col-lg-10')
}
}
</script>
</body>
</html>

2
var/www/templates/nav_bar.html
View File

@ -19,7 +19,7 @@
<a class="nav-link" id="navbar-tags" href="{{ url_for('tags_ui.tags_search_items') }}" aria-disabled="true"><i class="fas fa-tag"></i> Tags</a>
</li>
<li class="nav-item mr-3">
<a class="nav-link" id="page-Tracker" href="{{ url_for('hunter.tracked_menu') }}" aria-disabled="true"><i class="fas fa-crosshairs"></i> Leaks Hunter</a>
<a class="nav-link" id="page-Tracker" href="{{ url_for('hunters.trackers_dashboard') }}" aria-disabled="true"><i class="fas fa-crosshairs"></i> Leaks Hunter</a>
</li>
<li class="nav-item mr-3">
<a class="nav-link" id="page-Crawler" href="{{ url_for('crawler_splash.crawlers_dashboard') }}" tabindex="-1" aria-disabled="true"><i class="fas fa-spider"></i> Crawlers</a>