diff --git a/bin/lib/ail_stats.py b/bin/lib/ail_stats.py new file mode 100755 index 00000000..3961b8c9 --- /dev/null +++ b/bin/lib/ail_stats.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +import datetime +import os +import sys +import time + +# from datetime import datetime +from logging import lastResort + +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from lib.ConfigLoader import ConfigLoader +from lib.objects import ail_objects + + +# Config +config_loader = ConfigLoader() +r_stats = config_loader.get_db_conn("Kvrocks_Stats") +# r_cache = config_loader.get_redis_conn("Redis_Cache") +config_loader = None + + +def get_feeders(): + return r_stats.smembers(f'feeders:name') + +def get_current_feeder_timestamp(timestamp): + return int(timestamp - (timestamp % 30)) + +def get_next_feeder_timestamp(timestamp): + return int(timestamp + 30 - (timestamp % 30)) + +def get_feeders_by_time(timestamp): # TODO + feeders = {} + for row in r_stats.zrange(f'feeders:{timestamp}', 0, -1, withscores=True): + feeders[row[0]] = int(row[1]) + return feeders + +def get_feeders_dashboard_full(): + timestamp = get_current_feeder_timestamp(int(time.time())) + print(timestamp) + # timestamp = 1731491970 + f_dashboard = {} + + feeders = get_feeders() + d_time = [] + for i in range(timestamp - 30*20, timestamp +30, 30): + t_feeders = get_feeders_by_time(i) + for feeder in feeders: + if feeder not in f_dashboard: + f_dashboard[feeder] = [] + if feeder in t_feeders: + f_dashboard[feeder].append(t_feeders[feeder]) + else: + f_dashboard[feeder].append(0) + d_time.append(datetime.datetime.utcfromtimestamp(i).strftime('%H:%M:%S')) + return {'data': f_dashboard, 'dates': d_time} + +def get_feeders_dashboard(): + timestamp = get_current_feeder_timestamp(int(time.time())) + print(timestamp) + + f_dashboard = {} + t_feeders = get_feeders_by_time(timestamp) + for feeder in get_feeders(): + if feeder in t_feeders: + f_dashboard[feeder] = t_feeders[feeder] + else: + f_dashboard[feeder] = 0 + + date = datetime.datetime.utcfromtimestamp(timestamp).strftime('%H:%M:%S') + return {'data': f_dashboard, 'date': date} + + +def add_feeders(timestamp, feeders): + if feeders: + r = r_stats.zadd(f'feeders:{timestamp}', feeders) + print(r) + for feeder in feeders: + r_stats.sadd(f'feeders:name', feeder) + # cleanup keys + r_stats.sadd(f'feeders:timestamps', timestamp) + +def get_nb_objs_today(): + date = datetime.date.today().strftime("%Y%m%d") + nb_objs = ail_objects.get_nb_objects_by_date(date) + return nb_objs + +def get_nb_objs_dashboard(): + date = datetime.date.today().strftime("%Y%m%d") + return ail_objects.get_nb_objects_dashboard(date) + + + diff --git a/bin/lib/objects/BarCodes.py b/bin/lib/objects/BarCodes.py index 80d4842d..e7bc2551 100755 --- a/bin/lib/objects/BarCodes.py +++ b/bin/lib/objects/BarCodes.py @@ -150,6 +150,19 @@ class Barcodes(AbstractDaterangeObjects): def __init__(self): super().__init__('barcode', Barcode) + def get_name(self): + return 'Barcodes' + + def get_icon(self): + return {'fa': 'fas', 'icon': 'barcode'} + + def get_link(self, flask_context=False): + if flask_context: + url = url_for('objects_barcode.objects_barcodes') + else: + url = f'{baseurl}/objects/barcodes' + return url + def sanitize_id_to_search(self, name_to_search): return name_to_search # TODO diff --git a/bin/lib/objects/CookiesNames.py b/bin/lib/objects/CookiesNames.py index df9e44ad..57eee5e9 100755 --- a/bin/lib/objects/CookiesNames.py +++ b/bin/lib/objects/CookiesNames.py @@ -109,6 +109,19 @@ class CookiesNames(AbstractDaterangeObjects): def __init__(self): super().__init__('cookie-name', CookieName) + def get_name(self): + return 'Cookie-Names' + + def get_icon(self): + return {'fa': 'fas', 'icon': 'cookie-bite'} + + def get_link(self, flask_context=False): + if flask_context: + url = url_for('objects_cookie_name.objects_cookies_names') + else: + url = f'{baseurl}/objects/cookie-name' + return url + def sanitize_id_to_search(self, name_to_search): return name_to_search # TODO diff --git a/bin/lib/objects/Cves.py b/bin/lib/objects/Cves.py index f8cb997b..0b020a44 100755 --- a/bin/lib/objects/Cves.py +++ b/bin/lib/objects/Cves.py @@ -16,7 +16,7 @@ sys.path.append(os.environ['AIL_BIN']) # Import Project packages ################################## from lib.ConfigLoader import ConfigLoader -from lib.objects.abstract_daterange_object import AbstractDaterangeObject +from lib.objects.abstract_daterange_object import AbstractDaterangeObject, AbstractDaterangeObjects from packages import Date config_loader = ConfigLoader() @@ -97,6 +97,29 @@ class Cve(AbstractDaterangeObject): except requests.exceptions.ReadTimeout: return {'error': f'Timeout Error'} +class Cves(AbstractDaterangeObjects): + """ + Barcodes Objects + """ + def __init__(self): + super().__init__('cve', Cve) + + def get_name(self): + return 'Cves' + + def get_icon(self): + return {'fa': 'fas', 'icon': 'bug'} + + def get_link(self, flask_context=False): + if flask_context: + url = url_for('objects_cve.objects_cves') + else: + url = f'{baseurl}/objects/cves' + return url + + def sanitize_id_to_search(self, name_to_search): + return name_to_search # TODO + # TODO ADD SEARCH FUNCTION diff --git a/bin/lib/objects/DomHashs.py b/bin/lib/objects/DomHashs.py index 996a30dd..08f0ac61 100755 --- a/bin/lib/objects/DomHashs.py +++ b/bin/lib/objects/DomHashs.py @@ -114,6 +114,19 @@ class DomHashs(AbstractDaterangeObjects): def __init__(self): super().__init__('dom-hash', DomHash) + def get_name(self): + return 'DomHashs' + + def get_icon(self): + return {'fa': 'fas', 'icon': 'align-left'} + + def get_link(self, flask_context=False): + if flask_context: + url = url_for('objects_dom_hash.objects_dom_hashs') + else: + url = f'{baseurl}/objects/dom-hashs' + return url + def sanitize_id_to_search(self, name_to_search): return name_to_search diff --git a/bin/lib/objects/Etags.py b/bin/lib/objects/Etags.py index 16b90573..89f3e00b 100755 --- a/bin/lib/objects/Etags.py +++ b/bin/lib/objects/Etags.py @@ -109,6 +109,19 @@ class Etags(AbstractDaterangeObjects): def __init__(self): super().__init__('etag', Etag) + def get_name(self): + return 'Etags' + + def get_icon(self): + return {'fa': 'fas', 'icon': 'tag'} + + def get_link(self, flask_context=False): + if flask_context: + url = url_for('objects_etag.objects_etags') + else: + url = f'{baseurl}/objects/etags' + return url + def sanitize_id_to_search(self, name_to_search): return name_to_search # TODO diff --git a/bin/lib/objects/Favicons.py b/bin/lib/objects/Favicons.py index 6dc7d5e5..99929b84 100755 --- a/bin/lib/objects/Favicons.py +++ b/bin/lib/objects/Favicons.py @@ -135,6 +135,19 @@ class Favicons(AbstractDaterangeObjects): def __init__(self): super().__init__('favicon', Favicon) + def get_name(self): + return 'Favicons' + + def get_icon(self): + return {'fa': 'fas', 'icon': 'star-half'} + + def get_link(self, flask_context=False): + if flask_context: + url = url_for('objects_favicon.objects_favicons') + else: + url = f'{baseurl}/objects/favicons' + return url + def sanitize_id_to_search(self, name_to_search): return name_to_search # TODO diff --git a/bin/lib/objects/FilesNames.py b/bin/lib/objects/FilesNames.py index 42c9335d..6c88da04 100755 --- a/bin/lib/objects/FilesNames.py +++ b/bin/lib/objects/FilesNames.py @@ -84,6 +84,20 @@ class FilesNames(AbstractDaterangeObjects): def __init__(self): super().__init__('file-name', FileName) + def get_name(self): + return 'File-Names' + + def get_icon(self): + return {'fa': 'far', 'icon': 'file'} + + def get_link(self, flask_context=False): + pass + # if flask_context: + # url = url_for('objects_favicon.objects_favicons') + # else: + # url = f'{baseurl}/objects/favicons' + # return url + def sanitize_id_to_search(self, name_to_search): return name_to_search diff --git a/bin/lib/objects/HHHashs.py b/bin/lib/objects/HHHashs.py index 836b3e1e..f34d1dbe 100755 --- a/bin/lib/objects/HHHashs.py +++ b/bin/lib/objects/HHHashs.py @@ -126,6 +126,19 @@ class HHHashs(AbstractDaterangeObjects): def __init__(self): super().__init__('hhhash', HHHash) + def get_name(self): + return 'HHHashs' + + def get_icon(self): + return {'fas': 'far', 'icon': 'align-left'} + + def get_link(self, flask_context=False): + if flask_context: + url = url_for('objects_hhhash.objects_hhhashs') + else: + url = f'{baseurl}/objects/hhhashs' + return url + def sanitize_id_to_search(self, name_to_search): return name_to_search # TODO diff --git a/bin/lib/objects/Images.py b/bin/lib/objects/Images.py index 0e8e1d7a..3cd882dc 100755 --- a/bin/lib/objects/Images.py +++ b/bin/lib/objects/Images.py @@ -152,6 +152,19 @@ class Images(AbstractDaterangeObjects): def __init__(self): super().__init__('image', Image) + def get_name(self): + return 'Images' + + def get_icon(self): + return {'fas': 'fas', 'icon': 'image'} + + def get_link(self, flask_context=False): + if flask_context: + url = url_for('objects_image.objects_images') + else: + url = f'{baseurl}/objects/images' + return url + def sanitize_id_to_search(self, name_to_search): return name_to_search # TODO diff --git a/bin/lib/objects/Ocrs.py b/bin/lib/objects/Ocrs.py index 3fb08575..08a7464b 100755 --- a/bin/lib/objects/Ocrs.py +++ b/bin/lib/objects/Ocrs.py @@ -323,6 +323,19 @@ class Ocrs(AbstractDaterangeObjects): def __init__(self): super().__init__('ocr', Ocr) + def get_name(self): + return 'Ocrs' + + def get_icon(self): + return {'fas': 'far', 'icon': 'expand'} + + def get_link(self, flask_context=False): + if flask_context: + url = url_for('objects_ocr.objects_ocrs') + else: + url = f'{baseurl}/objects/ocrs' + return url + def sanitize_id_to_search(self, name_to_search): return name_to_search # TODO diff --git a/bin/lib/objects/QrCodes.py b/bin/lib/objects/QrCodes.py index 69b8f546..28c68f49 100755 --- a/bin/lib/objects/QrCodes.py +++ b/bin/lib/objects/QrCodes.py @@ -150,6 +150,19 @@ class Qrcodes(AbstractDaterangeObjects): def __init__(self): super().__init__('qrcode', Qrcode) + def get_name(self): + return 'Qrcodes' + + def get_icon(self): + return {'fas': 'far', 'icon': 'qrcode'} + + def get_link(self, flask_context=False): + if flask_context: + url = url_for('objects_qrcode.objects_qrcodes') + else: + url = f'{baseurl}/objects/qrcodes' + return url + def sanitize_id_to_search(self, name_to_search): return name_to_search # TODO diff --git a/bin/lib/objects/Titles.py b/bin/lib/objects/Titles.py index 75a7ece9..c9d59f93 100755 --- a/bin/lib/objects/Titles.py +++ b/bin/lib/objects/Titles.py @@ -104,6 +104,19 @@ class Titles(AbstractDaterangeObjects): def __init__(self): super().__init__('title', Title) + def get_name(self): + return 'Titles' + + def get_icon(self): + return {'fas': 'far', 'icon': 'heading'} + + def get_link(self, flask_context=False): + if flask_context: + url = url_for('objects_title.objects_titles') + else: + url = f'{baseurl}/objects/titles' + return url + def sanitize_id_to_search(self, name_to_search): return name_to_search diff --git a/bin/lib/objects/abstract_daterange_object.py b/bin/lib/objects/abstract_daterange_object.py index 519328a7..c4717ce9 100755 --- a/bin/lib/objects/abstract_daterange_object.py +++ b/bin/lib/objects/abstract_daterange_object.py @@ -193,6 +193,18 @@ class AbstractDaterangeObjects(ABC): self.type = obj_type self.obj_class = obj_class + @abstractmethod + def get_name(self): + pass + + @abstractmethod + def get_icon(self): + pass + + @abstractmethod + def get_link(self, flask_context=False): + pass + ################################################ ################################################ diff --git a/bin/lib/objects/ail_objects.py b/bin/lib/objects/ail_objects.py index 41259c4d..af12a212 100755 --- a/bin/lib/objects/ail_objects.py +++ b/bin/lib/objects/ail_objects.py @@ -25,7 +25,7 @@ from lib.objects import ChatSubChannels from lib.objects import ChatThreads from lib.objects import CryptoCurrencies from lib.objects import CookiesNames -from lib.objects.Cves import Cve +from lib.objects import Cves from lib.objects.Decodeds import Decoded, get_all_decodeds_objects, get_nb_decodeds_objects from lib.objects.Domains import Domain from lib.objects import Etags @@ -44,9 +44,36 @@ from lib.objects import Titles from lib.objects import UsersAccount from lib.objects import Usernames -config_loader = ConfigLoader() - -config_loader = None +# config_loader = ConfigLoader() +# +# config_loader = None +# TODO INIT objs classes ???? +OBJECTS_CLASS = { + 'barcode': {'obj': BarCodes.Barcode, 'objs': BarCodes.Barcodes}, + 'chat': {'obj': Chats.Chat, 'objs': None}, ## SUBTYPE ######################################### + 'chat-subchannel': {'obj': ChatSubChannels.ChatSubChannel, 'objs': None}, ###### ###### + 'chat-thread': {'obj': ChatThreads.ChatThread, 'objs': None}, ###### ###### + 'cookie-name': {'obj': CookiesNames.CookieName, 'objs': CookiesNames.CookiesNames}, + 'cve': {'obj': Cves.Cve, 'objs': Cves.Cves}, + 'cryptocurrency': {'obj': CryptoCurrencies.CryptoCurrency, 'objs': None}, ## SUBTYPE ######################################### + 'decoded': {'obj': Decoded, 'objs': None}, ############################################################################################### + 'domain': {'obj': Domain, 'objs': None}, #################################################################################################### + 'dom-hash': {'obj': DomHashs.DomHash, 'objs': DomHashs.DomHashs}, + 'etag': {'obj': Etags.Etag, 'objs': Etags.Etags}, + 'favicon': {'obj': Favicons.Favicon, 'objs': Favicons.Favicons}, + 'file-name': {'obj': FilesNames.FileName, 'objs': FilesNames.FilesNames}, + 'hhhash': {'obj': HHHashs.HHHash, 'objs': HHHashs.HHHashs}, + 'item': {'obj': Item, 'objs': None}, ###### + 'image': {'obj': Images.Image, 'objs': Images.Images}, + 'message': {'obj': Messages.Message, 'objs': None}, ###### + 'ocr': {'obj': Ocrs.Ocr, 'objs': Ocrs.Ocrs}, + 'pgp': {'obj': Pgps.Pgp, 'objs': None}, ## SUBTYPE ########################################################################### + 'qrcode': {'obj': QrCodes.Qrcode, 'objs': QrCodes.Qrcodes}, + 'screenshot': {'obj': Screenshots.Screenshot, 'objs': None}, ###### + 'title': {'obj': Titles.Title, 'objs': Titles.Titles}, + 'user-account': {'obj': UsersAccount.UserAccount, 'objs': None}, ## SUBTYPE ########################################################################### + 'username': {'obj': Usernames.Username, 'objs': None}, ## SUBTYPE ########################################################################### +} def is_valid_object_type(obj_type): @@ -70,67 +97,29 @@ def sanitize_objs_types(objs, default=False): l_types = get_all_objects() return l_types + #### OBJECT #### +def get_obj_class(obj_type): + if obj_type in OBJECTS_CLASS: + return OBJECTS_CLASS[obj_type]['obj'] + +def get_objs_class(obj_type): + if obj_type in OBJECTS_CLASS: + return OBJECTS_CLASS[obj_type]['objs'] + def get_object(obj_type, subtype, obj_id): if subtype == 'None': subtype = None obj_id = str(obj_id) + obj_class = OBJECTS_CLASS[obj_type]['obj'] + if not obj_class: + raise AILObjectUnknown(f'Unknown AIL object: {obj_type} {subtype} {obj_id}') if not subtype: - if obj_type == 'item': - return Item(obj_id) - elif obj_type == 'domain': - return Domain(obj_id) - elif obj_type == 'decoded': - return Decoded(obj_id) - elif obj_type == 'cookie-name': - return CookiesNames.CookieName(obj_id) - elif obj_type == 'cve': - return Cve(obj_id) - elif obj_type == 'etag': - return Etags.Etag(obj_id) - elif obj_type == 'favicon': - return Favicons.Favicon(obj_id) - elif obj_type == 'file-name': - return FilesNames.FileName(obj_id) - elif obj_type == 'dom-hash': - return DomHashs.DomHash(obj_id) - elif obj_type == 'hhhash': - return HHHashs.HHHash(obj_id) - elif obj_type == 'image': - return Images.Image(obj_id) - elif obj_type == 'message': - return Messages.Message(obj_id) - elif obj_type == 'ocr': - return Ocrs.Ocr(obj_id) - elif obj_type == 'barcode': - return BarCodes.Barcode(obj_id) - elif obj_type == 'qrcode': - return QrCodes.Qrcode(obj_id) - elif obj_type == 'screenshot': - return Screenshots.Screenshot(obj_id) - elif obj_type == 'title': - return Titles.Title(obj_id) - else: - raise AILObjectUnknown(f'Unknown AIL object: {obj_type} {subtype} {obj_id}') + return obj_class(obj_id) # SUBTYPES else: - if obj_type == 'chat': - return Chats.Chat(obj_id, subtype) - elif obj_type == 'chat-subchannel': - return ChatSubChannels.ChatSubChannel(obj_id, subtype) - elif obj_type == 'chat-thread': - return ChatThreads.ChatThread(obj_id, subtype) - elif obj_type == 'cryptocurrency': - return CryptoCurrencies.CryptoCurrency(obj_id, subtype) - elif obj_type == 'pgp': - return Pgps.Pgp(obj_id, subtype) - elif obj_type == 'user-account': - return UsersAccount.UserAccount(obj_id, subtype) - elif obj_type == 'username': - return Usernames.Username(obj_id, subtype) - else: - raise AILObjectUnknown(f'Unknown AIL object: {obj_type} {subtype} {obj_id}') + obj_class(obj_id, subtype) def exists_obj(obj_type, subtype, obj_id): obj = get_object(obj_type, subtype, obj_id) @@ -172,6 +161,32 @@ def api_get_object_global_id(global_id): #### --API-- #### + +#### OBJECTS #### + +def get_nb_objects_by_date(date): + objs = {} + for obj_type in get_all_objects(): + objs_class = get_objs_class(obj_type) + if objs_class: + objs_class = objs_class() + objs[obj_type] = objs_class.get_nb_by_date(date) + return objs + +def get_nb_objects_dashboard(date, flask_context=True): + objs = {} + for obj_type in get_all_objects(): + objs_class = get_objs_class(obj_type) + if objs_class: + objs_class = objs_class() + objs[obj_type] = {} + objs[obj_type]['nb'] = objs_class.get_nb_by_date(date) + objs[obj_type]['name'] = objs_class.get_name() + objs[obj_type]['icon'] = objs_class.get_icon() + objs[obj_type]['link'] = objs_class.get_link(flask_context=flask_context) + return objs + + ######################################################################################### ######################################################################################### ######################################################################################### @@ -241,6 +256,9 @@ def add_obj_tags(obj_type, subtype, id, tags): # -TAGS- # +#### OBJ META #### + + def get_object_meta(obj_type, subtype, id, options=set(), flask_context=False): obj = get_object(obj_type, subtype, id) meta = obj.get_meta(options=options) diff --git a/bin/modules/CodeReader.py b/bin/modules/CodeReader.py index 090a9c40..1cd6ff7b 100755 --- a/bin/modules/CodeReader.py +++ b/bin/modules/CodeReader.py @@ -176,4 +176,3 @@ class CodeReader(AbstractModule): if __name__ == '__main__': module = CodeReader() module.run() - \ No newline at end of file diff --git a/bin/modules/Mixer.py b/bin/modules/Mixer.py index 659874fe..222652fd 100755 --- a/bin/modules/Mixer.py +++ b/bin/modules/Mixer.py @@ -31,16 +31,17 @@ Note that the hash of the content is defined as the sha1(gzip64encoded). """ import os import sys - -import hashlib import time +# import hashlib + sys.path.append(os.environ['AIL_BIN']) ################################## # Import Project packages ################################## from modules.abstract_module import AbstractModule from lib.ConfigLoader import ConfigLoader +from lib import ail_stats class Mixer(AbstractModule): @@ -51,12 +52,14 @@ class Mixer(AbstractModule): config_loader = ConfigLoader() self.r_cache = config_loader.get_redis_conn("Redis_Mixer_Cache") - # self.r_cache_s = config_loader.get_redis_conn("Redis_Log_submit") - self.pending_seconds = 5 + self.pending_seconds = 1 self.refresh_time = 30 - self.last_refresh = time.time() + timestamp = int(time.time()) + self.last_refresh = int(timestamp - (timestamp % 30)) + if timestamp > self.last_refresh: + self.last_refresh += 30 self.operation_mode = config_loader.get_config_int("Module_Mixer", "operation_mode") print(f'Operation mode {self.operation_mode}') @@ -64,71 +67,25 @@ class Mixer(AbstractModule): self.ttl_key = config_loader.get_config_int("Module_Mixer", "ttl_duplicate") self.default_feeder_name = config_loader.get_config_str("Module_Mixer", "default_unnamed_feed_name") - self.nb_processed_items = 0 self.feeders_processed = {} - self.feeders_duplicate = {} self.logger.info(f"Module: {self.module_name} Launched") - # TODO Save stats in cache - # def get_feeders(self): - # return self.r_cache_s.smembers("mixer_cache:feeders") - # - # def get_feeder_nb_last_processed(self, feeder): - # nb = self.r_cache_s.hget("mixer_cache:feeders:last_processed", feeder) - # if nb: - # return int(nb) - # else: - # return 0 - # - # def get_cache_feeders_nb_last_processed(self): - # feeders = {} - # for feeder in self.get_feeders(): - # feeders[feeder] = self.get_feeder_nb_last_processed(feeder) - # return feeders - - def clear_feeders_stat(self): - pass - # self.r_cache_s.delete("mixer_cache:feeders:last_processed") - def increase_stat_processed(self, feeder): - self.nb_processed_items += 1 try: self.feeders_processed[feeder] += 1 except KeyError: self.feeders_processed[feeder] = 1 - def increase_stat_duplicate(self, feeder): - self.nb_processed_items += 1 - try: - self.feeders_duplicate[feeder] += 1 - except KeyError: - self.feeders_duplicate[feeder] = 1 - - # TODO Save stats in cache def refresh_stats(self): - if int(time.time() - self.last_refresh) > self.refresh_time: - # update internal feeder - to_print = f'Mixer; ; ; ;mixer_all All_feeders Processed {self.nb_processed_items} item(s) in {self.refresh_time}sec' - print(to_print) - self.redis_logger.info(to_print) - self.nb_processed_items = 0 - - for feeder in self.feeders_processed: - to_print = f'Mixer; ; ; ;mixer_{feeder} {feeder} Processed {self.feeders_processed[feeder]} item(s) in {self.refresh_time}sec' - print(to_print) - self.redis_logger.info(to_print) - self.feeders_processed[feeder] = 0 - - for feeder in self.feeders_duplicate: - to_print = f'Mixer; ; ; ;mixer_{feeder} {feeder} Duplicated {self.feeders_duplicate[feeder]} item(s) in {self.refresh_time}sec' - print(to_print) - self.redis_logger.info(to_print) - self.feeders_duplicate[feeder] = 0 - - self.last_refresh = time.time() - self.clear_feeders_stat() - time.sleep(0.5) + timestamp = int(time.time()) + if timestamp >= self.last_refresh: + timestamp = timestamp - timestamp % self.refresh_time + print('update', timestamp) + print(self.feeders_processed) + ail_stats.add_feeders(timestamp, self.feeders_processed) + self.feeders_processed = {} + self.last_refresh = self.last_refresh + 30 def computeNone(self): self.refresh_stats() @@ -163,22 +120,19 @@ class Mixer(AbstractModule): self.queue.rename_message_obj(self.obj.id, obj_id) - relay_message = gzip64encoded - # print(relay_message) - - # TODO only work for item object - # Avoid any duplicate coming from any sources - if self.operation_mode == 1: - digest = hashlib.sha1(gzip64encoded.encode('utf8')).hexdigest() - if self.r_cache.exists(digest): # Content already exists - # STATS - self.increase_stat_duplicate(feeder_name) - else: # New content - self.r_cache.sadd(digest, feeder_name) - self.r_cache.expire(digest, self.ttl_key) - - self.increase_stat_processed(feeder_name) - self.add_message_to_queue(message=relay_message) + # # TODO only work for item object + # # Avoid any duplicate coming from any sources + # if self.operation_mode == 1: + # digest = hashlib.sha1(gzip64encoded.encode('utf8')).hexdigest() + # if self.r_cache.exists(digest): # Content already exists + # # STATS + # self.increase_stat_duplicate(feeder_name) + # else: # New content + # self.r_cache.sadd(digest, feeder_name) + # self.r_cache.expire(digest, self.ttl_key) + # + # self.increase_stat_processed(feeder_name) + # self.add_message_to_queue(message=relay_message) # Need To Be Fixed, Currently doesn't check the source (-> same as operation 1) # # Keep duplicate coming from different sources @@ -213,12 +167,10 @@ class Mixer(AbstractModule): # self.increase_stat_duplicate(feeder_name) # No Filtering - else: - self.increase_stat_processed(feeder_name) - if self.obj.type == 'item': - self.add_message_to_queue(obj=self.obj, message=gzip64encoded) - else: - self.add_message_to_queue(obj=self.obj, message=gzip64encoded) + # else: + + self.increase_stat_processed(feeder_name) + self.add_message_to_queue(obj=self.obj, message=gzip64encoded) if __name__ == "__main__": diff --git a/requirements.txt b/requirements.txt index 831582b0..b9da93dc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -77,6 +77,7 @@ phonenumbers>8.12.1 # Web flask>=2.3.3 flask-login +flask-sock bcrypt>3.1.6 pyotp segno diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py index ec779a92..af878db7 100755 --- a/var/www/Flask_server.py +++ b/var/www/Flask_server.py @@ -6,13 +6,13 @@ import sys import ssl import json import time -import uuid import random import logging import logging.config from flask import Flask, render_template, jsonify, request, Request, Response, session, redirect, url_for from flask_login import LoginManager, current_user, login_user, logout_user, login_required +from flask_sock import Sock import importlib from os.path import join @@ -28,6 +28,7 @@ from lib.ail_users import AILUser, get_session_user from lib import Tag from lib import ail_core from lib import ail_logger +from lib import ail_stats from packages.git_status import clear_git_meta_cache @@ -47,6 +48,7 @@ from blueprints.hunters import hunters from blueprints.old_endpoints import old_endpoints from blueprints.ail_2_ail_sync import ail_2_ail_sync from blueprints.settings_b import settings_b +from blueprints.objects_objs import objects_objs from blueprints.objects_cve import objects_cve from blueprints.objects_decoded import objects_decoded from blueprints.objects_subtypes import objects_subtypes @@ -134,6 +136,7 @@ app.register_blueprint(old_endpoints, url_prefix=baseUrl) app.register_blueprint(ail_2_ail_sync, url_prefix=baseUrl) app.register_blueprint(settings_b, url_prefix=baseUrl) app.register_blueprint(objects_cve, url_prefix=baseUrl) +app.register_blueprint(objects_objs, url_prefix=baseUrl) app.register_blueprint(objects_decoded, url_prefix=baseUrl) app.register_blueprint(objects_subtypes, url_prefix=baseUrl) app.register_blueprint(objects_title, url_prefix=baseUrl) @@ -163,7 +166,7 @@ login_manager.init_app(app) # ========= LOGIN MANAGER ======== @login_manager.user_loader -def load_user(session_id): # TODO USE Alternative ID +def load_user(session_id): # print(session) user_id = get_session_user(session_id) if user_id: @@ -186,9 +189,7 @@ try: except IOError: pass -# Dynamically import routes and functions from modules -# Also, prepare header.html -to_add_to_header_dico = {} +# Dynamically import routes and functions from modules # # # # TODO REMOVE ME ################################################ for root, dirs, files in os.walk(os.path.join(Flask_dir, 'modules')): sys.path.append(join(root)) @@ -204,36 +205,13 @@ for root, dirs, files in os.walk(os.path.join(Flask_dir, 'modules')): continue name = name.strip('.py') importlib.import_module(name) - elif name == 'header_{}.html'.format(module_name): - with open(join(root, name), 'r') as f: - to_add_to_header_dico[module_name] = f.read() -# create header.html -with open(os.path.join(Flask_dir, 'templates', 'header_base.html'), 'r') as f: - complete_header = f.read() -modified_header = complete_header - -# Add the header in the supplied order -for module_name, txt in list(to_add_to_header_dico.items()): - to_replace = ''.format(module_name) - if to_replace in complete_header: - modified_header = modified_header.replace(to_replace, txt) - del to_add_to_header_dico[module_name] - -# Add the header for no-supplied order -to_add_to_header = [] -for module_name, txt in to_add_to_header_dico.items(): - to_add_to_header.append(txt) - -modified_header = modified_header.replace('', '\n'.join(to_add_to_header)) - -# Write the header.html file -with open(os.path.join(Flask_dir, 'templates', 'header.html'), 'w') as f: - f.write(modified_header) # ========= JINJA2 FUNCTIONS ======== def list_len(s): return len(s) + + app.jinja_env.filters['list_len'] = list_len @@ -316,6 +294,33 @@ def page_not_found(e): return render_template('error/404.html'), 404 +# ========== WEBSOCKET ============ + +app.config['SOCK_SERVER_OPTIONS'] = {'ping_interval': 25} +sock = Sock(app) + +@login_required +@sock.route('/ws/dashboard') +def ws_dashboard(ws): + # TODO wait %30 + next_feeders = ail_stats.get_next_feeder_timestamp(int(time.time())) + 1 + try: + while True: + # TODO CHECK IF NEEDED + # if ws.closed: + # print('WebSocket connection closed') + # break + if int(time.time()) >= next_feeders: + feeders = ail_stats.get_feeders_dashboard() + # feeders['data']['telegram'] = 600 + # feeders['data']['test'] = 1300 + ws.send(json.dumps({'feeders': feeders})) + next_feeders = next_feeders + 30 + time.sleep(1) + except Exception as e: # ConnectionClosed ? + print("WEBSOCKET", e) + + # ========== INITIAL taxonomies ============ default_taxonomies = ["infoleak", "gdpr", "fpf", "dark-web"] # enable default taxonomies diff --git a/var/www/blueprints/objects_cve.py b/var/www/blueprints/objects_cve.py index bba3532b..4cabbc3e 100644 --- a/var/www/blueprints/objects_cve.py +++ b/var/www/blueprints/objects_cve.py @@ -30,7 +30,7 @@ bootstrap_label = ['primary', 'success', 'danger', 'warning', 'info'] # ============ FUNCTIONS ============ -@objects_cve.route("/objects/cve", methods=['GET']) +@objects_cve.route("/objects/cves", methods=['GET']) @login_required @login_read_only def objects_cves(): diff --git a/var/www/blueprints/objects_objs.py b/var/www/blueprints/objects_objs.py new file mode 100644 index 00000000..22f51e09 --- /dev/null +++ b/var/www/blueprints/objects_objs.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +''' + Blueprint Flask: crawler splash endpoints: dashboard, onion crawler ... +''' + +import os +import sys +import json + +from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response, abort, send_file, stream_with_context +from flask_login import login_required + +# Import Role_Manager +from Role_Manager import login_admin, login_read_only + +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from lib.objects import ail_objects +from lib import ail_stats + + + +# ============ BLUEPRINT ============ +objects_objs = Blueprint('objects_objs', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/objects')) + + +# ============ VARIABLES ============ +bootstrap_label = ['primary', 'success', 'danger', 'warning', 'info'] + + +# ============ FUNCTIONS ============ +@objects_objs.route("/objects", methods=['GET']) +@login_required +@login_read_only +def objects(): + nb_objects = ail_stats.get_nb_objs_dashboard() + print(nb_objects) + feeders_dashboard = ail_stats.get_feeders_dashboard_full() + return render_template("objs_dashboard.html", feeders_dashboard=feeders_dashboard, nb_objects=nb_objects) + + + +# ============= ROUTES ============== + diff --git a/var/www/blueprints/objects_title.py b/var/www/blueprints/objects_title.py index 558103d7..04a5b7f9 100644 --- a/var/www/blueprints/objects_title.py +++ b/var/www/blueprints/objects_title.py @@ -34,7 +34,7 @@ def create_json_response(data, status_code): return Response(json.dumps(data, indent=2, sort_keys=True), mimetype='application/json'), status_code # ============= ROUTES ============== -@objects_title.route("/objects/title", methods=['GET']) +@objects_title.route("/objects/titles", methods=['GET']) @login_required @login_read_only def objects_titles(): diff --git a/var/www/templates/objects/block_obj_button.html b/var/www/templates/objects/block_obj_button.html new file mode 100644 index 00000000..4e9c1c30 --- /dev/null +++ b/var/www/templates/objects/block_obj_button.html @@ -0,0 +1,9 @@ + +
+ + + {{ nb }} + +
+
{{ name }}
+
\ No newline at end of file diff --git a/var/www/templates/objects/objs_dashboard.html b/var/www/templates/objects/objs_dashboard.html new file mode 100644 index 00000000..6e89b3b4 --- /dev/null +++ b/var/www/templates/objects/objs_dashboard.html @@ -0,0 +1,484 @@ + + + + + Objects - AIL + + + + + + + + + + + + +{# #} + +{# #} +{# #} +{# #} + + + + + + + + + {% include 'nav_bar.html' %} + +
+ + +
+ +
+
+
+ +
+ +
+ + {% for obj_type in nb_objects %} +
+ {% with name=nb_objects[obj_type]['name'], icon=nb_objects[obj_type]['icon']['icon'], nb=nb_objects[obj_type]['nb'], url=nb_objects[obj_type]['link'] %} + {% include 'objects/block_obj_button.html' %} + {% endwith %} +
+ {% endfor %} + + + +
+ +
+ +
+ +
+
+ + + + + +
+
+ + + 5896 + +
+
CVEs
+
+ +
+
+ + + 5896 + +
+
CVEs
+
+ +
+
+ + + 5896 + +
+
CVEs
+
+ +
+
+ + + 5896 + +
+
CVEs
+
+ +
+
+ + + 5896 + +
+
CVEs
+
+ +
+
+ + + 5896 + +
+
CVEs
+
+ +
+
+ + + 5896 + +
+
CVEs
+
+ +
+
+ + + 5896 + +
+
CVEs
+
+ +
+
+ + + 5896 + +
+
CVEs
+
+ +
+
+ + + 5896 + +
+
CVEs
+
+ +
+
+ + + 5896 + +
+
CVEs
+
+ +
+
+ + + 5896 + +
+
CVEs
+
+ +
+
+ + + 5896 + +
+
CVEs
+
+ +
+
+ + + 5896 + +
+
CVEs
+
+ +
+
+ + + 5896 + +
+
CVEs
+
+ +
+ + + +
+ + + + + + + 1,419 + + + + 1,419 + + + + 1,419 + + + + 1,419 + + + + 1,419 + + + + 1,419 + + + + 1,419 + + + + 1,419 + + + + 1,419 + + + + 1,419 + + + + 1,419 + + + + 1,419 + + + + 1,419 + + + + 1,419 + + + + 1,419 + + + + 1,419 + + + + 1,419 + + + + +
+
+
+ + + + + + + diff --git a/var/www/update_thirdparty.sh b/var/www/update_thirdparty.sh index 39f49b7a..5db61adb 100755 --- a/var/www/update_thirdparty.sh +++ b/var/www/update_thirdparty.sh @@ -5,28 +5,43 @@ # submodules git submodule update -wget -q http://dygraphs.com/dygraph-combined.js -O ./static/js/dygraph-combined.js - -BOOTSTRAP_VERSION='4.2.1' -FONT_AWESOME_VERSION='6.6.0' - -D3_JS_VERSION='5.16.0' -wget https://d3js.org/d3.v7.min.js -O ./static/js/d3.v7.min.js - rm -rf temp mkdir temp -wget https://github.com/twbs/bootstrap/releases/download/v${BOOTSTRAP_VERSION}/bootstrap-${BOOTSTRAP_VERSION}-dist.zip -O temp/bootstrap${BOOTSTRAP_VERSION}.zip -wget https://github.com/FortAwesome/Font-Awesome/archive/${FONT_AWESOME_VERSION}.zip -O temp/FONT_AWESOME_${FONT_AWESOME_VERSION}.zip +#### D3JS #### +#### TODO UPDATE ALL D3 JS to V7 +wget https://d3js.org/d3.v7.min.js -O ./static/js/d3.v7.min.js + +# OLD +D3_JS_VERSION='5.16.0' wget https://github.com/d3/d3/releases/download/v${D3_JS_VERSION}/d3.zip -O temp/d3_${D3_JS_VERSION}.zip +unzip -qq temp/d3_${D3_JS_VERSION}.zip -d temp/ + +mv temp/d3.min.js ./static/js/ +#### #### + +#### FONT_AWESOME #### +FONT_AWESOME_VERSION='6.6.0' +wget https://github.com/FortAwesome/Font-Awesome/archive/${FONT_AWESOME_VERSION}.zip -O temp/FONT_AWESOME_${FONT_AWESOME_VERSION}.zip +unzip temp/FONT_AWESOME_${FONT_AWESOME_VERSION}.zip -d temp/ + +rm -rf ./static/webfonts/ +mv temp/Font-Awesome-${FONT_AWESOME_VERSION}/css/all.min.css ./static/css/font-awesome.min.css +mv temp/Font-Awesome-${FONT_AWESOME_VERSION}/webfonts ./static/webfonts + +rm -rf ./static/fonts/ ./static/font-awesome/ +mv temp/font-awesome/ ./static +#### #### + +BOOTSTRAP_VERSION='4.2.1' + +wget https://github.com/twbs/bootstrap/releases/download/v${BOOTSTRAP_VERSION}/bootstrap-${BOOTSTRAP_VERSION}-dist.zip -O temp/bootstrap${BOOTSTRAP_VERSION}.zip # dateRangePicker wget https://github.com/moment/moment/archive/2.24.0.zip -O temp/moment.zip wget https://github.com/longbill/jquery-date-range-picker/archive/v0.20.0.zip -O temp/daterangepicker.zip unzip -qq temp/bootstrap${BOOTSTRAP_VERSION}.zip -d temp/ -unzip temp/FONT_AWESOME_${FONT_AWESOME_VERSION}.zip -d temp/ -unzip -qq temp/d3_${D3_JS_VERSION}.zip -d temp/ unzip -qq temp/moment.zip -d temp/ unzip -qq temp/daterangepicker.zip -d temp/ @@ -36,19 +51,10 @@ mv temp/bootstrap-${BOOTSTRAP_VERSION}-dist/js/bootstrap.min.js.map ./static/js/ mv temp/bootstrap-${BOOTSTRAP_VERSION}-dist/css/bootstrap.min.css ./static/css/bootstrap4.min.css mv temp/bootstrap-${BOOTSTRAP_VERSION}-dist/css/bootstrap.min.css.map ./static/css/bootstrap4.min.css.map -rm -rf ./static/webfonts/ -mv temp/Font-Awesome-${FONT_AWESOME_VERSION}/css/all.min.css ./static/css/font-awesome.min.css -mv temp/Font-Awesome-${FONT_AWESOME_VERSION}/webfonts ./static/webfonts - -rm -rf ./static/js/plugins - -rm -rf ./static/fonts/ ./static/font-awesome/ -mv temp/font-awesome/ ./static/ rm -rf ./static/css/plugins/ mv temp/jquery-date-range-picker-0.20.0/dist/daterangepicker.min.css ./static/css/ -mv temp/d3.min.js ./static/js/ mv temp/moment-2.24.0/min/moment.min.js ./static/js/ mv temp/jquery-date-range-picker-0.20.0/dist/jquery.daterangepicker.min.js ./static/js/ @@ -57,8 +63,6 @@ wget http://code.jquery.com/jquery-${JQVERSION}.js -O ./static/js/jquery.js #Ressources for dataTable wget https://cdn.datatables.net/1.10.20/js/jquery.dataTables.min.js -O ./static/js/jquery.dataTables.min.js -wget https://cdn.datatables.net/plug-ins/1.10.20/integration/bootstrap/3/dataTables.bootstrap.css -O ./static/css/dataTables.bootstrap.css -wget https://cdn.datatables.net/plug-ins/1.10.20/integration/bootstrap/3/dataTables.bootstrap.js -O ./static/js/dataTables.bootstrap.js wget https://cdn.datatables.net/1.10.20/css/dataTables.bootstrap4.min.css -O ./static/css/dataTables.bootstrap.min.css wget https://cdn.datatables.net/1.10.20/js/dataTables.bootstrap4.min.js -O ./static/js/dataTables.bootstrap.min.js @@ -71,20 +75,10 @@ mv temp/floating-ui-${POPPER_VERSION}/dist/umd/popper.min.js ./static/js/ mv temp/floating-ui-${POPPER_VERSION}/dist/umd/popper.min.js.map ./static/js/ #Ressource for graph -# DASHBOARD # TODO REFACTOR DASHBOARD GRAPHS -wget https://raw.githubusercontent.com/flot/flot/958e5fd43c6dff4bab3e1fd5cb6109df5c1e8003/jquery.flot.js -O ./static/js/jquery.flot.js -wget https://raw.githubusercontent.com/flot/flot/958e5fd43c6dff4bab3e1fd5cb6109df5c1e8003/jquery.flot.pie.js -O ./static/js/jquery.flot.pie.js -wget https://raw.githubusercontent.com/flot/flot/958e5fd43c6dff4bab3e1fd5cb6109df5c1e8003/jquery.flot.time.js -O ./static/js/jquery.flot.time.js -wget https://raw.githubusercontent.com/flot/flot/958e5fd43c6dff4bab3e1fd5cb6109df5c1e8003/jquery.flot.stack.js -O ./static/js/jquery.flot.stack.js +# DASHBOARD # TODO Extract from github +wget https://cdn.jsdelivr.net/npm/echarts@5.5.1/dist/echarts.min.js - O ./static/js/echarts.min.js -#Ressources for sparkline and canvasJS and slider -#wget http://omnipotent.net/jquery.sparkline/2.1.2/jquery.sparkline.min.js -O ./static/js/jquery.sparkline.min.js -#wget https://canvasjs.com/assets/script/canvasjs.min.js -O ./static/js/jquery.canvasjs.min.js -wget https://jqueryui.com/resources/download/jquery-ui-1.12.1.zip -O temp/jquery-ui.zip -unzip -qq temp/jquery-ui.zip -d temp/ -mv temp/jquery-ui-1.12.1/jquery-ui.min.js ./static/js/jquery-ui.min.js -mv temp/jquery-ui-1.12.1/jquery-ui.min.css ./static/css/jquery-ui.min.css # INSTALL YARA YARA_VERSION="4.3.0"