diff --git a/bin/lib/ail_stats.py b/bin/lib/ail_stats.py
new file mode 100755
index 00000000..3961b8c9
--- /dev/null
+++ b/bin/lib/ail_stats.py
@@ -0,0 +1,97 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+import datetime
+import os
+import sys
+import time
+
+# from datetime import datetime
+from logging import lastResort
+
+sys.path.append(os.environ['AIL_BIN'])
+##################################
+# Import Project packages
+##################################
+from lib.ConfigLoader import ConfigLoader
+from lib.objects import ail_objects
+
+
+# Config
+config_loader = ConfigLoader()
+r_stats = config_loader.get_db_conn("Kvrocks_Stats")
+# r_cache = config_loader.get_redis_conn("Redis_Cache")
+config_loader = None
+
+
+def get_feeders():
+ return r_stats.smembers(f'feeders:name')
+
+def get_current_feeder_timestamp(timestamp):
+ return int(timestamp - (timestamp % 30))
+
+def get_next_feeder_timestamp(timestamp):
+ return int(timestamp + 30 - (timestamp % 30))
+
+def get_feeders_by_time(timestamp): # TODO
+ feeders = {}
+ for row in r_stats.zrange(f'feeders:{timestamp}', 0, -1, withscores=True):
+ feeders[row[0]] = int(row[1])
+ return feeders
+
+def get_feeders_dashboard_full():
+ timestamp = get_current_feeder_timestamp(int(time.time()))
+ print(timestamp)
+ # timestamp = 1731491970
+ f_dashboard = {}
+
+ feeders = get_feeders()
+ d_time = []
+ for i in range(timestamp - 30*20, timestamp +30, 30):
+ t_feeders = get_feeders_by_time(i)
+ for feeder in feeders:
+ if feeder not in f_dashboard:
+ f_dashboard[feeder] = []
+ if feeder in t_feeders:
+ f_dashboard[feeder].append(t_feeders[feeder])
+ else:
+ f_dashboard[feeder].append(0)
+ d_time.append(datetime.datetime.utcfromtimestamp(i).strftime('%H:%M:%S'))
+ return {'data': f_dashboard, 'dates': d_time}
+
+def get_feeders_dashboard():
+ timestamp = get_current_feeder_timestamp(int(time.time()))
+ print(timestamp)
+
+ f_dashboard = {}
+ t_feeders = get_feeders_by_time(timestamp)
+ for feeder in get_feeders():
+ if feeder in t_feeders:
+ f_dashboard[feeder] = t_feeders[feeder]
+ else:
+ f_dashboard[feeder] = 0
+
+ date = datetime.datetime.utcfromtimestamp(timestamp).strftime('%H:%M:%S')
+ return {'data': f_dashboard, 'date': date}
+
+
+def add_feeders(timestamp, feeders):
+ if feeders:
+ r = r_stats.zadd(f'feeders:{timestamp}', feeders)
+ print(r)
+ for feeder in feeders:
+ r_stats.sadd(f'feeders:name', feeder)
+ # cleanup keys
+ r_stats.sadd(f'feeders:timestamps', timestamp)
+
+def get_nb_objs_today():
+ date = datetime.date.today().strftime("%Y%m%d")
+ nb_objs = ail_objects.get_nb_objects_by_date(date)
+ return nb_objs
+
+def get_nb_objs_dashboard():
+ date = datetime.date.today().strftime("%Y%m%d")
+ return ail_objects.get_nb_objects_dashboard(date)
+
+
+
diff --git a/bin/lib/objects/BarCodes.py b/bin/lib/objects/BarCodes.py
index 80d4842d..e7bc2551 100755
--- a/bin/lib/objects/BarCodes.py
+++ b/bin/lib/objects/BarCodes.py
@@ -150,6 +150,19 @@ class Barcodes(AbstractDaterangeObjects):
def __init__(self):
super().__init__('barcode', Barcode)
+ def get_name(self):
+ return 'Barcodes'
+
+ def get_icon(self):
+ return {'fa': 'fas', 'icon': 'barcode'}
+
+ def get_link(self, flask_context=False):
+ if flask_context:
+ url = url_for('objects_barcode.objects_barcodes')
+ else:
+ url = f'{baseurl}/objects/barcodes'
+ return url
+
def sanitize_id_to_search(self, name_to_search):
return name_to_search # TODO
diff --git a/bin/lib/objects/CookiesNames.py b/bin/lib/objects/CookiesNames.py
index df9e44ad..57eee5e9 100755
--- a/bin/lib/objects/CookiesNames.py
+++ b/bin/lib/objects/CookiesNames.py
@@ -109,6 +109,19 @@ class CookiesNames(AbstractDaterangeObjects):
def __init__(self):
super().__init__('cookie-name', CookieName)
+ def get_name(self):
+ return 'Cookie-Names'
+
+ def get_icon(self):
+ return {'fa': 'fas', 'icon': 'cookie-bite'}
+
+ def get_link(self, flask_context=False):
+ if flask_context:
+ url = url_for('objects_cookie_name.objects_cookies_names')
+ else:
+ url = f'{baseurl}/objects/cookie-name'
+ return url
+
def sanitize_id_to_search(self, name_to_search):
return name_to_search # TODO
diff --git a/bin/lib/objects/Cves.py b/bin/lib/objects/Cves.py
index f8cb997b..0b020a44 100755
--- a/bin/lib/objects/Cves.py
+++ b/bin/lib/objects/Cves.py
@@ -16,7 +16,7 @@ sys.path.append(os.environ['AIL_BIN'])
# Import Project packages
##################################
from lib.ConfigLoader import ConfigLoader
-from lib.objects.abstract_daterange_object import AbstractDaterangeObject
+from lib.objects.abstract_daterange_object import AbstractDaterangeObject, AbstractDaterangeObjects
from packages import Date
config_loader = ConfigLoader()
@@ -97,6 +97,29 @@ class Cve(AbstractDaterangeObject):
except requests.exceptions.ReadTimeout:
return {'error': f'Timeout Error'}
+class Cves(AbstractDaterangeObjects):
+ """
+ Barcodes Objects
+ """
+ def __init__(self):
+ super().__init__('cve', Cve)
+
+ def get_name(self):
+ return 'Cves'
+
+ def get_icon(self):
+ return {'fa': 'fas', 'icon': 'bug'}
+
+ def get_link(self, flask_context=False):
+ if flask_context:
+ url = url_for('objects_cve.objects_cves')
+ else:
+ url = f'{baseurl}/objects/cves'
+ return url
+
+ def sanitize_id_to_search(self, name_to_search):
+ return name_to_search # TODO
+
# TODO ADD SEARCH FUNCTION
diff --git a/bin/lib/objects/DomHashs.py b/bin/lib/objects/DomHashs.py
index 996a30dd..08f0ac61 100755
--- a/bin/lib/objects/DomHashs.py
+++ b/bin/lib/objects/DomHashs.py
@@ -114,6 +114,19 @@ class DomHashs(AbstractDaterangeObjects):
def __init__(self):
super().__init__('dom-hash', DomHash)
+ def get_name(self):
+ return 'DomHashs'
+
+ def get_icon(self):
+ return {'fa': 'fas', 'icon': 'align-left'}
+
+ def get_link(self, flask_context=False):
+ if flask_context:
+ url = url_for('objects_dom_hash.objects_dom_hashs')
+ else:
+ url = f'{baseurl}/objects/dom-hashs'
+ return url
+
def sanitize_id_to_search(self, name_to_search):
return name_to_search
diff --git a/bin/lib/objects/Etags.py b/bin/lib/objects/Etags.py
index 16b90573..89f3e00b 100755
--- a/bin/lib/objects/Etags.py
+++ b/bin/lib/objects/Etags.py
@@ -109,6 +109,19 @@ class Etags(AbstractDaterangeObjects):
def __init__(self):
super().__init__('etag', Etag)
+ def get_name(self):
+ return 'Etags'
+
+ def get_icon(self):
+ return {'fa': 'fas', 'icon': 'tag'}
+
+ def get_link(self, flask_context=False):
+ if flask_context:
+ url = url_for('objects_etag.objects_etags')
+ else:
+ url = f'{baseurl}/objects/etags'
+ return url
+
def sanitize_id_to_search(self, name_to_search):
return name_to_search # TODO
diff --git a/bin/lib/objects/Favicons.py b/bin/lib/objects/Favicons.py
index 6dc7d5e5..99929b84 100755
--- a/bin/lib/objects/Favicons.py
+++ b/bin/lib/objects/Favicons.py
@@ -135,6 +135,19 @@ class Favicons(AbstractDaterangeObjects):
def __init__(self):
super().__init__('favicon', Favicon)
+ def get_name(self):
+ return 'Favicons'
+
+ def get_icon(self):
+ return {'fa': 'fas', 'icon': 'star-half'}
+
+ def get_link(self, flask_context=False):
+ if flask_context:
+ url = url_for('objects_favicon.objects_favicons')
+ else:
+ url = f'{baseurl}/objects/favicons'
+ return url
+
def sanitize_id_to_search(self, name_to_search):
return name_to_search # TODO
diff --git a/bin/lib/objects/FilesNames.py b/bin/lib/objects/FilesNames.py
index 42c9335d..6c88da04 100755
--- a/bin/lib/objects/FilesNames.py
+++ b/bin/lib/objects/FilesNames.py
@@ -84,6 +84,20 @@ class FilesNames(AbstractDaterangeObjects):
def __init__(self):
super().__init__('file-name', FileName)
+ def get_name(self):
+ return 'File-Names'
+
+ def get_icon(self):
+ return {'fa': 'far', 'icon': 'file'}
+
+ def get_link(self, flask_context=False):
+ pass
+ # if flask_context:
+ # url = url_for('objects_favicon.objects_favicons')
+ # else:
+ # url = f'{baseurl}/objects/favicons'
+ # return url
+
def sanitize_id_to_search(self, name_to_search):
return name_to_search
diff --git a/bin/lib/objects/HHHashs.py b/bin/lib/objects/HHHashs.py
index 836b3e1e..f34d1dbe 100755
--- a/bin/lib/objects/HHHashs.py
+++ b/bin/lib/objects/HHHashs.py
@@ -126,6 +126,19 @@ class HHHashs(AbstractDaterangeObjects):
def __init__(self):
super().__init__('hhhash', HHHash)
+ def get_name(self):
+ return 'HHHashs'
+
+ def get_icon(self):
+ return {'fas': 'far', 'icon': 'align-left'}
+
+ def get_link(self, flask_context=False):
+ if flask_context:
+ url = url_for('objects_hhhash.objects_hhhashs')
+ else:
+ url = f'{baseurl}/objects/hhhashs'
+ return url
+
def sanitize_id_to_search(self, name_to_search):
return name_to_search # TODO
diff --git a/bin/lib/objects/Images.py b/bin/lib/objects/Images.py
index 0e8e1d7a..3cd882dc 100755
--- a/bin/lib/objects/Images.py
+++ b/bin/lib/objects/Images.py
@@ -152,6 +152,19 @@ class Images(AbstractDaterangeObjects):
def __init__(self):
super().__init__('image', Image)
+ def get_name(self):
+ return 'Images'
+
+ def get_icon(self):
+ return {'fas': 'fas', 'icon': 'image'}
+
+ def get_link(self, flask_context=False):
+ if flask_context:
+ url = url_for('objects_image.objects_images')
+ else:
+ url = f'{baseurl}/objects/images'
+ return url
+
def sanitize_id_to_search(self, name_to_search):
return name_to_search # TODO
diff --git a/bin/lib/objects/Ocrs.py b/bin/lib/objects/Ocrs.py
index 3fb08575..08a7464b 100755
--- a/bin/lib/objects/Ocrs.py
+++ b/bin/lib/objects/Ocrs.py
@@ -323,6 +323,19 @@ class Ocrs(AbstractDaterangeObjects):
def __init__(self):
super().__init__('ocr', Ocr)
+ def get_name(self):
+ return 'Ocrs'
+
+ def get_icon(self):
+ return {'fas': 'far', 'icon': 'expand'}
+
+ def get_link(self, flask_context=False):
+ if flask_context:
+ url = url_for('objects_ocr.objects_ocrs')
+ else:
+ url = f'{baseurl}/objects/ocrs'
+ return url
+
def sanitize_id_to_search(self, name_to_search):
return name_to_search # TODO
diff --git a/bin/lib/objects/QrCodes.py b/bin/lib/objects/QrCodes.py
index 69b8f546..28c68f49 100755
--- a/bin/lib/objects/QrCodes.py
+++ b/bin/lib/objects/QrCodes.py
@@ -150,6 +150,19 @@ class Qrcodes(AbstractDaterangeObjects):
def __init__(self):
super().__init__('qrcode', Qrcode)
+ def get_name(self):
+ return 'Qrcodes'
+
+ def get_icon(self):
+ return {'fas': 'far', 'icon': 'qrcode'}
+
+ def get_link(self, flask_context=False):
+ if flask_context:
+ url = url_for('objects_qrcode.objects_qrcodes')
+ else:
+ url = f'{baseurl}/objects/qrcodes'
+ return url
+
def sanitize_id_to_search(self, name_to_search):
return name_to_search # TODO
diff --git a/bin/lib/objects/Titles.py b/bin/lib/objects/Titles.py
index 75a7ece9..c9d59f93 100755
--- a/bin/lib/objects/Titles.py
+++ b/bin/lib/objects/Titles.py
@@ -104,6 +104,19 @@ class Titles(AbstractDaterangeObjects):
def __init__(self):
super().__init__('title', Title)
+ def get_name(self):
+ return 'Titles'
+
+ def get_icon(self):
+ return {'fas': 'far', 'icon': 'heading'}
+
+ def get_link(self, flask_context=False):
+ if flask_context:
+ url = url_for('objects_title.objects_titles')
+ else:
+ url = f'{baseurl}/objects/titles'
+ return url
+
def sanitize_id_to_search(self, name_to_search):
return name_to_search
diff --git a/bin/lib/objects/abstract_daterange_object.py b/bin/lib/objects/abstract_daterange_object.py
index 519328a7..c4717ce9 100755
--- a/bin/lib/objects/abstract_daterange_object.py
+++ b/bin/lib/objects/abstract_daterange_object.py
@@ -193,6 +193,18 @@ class AbstractDaterangeObjects(ABC):
self.type = obj_type
self.obj_class = obj_class
+ @abstractmethod
+ def get_name(self):
+ pass
+
+ @abstractmethod
+ def get_icon(self):
+ pass
+
+ @abstractmethod
+ def get_link(self, flask_context=False):
+ pass
+
################################################
################################################
diff --git a/bin/lib/objects/ail_objects.py b/bin/lib/objects/ail_objects.py
index 41259c4d..af12a212 100755
--- a/bin/lib/objects/ail_objects.py
+++ b/bin/lib/objects/ail_objects.py
@@ -25,7 +25,7 @@ from lib.objects import ChatSubChannels
from lib.objects import ChatThreads
from lib.objects import CryptoCurrencies
from lib.objects import CookiesNames
-from lib.objects.Cves import Cve
+from lib.objects import Cves
from lib.objects.Decodeds import Decoded, get_all_decodeds_objects, get_nb_decodeds_objects
from lib.objects.Domains import Domain
from lib.objects import Etags
@@ -44,9 +44,36 @@ from lib.objects import Titles
from lib.objects import UsersAccount
from lib.objects import Usernames
-config_loader = ConfigLoader()
-
-config_loader = None
+# config_loader = ConfigLoader()
+#
+# config_loader = None
+# TODO INIT objs classes ????
+OBJECTS_CLASS = {
+ 'barcode': {'obj': BarCodes.Barcode, 'objs': BarCodes.Barcodes},
+ 'chat': {'obj': Chats.Chat, 'objs': None}, ## SUBTYPE #########################################
+ 'chat-subchannel': {'obj': ChatSubChannels.ChatSubChannel, 'objs': None}, ###### ######
+ 'chat-thread': {'obj': ChatThreads.ChatThread, 'objs': None}, ###### ######
+ 'cookie-name': {'obj': CookiesNames.CookieName, 'objs': CookiesNames.CookiesNames},
+ 'cve': {'obj': Cves.Cve, 'objs': Cves.Cves},
+ 'cryptocurrency': {'obj': CryptoCurrencies.CryptoCurrency, 'objs': None}, ## SUBTYPE #########################################
+ 'decoded': {'obj': Decoded, 'objs': None}, ###############################################################################################
+ 'domain': {'obj': Domain, 'objs': None}, ####################################################################################################
+ 'dom-hash': {'obj': DomHashs.DomHash, 'objs': DomHashs.DomHashs},
+ 'etag': {'obj': Etags.Etag, 'objs': Etags.Etags},
+ 'favicon': {'obj': Favicons.Favicon, 'objs': Favicons.Favicons},
+ 'file-name': {'obj': FilesNames.FileName, 'objs': FilesNames.FilesNames},
+ 'hhhash': {'obj': HHHashs.HHHash, 'objs': HHHashs.HHHashs},
+ 'item': {'obj': Item, 'objs': None}, ######
+ 'image': {'obj': Images.Image, 'objs': Images.Images},
+ 'message': {'obj': Messages.Message, 'objs': None}, ######
+ 'ocr': {'obj': Ocrs.Ocr, 'objs': Ocrs.Ocrs},
+ 'pgp': {'obj': Pgps.Pgp, 'objs': None}, ## SUBTYPE ###########################################################################
+ 'qrcode': {'obj': QrCodes.Qrcode, 'objs': QrCodes.Qrcodes},
+ 'screenshot': {'obj': Screenshots.Screenshot, 'objs': None}, ######
+ 'title': {'obj': Titles.Title, 'objs': Titles.Titles},
+ 'user-account': {'obj': UsersAccount.UserAccount, 'objs': None}, ## SUBTYPE ###########################################################################
+ 'username': {'obj': Usernames.Username, 'objs': None}, ## SUBTYPE ###########################################################################
+}
def is_valid_object_type(obj_type):
@@ -70,67 +97,29 @@ def sanitize_objs_types(objs, default=False):
l_types = get_all_objects()
return l_types
+
#### OBJECT ####
+def get_obj_class(obj_type):
+ if obj_type in OBJECTS_CLASS:
+ return OBJECTS_CLASS[obj_type]['obj']
+
+def get_objs_class(obj_type):
+ if obj_type in OBJECTS_CLASS:
+ return OBJECTS_CLASS[obj_type]['objs']
+
def get_object(obj_type, subtype, obj_id):
if subtype == 'None':
subtype = None
obj_id = str(obj_id)
+ obj_class = OBJECTS_CLASS[obj_type]['obj']
+ if not obj_class:
+ raise AILObjectUnknown(f'Unknown AIL object: {obj_type} {subtype} {obj_id}')
if not subtype:
- if obj_type == 'item':
- return Item(obj_id)
- elif obj_type == 'domain':
- return Domain(obj_id)
- elif obj_type == 'decoded':
- return Decoded(obj_id)
- elif obj_type == 'cookie-name':
- return CookiesNames.CookieName(obj_id)
- elif obj_type == 'cve':
- return Cve(obj_id)
- elif obj_type == 'etag':
- return Etags.Etag(obj_id)
- elif obj_type == 'favicon':
- return Favicons.Favicon(obj_id)
- elif obj_type == 'file-name':
- return FilesNames.FileName(obj_id)
- elif obj_type == 'dom-hash':
- return DomHashs.DomHash(obj_id)
- elif obj_type == 'hhhash':
- return HHHashs.HHHash(obj_id)
- elif obj_type == 'image':
- return Images.Image(obj_id)
- elif obj_type == 'message':
- return Messages.Message(obj_id)
- elif obj_type == 'ocr':
- return Ocrs.Ocr(obj_id)
- elif obj_type == 'barcode':
- return BarCodes.Barcode(obj_id)
- elif obj_type == 'qrcode':
- return QrCodes.Qrcode(obj_id)
- elif obj_type == 'screenshot':
- return Screenshots.Screenshot(obj_id)
- elif obj_type == 'title':
- return Titles.Title(obj_id)
- else:
- raise AILObjectUnknown(f'Unknown AIL object: {obj_type} {subtype} {obj_id}')
+ return obj_class(obj_id)
# SUBTYPES
else:
- if obj_type == 'chat':
- return Chats.Chat(obj_id, subtype)
- elif obj_type == 'chat-subchannel':
- return ChatSubChannels.ChatSubChannel(obj_id, subtype)
- elif obj_type == 'chat-thread':
- return ChatThreads.ChatThread(obj_id, subtype)
- elif obj_type == 'cryptocurrency':
- return CryptoCurrencies.CryptoCurrency(obj_id, subtype)
- elif obj_type == 'pgp':
- return Pgps.Pgp(obj_id, subtype)
- elif obj_type == 'user-account':
- return UsersAccount.UserAccount(obj_id, subtype)
- elif obj_type == 'username':
- return Usernames.Username(obj_id, subtype)
- else:
- raise AILObjectUnknown(f'Unknown AIL object: {obj_type} {subtype} {obj_id}')
+ obj_class(obj_id, subtype)
def exists_obj(obj_type, subtype, obj_id):
obj = get_object(obj_type, subtype, obj_id)
@@ -172,6 +161,32 @@ def api_get_object_global_id(global_id):
#### --API-- ####
+
+#### OBJECTS ####
+
+def get_nb_objects_by_date(date):
+ objs = {}
+ for obj_type in get_all_objects():
+ objs_class = get_objs_class(obj_type)
+ if objs_class:
+ objs_class = objs_class()
+ objs[obj_type] = objs_class.get_nb_by_date(date)
+ return objs
+
+def get_nb_objects_dashboard(date, flask_context=True):
+ objs = {}
+ for obj_type in get_all_objects():
+ objs_class = get_objs_class(obj_type)
+ if objs_class:
+ objs_class = objs_class()
+ objs[obj_type] = {}
+ objs[obj_type]['nb'] = objs_class.get_nb_by_date(date)
+ objs[obj_type]['name'] = objs_class.get_name()
+ objs[obj_type]['icon'] = objs_class.get_icon()
+ objs[obj_type]['link'] = objs_class.get_link(flask_context=flask_context)
+ return objs
+
+
#########################################################################################
#########################################################################################
#########################################################################################
@@ -241,6 +256,9 @@ def add_obj_tags(obj_type, subtype, id, tags):
# -TAGS- #
+#### OBJ META ####
+
+
def get_object_meta(obj_type, subtype, id, options=set(), flask_context=False):
obj = get_object(obj_type, subtype, id)
meta = obj.get_meta(options=options)
diff --git a/bin/modules/CodeReader.py b/bin/modules/CodeReader.py
index 090a9c40..1cd6ff7b 100755
--- a/bin/modules/CodeReader.py
+++ b/bin/modules/CodeReader.py
@@ -176,4 +176,3 @@ class CodeReader(AbstractModule):
if __name__ == '__main__':
module = CodeReader()
module.run()
-
\ No newline at end of file
diff --git a/bin/modules/Mixer.py b/bin/modules/Mixer.py
index 659874fe..222652fd 100755
--- a/bin/modules/Mixer.py
+++ b/bin/modules/Mixer.py
@@ -31,16 +31,17 @@ Note that the hash of the content is defined as the sha1(gzip64encoded).
"""
import os
import sys
-
-import hashlib
import time
+# import hashlib
+
sys.path.append(os.environ['AIL_BIN'])
##################################
# Import Project packages
##################################
from modules.abstract_module import AbstractModule
from lib.ConfigLoader import ConfigLoader
+from lib import ail_stats
class Mixer(AbstractModule):
@@ -51,12 +52,14 @@ class Mixer(AbstractModule):
config_loader = ConfigLoader()
self.r_cache = config_loader.get_redis_conn("Redis_Mixer_Cache")
- # self.r_cache_s = config_loader.get_redis_conn("Redis_Log_submit")
- self.pending_seconds = 5
+ self.pending_seconds = 1
self.refresh_time = 30
- self.last_refresh = time.time()
+ timestamp = int(time.time())
+ self.last_refresh = int(timestamp - (timestamp % 30))
+ if timestamp > self.last_refresh:
+ self.last_refresh += 30
self.operation_mode = config_loader.get_config_int("Module_Mixer", "operation_mode")
print(f'Operation mode {self.operation_mode}')
@@ -64,71 +67,25 @@ class Mixer(AbstractModule):
self.ttl_key = config_loader.get_config_int("Module_Mixer", "ttl_duplicate")
self.default_feeder_name = config_loader.get_config_str("Module_Mixer", "default_unnamed_feed_name")
- self.nb_processed_items = 0
self.feeders_processed = {}
- self.feeders_duplicate = {}
self.logger.info(f"Module: {self.module_name} Launched")
- # TODO Save stats in cache
- # def get_feeders(self):
- # return self.r_cache_s.smembers("mixer_cache:feeders")
- #
- # def get_feeder_nb_last_processed(self, feeder):
- # nb = self.r_cache_s.hget("mixer_cache:feeders:last_processed", feeder)
- # if nb:
- # return int(nb)
- # else:
- # return 0
- #
- # def get_cache_feeders_nb_last_processed(self):
- # feeders = {}
- # for feeder in self.get_feeders():
- # feeders[feeder] = self.get_feeder_nb_last_processed(feeder)
- # return feeders
-
- def clear_feeders_stat(self):
- pass
- # self.r_cache_s.delete("mixer_cache:feeders:last_processed")
-
def increase_stat_processed(self, feeder):
- self.nb_processed_items += 1
try:
self.feeders_processed[feeder] += 1
except KeyError:
self.feeders_processed[feeder] = 1
- def increase_stat_duplicate(self, feeder):
- self.nb_processed_items += 1
- try:
- self.feeders_duplicate[feeder] += 1
- except KeyError:
- self.feeders_duplicate[feeder] = 1
-
- # TODO Save stats in cache
def refresh_stats(self):
- if int(time.time() - self.last_refresh) > self.refresh_time:
- # update internal feeder
- to_print = f'Mixer; ; ; ;mixer_all All_feeders Processed {self.nb_processed_items} item(s) in {self.refresh_time}sec'
- print(to_print)
- self.redis_logger.info(to_print)
- self.nb_processed_items = 0
-
- for feeder in self.feeders_processed:
- to_print = f'Mixer; ; ; ;mixer_{feeder} {feeder} Processed {self.feeders_processed[feeder]} item(s) in {self.refresh_time}sec'
- print(to_print)
- self.redis_logger.info(to_print)
- self.feeders_processed[feeder] = 0
-
- for feeder in self.feeders_duplicate:
- to_print = f'Mixer; ; ; ;mixer_{feeder} {feeder} Duplicated {self.feeders_duplicate[feeder]} item(s) in {self.refresh_time}sec'
- print(to_print)
- self.redis_logger.info(to_print)
- self.feeders_duplicate[feeder] = 0
-
- self.last_refresh = time.time()
- self.clear_feeders_stat()
- time.sleep(0.5)
+ timestamp = int(time.time())
+ if timestamp >= self.last_refresh:
+ timestamp = timestamp - timestamp % self.refresh_time
+ print('update', timestamp)
+ print(self.feeders_processed)
+ ail_stats.add_feeders(timestamp, self.feeders_processed)
+ self.feeders_processed = {}
+ self.last_refresh = self.last_refresh + 30
def computeNone(self):
self.refresh_stats()
@@ -163,22 +120,19 @@ class Mixer(AbstractModule):
self.queue.rename_message_obj(self.obj.id, obj_id)
- relay_message = gzip64encoded
- # print(relay_message)
-
- # TODO only work for item object
- # Avoid any duplicate coming from any sources
- if self.operation_mode == 1:
- digest = hashlib.sha1(gzip64encoded.encode('utf8')).hexdigest()
- if self.r_cache.exists(digest): # Content already exists
- # STATS
- self.increase_stat_duplicate(feeder_name)
- else: # New content
- self.r_cache.sadd(digest, feeder_name)
- self.r_cache.expire(digest, self.ttl_key)
-
- self.increase_stat_processed(feeder_name)
- self.add_message_to_queue(message=relay_message)
+ # # TODO only work for item object
+ # # Avoid any duplicate coming from any sources
+ # if self.operation_mode == 1:
+ # digest = hashlib.sha1(gzip64encoded.encode('utf8')).hexdigest()
+ # if self.r_cache.exists(digest): # Content already exists
+ # # STATS
+ # self.increase_stat_duplicate(feeder_name)
+ # else: # New content
+ # self.r_cache.sadd(digest, feeder_name)
+ # self.r_cache.expire(digest, self.ttl_key)
+ #
+ # self.increase_stat_processed(feeder_name)
+ # self.add_message_to_queue(message=relay_message)
# Need To Be Fixed, Currently doesn't check the source (-> same as operation 1)
# # Keep duplicate coming from different sources
@@ -213,12 +167,10 @@ class Mixer(AbstractModule):
# self.increase_stat_duplicate(feeder_name)
# No Filtering
- else:
- self.increase_stat_processed(feeder_name)
- if self.obj.type == 'item':
- self.add_message_to_queue(obj=self.obj, message=gzip64encoded)
- else:
- self.add_message_to_queue(obj=self.obj, message=gzip64encoded)
+ # else:
+
+ self.increase_stat_processed(feeder_name)
+ self.add_message_to_queue(obj=self.obj, message=gzip64encoded)
if __name__ == "__main__":
diff --git a/requirements.txt b/requirements.txt
index 831582b0..b9da93dc 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -77,6 +77,7 @@ phonenumbers>8.12.1
# Web
flask>=2.3.3
flask-login
+flask-sock
bcrypt>3.1.6
pyotp
segno
diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py
index ec779a92..af878db7 100755
--- a/var/www/Flask_server.py
+++ b/var/www/Flask_server.py
@@ -6,13 +6,13 @@ import sys
import ssl
import json
import time
-import uuid
import random
import logging
import logging.config
from flask import Flask, render_template, jsonify, request, Request, Response, session, redirect, url_for
from flask_login import LoginManager, current_user, login_user, logout_user, login_required
+from flask_sock import Sock
import importlib
from os.path import join
@@ -28,6 +28,7 @@ from lib.ail_users import AILUser, get_session_user
from lib import Tag
from lib import ail_core
from lib import ail_logger
+from lib import ail_stats
from packages.git_status import clear_git_meta_cache
@@ -47,6 +48,7 @@ from blueprints.hunters import hunters
from blueprints.old_endpoints import old_endpoints
from blueprints.ail_2_ail_sync import ail_2_ail_sync
from blueprints.settings_b import settings_b
+from blueprints.objects_objs import objects_objs
from blueprints.objects_cve import objects_cve
from blueprints.objects_decoded import objects_decoded
from blueprints.objects_subtypes import objects_subtypes
@@ -134,6 +136,7 @@ app.register_blueprint(old_endpoints, url_prefix=baseUrl)
app.register_blueprint(ail_2_ail_sync, url_prefix=baseUrl)
app.register_blueprint(settings_b, url_prefix=baseUrl)
app.register_blueprint(objects_cve, url_prefix=baseUrl)
+app.register_blueprint(objects_objs, url_prefix=baseUrl)
app.register_blueprint(objects_decoded, url_prefix=baseUrl)
app.register_blueprint(objects_subtypes, url_prefix=baseUrl)
app.register_blueprint(objects_title, url_prefix=baseUrl)
@@ -163,7 +166,7 @@ login_manager.init_app(app)
# ========= LOGIN MANAGER ========
@login_manager.user_loader
-def load_user(session_id): # TODO USE Alternative ID
+def load_user(session_id):
# print(session)
user_id = get_session_user(session_id)
if user_id:
@@ -186,9 +189,7 @@ try:
except IOError:
pass
-# Dynamically import routes and functions from modules
-# Also, prepare header.html
-to_add_to_header_dico = {}
+# Dynamically import routes and functions from modules # # # # TODO REMOVE ME ################################################
for root, dirs, files in os.walk(os.path.join(Flask_dir, 'modules')):
sys.path.append(join(root))
@@ -204,36 +205,13 @@ for root, dirs, files in os.walk(os.path.join(Flask_dir, 'modules')):
continue
name = name.strip('.py')
importlib.import_module(name)
- elif name == 'header_{}.html'.format(module_name):
- with open(join(root, name), 'r') as f:
- to_add_to_header_dico[module_name] = f.read()
-# create header.html
-with open(os.path.join(Flask_dir, 'templates', 'header_base.html'), 'r') as f:
- complete_header = f.read()
-modified_header = complete_header
-
-# Add the header in the supplied order
-for module_name, txt in list(to_add_to_header_dico.items()):
- to_replace = ''.format(module_name)
- if to_replace in complete_header:
- modified_header = modified_header.replace(to_replace, txt)
- del to_add_to_header_dico[module_name]
-
-# Add the header for no-supplied order
-to_add_to_header = []
-for module_name, txt in to_add_to_header_dico.items():
- to_add_to_header.append(txt)
-
-modified_header = modified_header.replace('', '\n'.join(to_add_to_header))
-
-# Write the header.html file
-with open(os.path.join(Flask_dir, 'templates', 'header.html'), 'w') as f:
- f.write(modified_header)
# ========= JINJA2 FUNCTIONS ========
def list_len(s):
return len(s)
+
+
app.jinja_env.filters['list_len'] = list_len
@@ -316,6 +294,33 @@ def page_not_found(e):
return render_template('error/404.html'), 404
+# ========== WEBSOCKET ============
+
+app.config['SOCK_SERVER_OPTIONS'] = {'ping_interval': 25}
+sock = Sock(app)
+
+@login_required
+@sock.route('/ws/dashboard')
+def ws_dashboard(ws):
+ # TODO wait %30
+ next_feeders = ail_stats.get_next_feeder_timestamp(int(time.time())) + 1
+ try:
+ while True:
+ # TODO CHECK IF NEEDED
+ # if ws.closed:
+ # print('WebSocket connection closed')
+ # break
+ if int(time.time()) >= next_feeders:
+ feeders = ail_stats.get_feeders_dashboard()
+ # feeders['data']['telegram'] = 600
+ # feeders['data']['test'] = 1300
+ ws.send(json.dumps({'feeders': feeders}))
+ next_feeders = next_feeders + 30
+ time.sleep(1)
+ except Exception as e: # ConnectionClosed ?
+ print("WEBSOCKET", e)
+
+
# ========== INITIAL taxonomies ============
default_taxonomies = ["infoleak", "gdpr", "fpf", "dark-web"]
# enable default taxonomies
diff --git a/var/www/blueprints/objects_cve.py b/var/www/blueprints/objects_cve.py
index bba3532b..4cabbc3e 100644
--- a/var/www/blueprints/objects_cve.py
+++ b/var/www/blueprints/objects_cve.py
@@ -30,7 +30,7 @@ bootstrap_label = ['primary', 'success', 'danger', 'warning', 'info']
# ============ FUNCTIONS ============
-@objects_cve.route("/objects/cve", methods=['GET'])
+@objects_cve.route("/objects/cves", methods=['GET'])
@login_required
@login_read_only
def objects_cves():
diff --git a/var/www/blueprints/objects_objs.py b/var/www/blueprints/objects_objs.py
new file mode 100644
index 00000000..22f51e09
--- /dev/null
+++ b/var/www/blueprints/objects_objs.py
@@ -0,0 +1,48 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+'''
+ Blueprint Flask: crawler splash endpoints: dashboard, onion crawler ...
+'''
+
+import os
+import sys
+import json
+
+from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response, abort, send_file, stream_with_context
+from flask_login import login_required
+
+# Import Role_Manager
+from Role_Manager import login_admin, login_read_only
+
+sys.path.append(os.environ['AIL_BIN'])
+##################################
+# Import Project packages
+##################################
+from lib.objects import ail_objects
+from lib import ail_stats
+
+
+
+# ============ BLUEPRINT ============
+objects_objs = Blueprint('objects_objs', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/objects'))
+
+
+# ============ VARIABLES ============
+bootstrap_label = ['primary', 'success', 'danger', 'warning', 'info']
+
+
+# ============ FUNCTIONS ============
+@objects_objs.route("/objects", methods=['GET'])
+@login_required
+@login_read_only
+def objects():
+ nb_objects = ail_stats.get_nb_objs_dashboard()
+ print(nb_objects)
+ feeders_dashboard = ail_stats.get_feeders_dashboard_full()
+ return render_template("objs_dashboard.html", feeders_dashboard=feeders_dashboard, nb_objects=nb_objects)
+
+
+
+# ============= ROUTES ==============
+
diff --git a/var/www/blueprints/objects_title.py b/var/www/blueprints/objects_title.py
index 558103d7..04a5b7f9 100644
--- a/var/www/blueprints/objects_title.py
+++ b/var/www/blueprints/objects_title.py
@@ -34,7 +34,7 @@ def create_json_response(data, status_code):
return Response(json.dumps(data, indent=2, sort_keys=True), mimetype='application/json'), status_code
# ============= ROUTES ==============
-@objects_title.route("/objects/title", methods=['GET'])
+@objects_title.route("/objects/titles", methods=['GET'])
@login_required
@login_read_only
def objects_titles():
diff --git a/var/www/templates/objects/block_obj_button.html b/var/www/templates/objects/block_obj_button.html
new file mode 100644
index 00000000..4e9c1c30
--- /dev/null
+++ b/var/www/templates/objects/block_obj_button.html
@@ -0,0 +1,9 @@
+
+