diff --git a/bin/lib/objects/ChatSubChannels.py b/bin/lib/objects/ChatSubChannels.py index 331c29ea..54952471 100755 --- a/bin/lib/objects/ChatSubChannels.py +++ b/bin/lib/objects/ChatSubChannels.py @@ -158,7 +158,23 @@ class ChatSubChannel(AbstractChatObject): class ChatSubChannels(AbstractChatObjects): def __init__(self): - super().__init__('chat-subchannel') + super().__init__('chat-subchannel', ChatSubChannel) + + def get_name(self): + return 'Chat-SubChannels' + + def get_icon(self): + return {'fas': 'far', 'icon': 'comments'} + + def get_link(self, flask_context=False): + if flask_context: + url = url_for('chats_explorer.chats_explorer_protocols') + else: + url = f'{baseurl}/chats/explorer/protocols' + return url + + def sanitize_id_to_search(self, subtypes, name_to_search): + return name_to_search # if __name__ == '__main__': # chat = Chat('test', 'telegram') diff --git a/bin/lib/objects/ChatThreads.py b/bin/lib/objects/ChatThreads.py index a2559cc4..bb7eddd7 100755 --- a/bin/lib/objects/ChatThreads.py +++ b/bin/lib/objects/ChatThreads.py @@ -112,7 +112,23 @@ def create(thread_id, chat_instance, chat_id, subchannel_id, message_id, contain class ChatThreads(AbstractChatObjects): def __init__(self): - super().__init__('chat-thread') + super().__init__('chat-thread', ChatThread) + + def get_name(self): + return 'Chat-Threads' + + def get_icon(self): + return {'fas': 'fas', 'icon': 'grip-lines'} + + def get_link(self, flask_context=False): + if flask_context: + url = url_for('chats_explorer.chats_explorer_protocols') + else: + url = f'{baseurl}/chats/explorer/protocols' + return url + + def sanitize_id_to_search(self, subtypes, name_to_search): + return name_to_search # if __name__ == '__main__': # chat = Chat('test', 'telegram') diff --git a/bin/lib/objects/Chats.py b/bin/lib/objects/Chats.py index 8b127524..3a9407ba 100755 --- a/bin/lib/objects/Chats.py +++ b/bin/lib/objects/Chats.py @@ -60,6 +60,14 @@ class Chat(AbstractChatObject): username = username.split(':', 2)[2] return f'https://t.me/{username}' + def get_chat_instance(self): + if self.subtype == '00098785-7e70-5d12-a120-c5cdc1252b2b': + return 'telegram' + elif self.subtype == 'd2426e3f-22f3-5a57-9a98-d2ae9794e683': + return 'discord' + else: + return self.subtype + def get_svg_icon(self): # TODO # if self.subtype == 'telegram': # style = 'fab' @@ -164,6 +172,21 @@ class Chat(AbstractChatObject): def update_username_timeline(self, username_global_id, timestamp): self._get_timeline_username().add_timestamp(timestamp, username_global_id) + def get_label(self): + username = self.get_username() + if username: + username = username.split(':', 2)[2] + name = self.get_name() + if username and name: + label = f'{username} - {name}' + elif username: + label = username + elif name: + label = name + else: + label = '' + return label + #### ChatSubChannels #### @@ -203,7 +226,23 @@ class Chat(AbstractChatObject): class Chats(AbstractChatObjects): def __init__(self): - super().__init__('chat') + super().__init__('chat', Chat) + + def get_name(self): + return 'Chats' + + def get_icon(self): + return {'fas': 'fas', 'icon': 'comment'} + + def get_link(self, flask_context=False): + if flask_context: + url = url_for('chats_explorer.chats_explorer_protocols') + else: + url = f'{baseurl}/chats/explorer/protocols' + return url + + def sanitize_id_to_search(self, subtypes, name_to_search): + return name_to_search def get_ids_with_messages_by_subtype(self, subtype): return r_object.smembers(f'{self.type}_w_mess:{subtype}') diff --git a/bin/lib/objects/CryptoCurrencies.py b/bin/lib/objects/CryptoCurrencies.py index e633cf7b..0e048927 100755 --- a/bin/lib/objects/CryptoCurrencies.py +++ b/bin/lib/objects/CryptoCurrencies.py @@ -15,7 +15,7 @@ sys.path.append(os.environ['AIL_BIN']) # Import Project packages ################################## from lib.ConfigLoader import ConfigLoader -from lib.objects.abstract_subtype_object import AbstractSubtypeObject, get_all_id +from lib.objects.abstract_subtype_object import AbstractSubtypeObject, AbstractSubtypeObjects, get_all_id config_loader = ConfigLoader() baseurl = config_loader.get_config_str("Notifications", "ail_domain") @@ -166,6 +166,31 @@ class CryptoCurrency(AbstractSubtypeObject): meta['tags'] = self.get_tags(r_list=True) return meta + +class CryptoCurrencies(AbstractSubtypeObjects): + """ + Usernames Objects + """ + def __init__(self): + super().__init__('cryptocurrency', CryptoCurrency) + + def get_name(self): + return 'Cryptocurrencies' + + def get_icon(self): + return {'fas': 'fas', 'icon': 'coins'} + + def get_link(self, flask_context=False): + if flask_context: + url = url_for('objects_subtypes.objects_dashboard_cryptocurrency') + else: + url = f'{baseurl}/objects/cryptocurrencies' + return url + + def sanitize_id_to_search(self, subtypes, name_to_search): + return name_to_search + + ############################################################################ ############################################################################ diff --git a/bin/lib/objects/Decodeds.py b/bin/lib/objects/Decodeds.py index 15cea0c3..ce79cbec 100755 --- a/bin/lib/objects/Decodeds.py +++ b/bin/lib/objects/Decodeds.py @@ -18,7 +18,7 @@ sys.path.append(os.environ['AIL_BIN']) # Import Project packages ################################## from lib.ConfigLoader import ConfigLoader -from lib.objects.abstract_daterange_object import AbstractDaterangeObject +from lib.objects.abstract_daterange_object import AbstractDaterangeObject, AbstractDaterangeObjects from packages import Date sys.path.append('../../configs/keys') @@ -321,6 +321,32 @@ class Decoded(AbstractDaterangeObject): ############################################################################ +class Decodeds(AbstractDaterangeObjects): + """ + Barcodes Objects + """ + def __init__(self): + super().__init__('decoded', Decoded) + + def get_name(self): + return 'Decodeds' + + def get_icon(self): + return {'fa': 'fas', 'icon': 'lock-open'} + + def get_link(self, flask_context=False): + if flask_context: + url = url_for('objects_decoded.decodeds_dashboard') + else: + url = f'{baseurl}/objects/decodeds' + return url + + def sanitize_id_to_search(self, name_to_search): + return name_to_search + + +############################################################################ + def is_vt_enabled(): return VT_ENABLED diff --git a/bin/lib/objects/DomHashs.py b/bin/lib/objects/DomHashs.py index 08f0ac61..e03d7bfe 100755 --- a/bin/lib/objects/DomHashs.py +++ b/bin/lib/objects/DomHashs.py @@ -57,7 +57,7 @@ class DomHash(AbstractDaterangeObject): return url def get_svg_icon(self): - return {'style': 'fas', 'icon': '\uf714', 'color': 'grey', 'radius': 5} + return {'style': 'fas', 'icon': '\ue58a', 'color': 'grey', 'radius': 5} def get_misp_object(self): obj_attrs = [] @@ -118,7 +118,7 @@ class DomHashs(AbstractDaterangeObjects): return 'DomHashs' def get_icon(self): - return {'fa': 'fas', 'icon': 'align-left'} + return {'fa': 'fa-solid', 'icon': 'trowel-bricks'} def get_link(self, flask_context=False): if flask_context: diff --git a/bin/lib/objects/Domains.py b/bin/lib/objects/Domains.py index b20208eb..1c84c7c9 100755 --- a/bin/lib/objects/Domains.py +++ b/bin/lib/objects/Domains.py @@ -727,6 +727,32 @@ def cluster_onion_domain_vanity(len_vanity=4): res = dict(sorted(occurrences.items(), key=lambda item: item[1], reverse=True)) print(json.dumps(res)) +class Domains: + def __init__(self): + self.type = 'message' + self.obj_class = Domain + + def get_name(self): + return 'Domains' + + def get_icon(self): + return {'fas': 'fas', 'icon': 'spider'} + + def get_link(self, flask_context=False): + if flask_context: + url = url_for('crawler_splash.crawlers_dashboard') + else: + url = f'{baseurl}/crawlers/dashboard' + return url + + # def get_by_date(self, date): + # pass + + def get_nb_by_date(self, date): + nb = 0 + for domain_type in get_all_domains_types(): + nb += r_crawler.scard(f'{domain_type}_up:{date}') + return nb if __name__ == '__main__': _rebuild_vanity_clusters() diff --git a/bin/lib/objects/Pgps.py b/bin/lib/objects/Pgps.py index 81485b06..963ddc3e 100755 --- a/bin/lib/objects/Pgps.py +++ b/bin/lib/objects/Pgps.py @@ -13,7 +13,7 @@ sys.path.append(os.environ['AIL_BIN']) # Import Project packages ################################## from lib.ConfigLoader import ConfigLoader -from lib.objects.abstract_subtype_object import AbstractSubtypeObject, get_all_id, get_all_id_iterator +from lib.objects.abstract_subtype_object import AbstractSubtypeObject, AbstractSubtypeObjects, get_all_id, get_all_id_iterator config_loader = ConfigLoader() baseurl = config_loader.get_config_str("Notifications", "ail_domain") @@ -93,6 +93,30 @@ class Pgp(AbstractSubtypeObject): obj_attr.add_tag(tag) return obj + +class Pgps(AbstractSubtypeObjects): + """ + Usernames Objects + """ + def __init__(self): + super().__init__('pgp', Pgp) + + def get_name(self): + return 'PGP Dumps' + + def get_icon(self): + return {'fas': 'fas', 'icon': 'key'} + + def get_link(self, flask_context=False): + if flask_context: + url = url_for('objects_subtypes.objects_dashboard_pgp') + else: + url = f'{baseurl}/objects/pgps' + return url + + def sanitize_id_to_search(self, subtypes, name_to_search): + return name_to_search + ############################################################################ ############################################################################ diff --git a/bin/lib/objects/Usernames.py b/bin/lib/objects/Usernames.py index bcd808b7..c6438c9e 100755 --- a/bin/lib/objects/Usernames.py +++ b/bin/lib/objects/Usernames.py @@ -143,6 +143,19 @@ class Usernames(AbstractSubtypeObjects): def __init__(self): super().__init__('username', Username) + def get_name(self): + return 'Usernames' + + def get_icon(self): + return {'fas': 'far', 'icon': 'user'} + + def get_link(self, flask_context=False): + if flask_context: + url = url_for('objects_subtypes.objects_dashboard_username') + else: + url = f'{baseurl}/objects/usernames' + return url + def sanitize_id_to_search(self, subtypes, name_to_search): return name_to_search diff --git a/bin/lib/objects/UsersAccount.py b/bin/lib/objects/UsersAccount.py index dea1b7a2..1fba84d7 100755 --- a/bin/lib/objects/UsersAccount.py +++ b/bin/lib/objects/UsersAccount.py @@ -15,7 +15,7 @@ sys.path.append(os.environ['AIL_BIN']) ################################## from lib import ail_core from lib.ConfigLoader import ConfigLoader -from lib.objects.abstract_subtype_object import AbstractSubtypeObject, get_all_id +from lib.objects.abstract_subtype_object import AbstractSubtypeObject, AbstractSubtypeObjects, get_all_id from lib.timeline_engine import Timeline from lib.objects import Usernames @@ -220,8 +220,6 @@ class UserAccount(AbstractSubtypeObject): obj_attr.add_tag(tag) return obj -def get_user_by_username(): - pass def get_all_subtypes(): return ail_core.get_object_all_subtypes('user-account') @@ -236,6 +234,30 @@ def get_all_by_subtype(subtype): return get_all_id('user-account', subtype) +class UserAccounts(AbstractSubtypeObjects): + """ + Usernames Objects + """ + def __init__(self): + super().__init__('user-account', UserAccount) + + def get_name(self): + return 'User-Accounts' + + def get_icon(self): + return {'fas': 'fas', 'icon': 'user-circle'} + + def get_link(self, flask_context=False): + if flask_context: + url = url_for('objects_subtypes.objects_dashboard_user_account') + else: + url = f'{baseurl}/objects/user-accounts' + return url + + def sanitize_id_to_search(self, subtypes, name_to_search): + return name_to_search + + if __name__ == '__main__': from lib.objects import Chats chat = Chats.Chat('', '00098785-7e70-5d12-a120-c5cdc1252b2b') diff --git a/bin/lib/objects/abstract_chat_object.py b/bin/lib/objects/abstract_chat_object.py index 0d2fc203..b4ba87ad 100755 --- a/bin/lib/objects/abstract_chat_object.py +++ b/bin/lib/objects/abstract_chat_object.py @@ -18,7 +18,7 @@ sys.path.append(os.environ['AIL_BIN']) ################################## # Import Project packages ################################## -from lib.objects.abstract_subtype_object import AbstractSubtypeObject +from lib.objects.abstract_subtype_object import AbstractSubtypeObject, AbstractSubtypeObjects from lib.ail_core import unpack_correl_objs_id, zscan_iter ################ from lib.ConfigLoader import ConfigLoader from lib.objects import Messages @@ -314,9 +314,10 @@ class AbstractChatObject(AbstractSubtypeObject, ABC): return self.get_correlation_iter('user-account', self.subtype, user_id, 'message') # TODO move me to abstract subtype -class AbstractChatObjects(ABC): - def __init__(self, type): - self.type = type +class AbstractChatObjects(AbstractSubtypeObjects, ABC): + + def __init__(self, obj_type, obj_class): + super().__init__(obj_type, obj_class) def add_subtype(self, subtype): r_object.sadd(f'all_{self.type}:subtypes', subtype) diff --git a/bin/lib/objects/abstract_subtype_object.py b/bin/lib/objects/abstract_subtype_object.py index 17be1add..9598feec 100755 --- a/bin/lib/objects/abstract_subtype_object.py +++ b/bin/lib/objects/abstract_subtype_object.py @@ -18,7 +18,7 @@ sys.path.append(os.environ['AIL_BIN']) # Import Project packages ################################## from lib.objects.abstract_object import AbstractObject -from lib.ail_core import get_object_all_subtypes, zscan_iter +from lib.ail_core import get_object_all_subtypes, zscan_iter, get_object_all_subtypes from lib.ConfigLoader import ConfigLoader from lib.item_basic import is_crawled, get_item_domain from lib.data_retention_engine import update_obj_date @@ -198,10 +198,46 @@ class AbstractSubtypeObjects(ABC): """ Abstract Subtype Objects """ + def __init__(self, obj_type, obj_class): + """ Abstract for Daterange Objects + + :param obj_type: object type (item, ...) + :param obj_class: object python class (Item, ...) + """ self.type = obj_type self.obj_class = obj_class + def get_subtypes(self): + return get_object_all_subtypes(self.type) + + @abstractmethod + def get_name(self): + pass + + @abstractmethod + def get_icon(self): + pass + + @abstractmethod + def get_link(self, flask_context=False): + pass + + def get_by_date_subtype(self, subtype, date): + return r_object.hkeys(f'{self.type}:{subtype}:{date}') + + def get_by_date(self, date): + pass + + def get_nb_by_date_subtype(self, subtype, date): + return r_object.hlen(f'{self.type}:{subtype}:{date}') + + def get_nb_by_date(self, date): + nb = 0 + for subtype in self.get_subtypes(): + nb += self.get_nb_by_date_subtype(subtype, date) + return nb + def get_ids(self): # TODO FORMAT ids = [] for subtype in get_object_all_subtypes(self.type): @@ -249,6 +285,8 @@ class AbstractSubtypeObjects(ABC): ######################################################################## ######################################################################## +# TODO REFACTOR + def get_all_id(obj_type, subtype): return r_object.zrange(f'{obj_type}_all:{subtype}', 0, -1) @@ -303,5 +341,3 @@ def get_subtypes_objs_range_json(obj_type, date_from, date_to): objs_range.append(day_dict) return objs_range - - diff --git a/bin/lib/objects/ail_objects.py b/bin/lib/objects/ail_objects.py index af12a212..3df4688b 100755 --- a/bin/lib/objects/ail_objects.py +++ b/bin/lib/objects/ail_objects.py @@ -26,8 +26,8 @@ from lib.objects import ChatThreads from lib.objects import CryptoCurrencies from lib.objects import CookiesNames from lib.objects import Cves -from lib.objects.Decodeds import Decoded, get_all_decodeds_objects, get_nb_decodeds_objects -from lib.objects.Domains import Domain +from lib.objects import Decodeds +from lib.objects import Domains from lib.objects import Etags from lib.objects import Favicons from lib.objects import FilesNames @@ -50,29 +50,29 @@ from lib.objects import Usernames # TODO INIT objs classes ???? OBJECTS_CLASS = { 'barcode': {'obj': BarCodes.Barcode, 'objs': BarCodes.Barcodes}, - 'chat': {'obj': Chats.Chat, 'objs': None}, ## SUBTYPE ######################################### + 'chat': {'obj': Chats.Chat, 'objs': Chats.Chats}, 'chat-subchannel': {'obj': ChatSubChannels.ChatSubChannel, 'objs': None}, ###### ###### 'chat-thread': {'obj': ChatThreads.ChatThread, 'objs': None}, ###### ###### 'cookie-name': {'obj': CookiesNames.CookieName, 'objs': CookiesNames.CookiesNames}, 'cve': {'obj': Cves.Cve, 'objs': Cves.Cves}, - 'cryptocurrency': {'obj': CryptoCurrencies.CryptoCurrency, 'objs': None}, ## SUBTYPE ######################################### - 'decoded': {'obj': Decoded, 'objs': None}, ############################################################################################### - 'domain': {'obj': Domain, 'objs': None}, #################################################################################################### + 'cryptocurrency': {'obj': CryptoCurrencies.CryptoCurrency, 'objs': CryptoCurrencies.CryptoCurrencies}, + 'decoded': {'obj': Decodeds.Decoded, 'objs': Decodeds.Decodeds}, + 'domain': {'obj': Domains.Domain, 'objs': Domains.Domains}, 'dom-hash': {'obj': DomHashs.DomHash, 'objs': DomHashs.DomHashs}, 'etag': {'obj': Etags.Etag, 'objs': Etags.Etags}, 'favicon': {'obj': Favicons.Favicon, 'objs': Favicons.Favicons}, 'file-name': {'obj': FilesNames.FileName, 'objs': FilesNames.FilesNames}, 'hhhash': {'obj': HHHashs.HHHash, 'objs': HHHashs.HHHashs}, - 'item': {'obj': Item, 'objs': None}, ###### + 'item': {'obj': Item, 'objs': None}, #################################################################################################### 'image': {'obj': Images.Image, 'objs': Images.Images}, - 'message': {'obj': Messages.Message, 'objs': None}, ###### + 'message': {'obj': Messages.Message, 'objs': None}, ############################################################# 'ocr': {'obj': Ocrs.Ocr, 'objs': Ocrs.Ocrs}, - 'pgp': {'obj': Pgps.Pgp, 'objs': None}, ## SUBTYPE ########################################################################### + 'pgp': {'obj': Pgps.Pgp, 'objs': Pgps.Pgps}, 'qrcode': {'obj': QrCodes.Qrcode, 'objs': QrCodes.Qrcodes}, - 'screenshot': {'obj': Screenshots.Screenshot, 'objs': None}, ###### + 'screenshot': {'obj': Screenshots.Screenshot, 'objs': None}, #################################################################################################### 'title': {'obj': Titles.Title, 'objs': Titles.Titles}, - 'user-account': {'obj': UsersAccount.UserAccount, 'objs': None}, ## SUBTYPE ########################################################################### - 'username': {'obj': Usernames.Username, 'objs': None}, ## SUBTYPE ########################################################################### + 'user-account': {'obj': UsersAccount.UserAccount, 'objs': UsersAccount.UserAccounts}, + 'username': {'obj': Usernames.Username, 'objs': Usernames.Usernames}, } @@ -119,7 +119,7 @@ def get_object(obj_type, subtype, obj_id): return obj_class(obj_id) # SUBTYPES else: - obj_class(obj_id, subtype) + return obj_class(obj_id, subtype) def exists_obj(obj_type, subtype, obj_id): obj = get_object(obj_type, subtype, obj_id) @@ -354,7 +354,7 @@ def is_filtered(obj, filters): def obj_iterator(obj_type, filters): if obj_type == 'decoded': - return get_all_decodeds_objects(filters=filters) + return Decodeds.get_all_decodeds_objects(filters=filters) elif obj_type == 'image': return Images.get_all_images_objects(filters=filters) elif obj_type == 'screenshot': @@ -377,7 +377,7 @@ def card_objs_iterators(filters): def card_obj_iterator(obj_type, filters): if obj_type == 'decoded': - return get_nb_decodeds_objects(filters=filters) + return Decodeds.get_nb_decodeds_objects(filters=filters) elif obj_type == 'item': return get_nb_items_objects(filters=filters) elif obj_type == 'pgp': diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py index af878db7..a6050194 100755 --- a/var/www/Flask_server.py +++ b/var/www/Flask_server.py @@ -312,9 +312,8 @@ def ws_dashboard(ws): # break if int(time.time()) >= next_feeders: feeders = ail_stats.get_feeders_dashboard() - # feeders['data']['telegram'] = 600 - # feeders['data']['test'] = 1300 - ws.send(json.dumps({'feeders': feeders})) + objs = ail_stats.get_nb_objs_today() + ws.send(json.dumps({'feeders': feeders, 'objs': objs})) next_feeders = next_feeders + 30 time.sleep(1) except Exception as e: # ConnectionClosed ? diff --git a/var/www/templates/objects/block_obj_button.html b/var/www/templates/objects/block_obj_button.html index 4e9c1c30..e6e3752f 100644 --- a/var/www/templates/objects/block_obj_button.html +++ b/var/www/templates/objects/block_obj_button.html @@ -1,7 +1,7 @@
- + {{ nb }}
diff --git a/var/www/templates/objects/objs_dashboard.html b/var/www/templates/objects/objs_dashboard.html index 6e89b3b4..f1f9a3fd 100644 --- a/var/www/templates/objects/objs_dashboard.html +++ b/var/www/templates/objects/objs_dashboard.html @@ -71,291 +71,20 @@ {% for obj_type in nb_objects %}
- {% with name=nb_objects[obj_type]['name'], icon=nb_objects[obj_type]['icon']['icon'], nb=nb_objects[obj_type]['nb'], url=nb_objects[obj_type]['link'] %} + {% with type=obj_type, name=nb_objects[obj_type]['name'], icon=nb_objects[obj_type]['icon']['icon'], nb=nb_objects[obj_type]['nb'], url=nb_objects[obj_type]['link'] %} {% include 'objects/block_obj_button.html' %} {% endwith %}
{% endfor %} - -
- -
- - - 384556 - Chats by days - -
-
Chats
- -
-
- -
-
- -
- -
- - - 384556 - Chats by days - -
-
Chats
- -
- -
- -
- - - 5896 - -
-
CVEs
- -
- -
-
- - - 5896 - -
-
CVEs
-
- -
-
- - - 5896 - -
-
CVEs
-
- -
-
- - - 5896 - -
-
CVEs
-
- -
-
- - - 5896 - -
-
CVEs
-
- -
-
- - - 5896 - -
-
CVEs
-
- -
-
- - - 5896 - -
-
CVEs
-
- -
-
- - - 5896 - -
-
CVEs
-
- -
-
- - - 5896 - -
-
CVEs
-
- -
-
- - - 5896 - -
-
CVEs
-
- -
-
- - - 5896 - -
-
CVEs
-
- -
-
- - - 5896 - -
-
CVEs
-
- -
-
- - - 5896 - -
-
CVEs
-
- -
-
- - - 5896 - -
-
CVEs
-
- -
-
- - - 5896 - -
-
CVEs
-
- -
-
- - - 5896 - -
-
CVEs
-
- -
- - - -
- - - - - - - 1,419 - - - - 1,419 - - - - 1,419 - - - - 1,419 - - - - 1,419 - - - - 1,419 - - - - 1,419 - - - - 1,419 - - - - 1,419 - - - - 1,419 - - - - 1,419 - - - - 1,419 - - - - 1,419 - - - - 1,419 - - - - 1,419 - - - - 1,419 - - - - 1,419 - - - - -
+ + +