CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

chg: [chats] add chats explorer v0

dev
terrtia 2023-11-02 16:28:33 +01:00
parent c5cef5fd00
commit 9125119764
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
22 changed files with 2109 additions and 276 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
bin/importer/FeederImporter.py
View File

@ -56,6 +56,8 @@ class FeederImporter(AbstractImporter):
feeders = [f[:-3] for f in os.listdir(feeder_dir) if os.path.isfile(os.path.join(feeder_dir, f))]
self.feeders = {}
for feeder in feeders:
if feeder == 'abstract_chats_feeder':
continue
print(feeder)
part = feeder.split('.')[-1]
# import json importer class

116
bin/importer/feeders/Telegram.py
View File

@ -15,7 +15,7 @@ sys.path.append(os.environ['AIL_BIN'])
##################################
# Import Project packages
##################################
from importer.feeders.Default import DefaultFeeder
from importer.feeders.abstract_chats_feeder import AbstractChatFeeder
from lib.ConfigLoader import ConfigLoader
from lib.objects import ail_objects
from lib.objects.Chats import Chat
@ -24,115 +24,15 @@ from lib.objects import UsersAccount
from lib.objects.Usernames import Username
import base64
import io
import gzip
# TODO remove compression ???
def gunzip_bytes_obj(bytes_obj):
gunzipped_bytes_obj = None
try:
in_ = io.BytesIO()
in_.write(bytes_obj)
in_.seek(0)
with gzip.GzipFile(fileobj=in_, mode='rb') as fo:
gunzipped_bytes_obj = fo.read()
except Exception as e:
print(f'Global; Invalid Gzip file: {e}')
return gunzipped_bytes_obj
class TelegramFeeder(DefaultFeeder):
class TelegramFeeder(AbstractChatFeeder):
def __init__(self, json_data):
super().__init__(json_data)
self.name = 'telegram'
super().__init__('telegram', json_data)
def get_obj(self): # TODO handle others objects -> images, pdf, ...
# Get message date
timestamp = self.json_data['meta']['date']['timestamp'] # TODO CREATE DEFAULT TIMESTAMP
# if self.json_data['meta'].get('date'):
# date = datetime.datetime.fromtimestamp( self.json_data['meta']['date']['timestamp'])
# date = date.strftime('%Y/%m/%d')
# else:
# date = datetime.date.today().strftime("%Y/%m/%d")
chat_id = str(self.json_data['meta']['chat']['id'])
message_id = str(self.json_data['meta']['id'])
obj_id = Messages.create_obj_id('telegram', chat_id, message_id, timestamp)
obj_id = f'message:telegram:{obj_id}'
self.obj = ail_objects.get_obj_from_global_id(obj_id)
return self.obj
# def get_obj(self):.
# obj_id = Messages.create_obj_id('telegram', chat_id, message_id, timestamp)
# obj_id = f'message:telegram:{obj_id}'
# self.obj = ail_objects.get_obj_from_global_id(obj_id)
# return self.obj
def process_meta(self):
"""
Process JSON meta field.
"""
# message chat
meta = self.json_data['meta']
mess_id = self.json_data['meta']['id']
if meta.get('reply_to'):
reply_to_id = int(meta['reply_to'])
else:
reply_to_id = None
timestamp = meta['date']['timestamp']
date = datetime.datetime.fromtimestamp(timestamp)
date = date.strftime('%Y%m%d')
if self.json_data.get('translation'):
translation = self.json_data['translation']
else:
translation = None
decoded = base64.standard_b64decode(self.json_data['data'])
content = gunzip_bytes_obj(decoded)
message = Messages.create(self.obj.id, content, translation=translation)
if meta.get('chat'):
chat = Chat(meta['chat']['id'], 'telegram')
if meta['chat'].get('username'):
chat_username = Username(meta['chat']['username'], 'telegram')
chat.update_username_timeline(chat_username.get_global_id(), timestamp)
# Chat---Message
chat.add(date)
chat.add_message(message.get_global_id(), timestamp, mess_id, reply_id=reply_to_id)
else:
chat = None
# message sender
if meta.get('sender'): # TODO handle message channel forward - check if is user
user_id = meta['sender']['id']
user_account = UsersAccount.UserAccount(user_id, 'telegram')
# UserAccount---Message
user_account.add(date, obj=message)
# UserAccount---Chat
user_account.add_correlation(chat.type, chat.get_subtype(r_str=True), chat.id)
if meta['sender'].get('firstname'):
user_account.set_first_name(meta['sender']['firstname'])
if meta['sender'].get('lastname'):
user_account.set_last_name(meta['sender']['lastname'])
if meta['sender'].get('phone'):
user_account.set_phone(meta['sender']['phone'])
if meta['sender'].get('username'):
username = Username(meta['sender']['username'], 'telegram')
# TODO timeline or/and correlation ????
user_account.add_correlation(username.type, username.get_subtype(r_str=True), username.id)
user_account.update_username_timeline(username.get_global_id(), timestamp)
# Username---Message
username.add(date) # TODO # correlation message ???
# if chat: # TODO Chat---Username correlation ???
# # Chat---Username
# chat.add_correlation(username.type, username.get_subtype(r_str=True), username.id)
# if meta.get('fwd_from'):
# if meta['fwd_from'].get('post_author') # user first name
# TODO reply threads ????
# message edit ????
return None

249
bin/importer/feeders/abstract_chats_feeder.py Executable file
View File

@ -0,0 +1,249 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
"""
Abstract Chat JSON Feeder Importer Module
================
Process Feeder Json (example: Twitter feeder)
"""
import datetime
import os
import sys
from abc import abstractmethod, ABC
sys.path.append(os.environ['AIL_BIN'])
##################################
# Import Project packages
##################################
from importer.feeders.Default import DefaultFeeder
from lib.objects.Chats import Chat
from lib.objects import ChatSubChannels
from lib.objects import Messages
from lib.objects import UsersAccount
from lib.objects.Usernames import Username
from lib import chats_viewer
import base64
import io
import gzip
# TODO remove compression ???
def _gunzip_bytes_obj(bytes_obj):
gunzipped_bytes_obj = None
try:
in_ = io.BytesIO()
in_.write(bytes_obj)
in_.seek(0)
with gzip.GzipFile(fileobj=in_, mode='rb') as fo:
gunzipped_bytes_obj = fo.read()
except Exception as e:
print(f'Global; Invalid Gzip file: {e}')
return gunzipped_bytes_obj
class AbstractChatFeeder(DefaultFeeder, ABC):
def __init__(self, name, json_data):
super().__init__(json_data)
self.obj = None
self.name = name
def get_chat_protocol(self): # TODO # # # # # # # # # # # # #
return self.name
def get_chat_network(self):
self.json_data['meta'].get('network', None)
def get_chat_address(self):
self.json_data['meta'].get('address', None)
def get_chat_instance_uuid(self):
chat_instance_uuid = chats_viewer.create_chat_service_instance(self.get_chat_protocol(),
network=self.get_chat_network(),
address=self.get_chat_address())
# TODO SET
return chat_instance_uuid
def get_chat_id(self): # TODO RAISE ERROR IF NONE
return self.json_data['meta']['chat']['id']
def get_channel_id(self):
pass
def get_subchannels(self):
pass
def get_thread_id(self):
pass
def get_message_timestamp(self):
return self.json_data['meta']['date']['timestamp'] # TODO CREATE DEFAULT TIMESTAMP
# if self.json_data['meta'].get('date'):
# date = datetime.datetime.fromtimestamp( self.json_data['meta']['date']['timestamp'])
# date = date.strftime('%Y/%m/%d')
# else:
# date = datetime.date.today().strftime("%Y/%m/%d")
def get_message_date_timestamp(self):
timestamp = self.get_message_timestamp()
date = datetime.datetime.fromtimestamp(timestamp)
date = date.strftime('%Y%m%d')
return date, timestamp
def get_message_sender_id(self):
return self.json_data['meta']['sender']['id']
def get_message_reply(self):
return self.json_data['meta'].get('reply_to') # TODO change to reply ???
def get_message_reply_id(self):
return self.json_data['meta'].get('reply_to', None)
def get_message_content(self):
decoded = base64.standard_b64decode(self.json_data['data'])
return _gunzip_bytes_obj(decoded)
def get_obj(self): # TODO handle others objects -> images, pdf, ...
#### TIMESTAMP ####
timestamp = self.get_message_timestamp()
#### Create Object ID ####
chat_id = str(self.json_data['meta']['chat']['id'])
message_id = str(self.json_data['meta']['id'])
# channel id
# thread id
obj_id = Messages.create_obj_id(self.get_chat_instance_uuid(), chat_id, message_id, timestamp)
self.obj = Messages.Message(obj_id)
return self.obj
def process_chat(self, message, date, timestamp, reply_id=None): # TODO threads
meta = self.json_data['meta']['chat']
chat = Chat(self.get_chat_id(), self.get_chat_instance_uuid())
chat.add(date) # TODO ### Dynamic subtype ???
if meta.get('name'):
chat.set_name(meta['name'])
if meta.get('date'): # TODO check if already exists
chat.set_created_at(int(meta['date']['timestamp']))
if meta.get('username'):
username = Username(meta['username'], self.get_chat_protocol())
chat.update_username_timeline(username.get_global_id(), timestamp)
if meta.get('subchannel'):
subchannel = self.process_subchannel(message, date, timestamp, reply_id=reply_id)
chat.add_children(obj_global_id=subchannel.get_global_id())
else:
chat.add_message(message.get_global_id(), message.id, timestamp, reply_id=reply_id)
# if meta.get('subchannels'): # TODO Update icon + names
return chat
# def process_subchannels(self):
# pass
def process_subchannel(self, message, date, timestamp, reply_id=None): # TODO CREATE DATE
meta = self.json_data['meta']['chat']['subchannel']
subchannel = ChatSubChannels.ChatSubChannel(f'{self.get_chat_id()}/{meta["id"]}', self.get_chat_instance_uuid())
subchannel.add(date)
if meta.get('date'): # TODO check if already exists
subchannel.set_created_at(int(meta['date']['timestamp']))
if meta.get('name'):
subchannel.set_name(meta['name'])
# subchannel.update_name(meta['name'], timestamp) # TODO #################
subchannel.add_message(message.get_global_id(), message.id, timestamp, reply_id=reply_id)
return subchannel
def process_sender(self, date, timestamp):
meta = self.json_data['meta']['sender']
user_account = UsersAccount.UserAccount(meta['id'], self.get_chat_instance_uuid())
if meta.get('username'):
username = Username(meta['username'], self.get_chat_protocol())
# TODO timeline or/and correlation ????
user_account.add_correlation(username.type, username.get_subtype(r_str=True), username.id)
user_account.update_username_timeline(username.get_global_id(), timestamp)
# Username---Message
username.add(date) # TODO # correlation message ???
# ADDITIONAL METAS
if meta.get('firstname'):
user_account.set_first_name(meta['firstname'])
if meta.get('lastname'):
user_account.set_last_name(meta['lastname'])
if meta.get('phone'):
user_account.set_phone(meta['phone'])
return user_account
# Create abstract class: -> new API endpoint ??? => force field, check if already imported ?
# 1) Create/Get MessageInstance - # TODO uuidv5 + text like discord and telegram for default
# 2) Create/Get CHAT ID - Done
# 3) Create/Get Channel IF is in channel
# 4) Create/Get Thread IF is in thread
# 5) Create/Update Username and User-account - Done
def process_meta(self): # TODO CHECK MANDATORY FIELDS
"""
Process JSON meta filed.
"""
# meta = self.get_json_meta()
date, timestamp = self.get_message_date_timestamp()
# REPLY
reply_id = self.get_message_reply_id()
# TODO Translation
# Content
content = self.get_message_content()
message = Messages.create(self.obj.id, content) # TODO translation
# CHAT
chat = self.process_chat(message, date, timestamp, reply_id=reply_id)
# SENDER # TODO HANDLE NULL SENDER
user_account = self.process_sender(date, timestamp)
# UserAccount---Message
user_account.add(date, obj=message)
# UserAccount---Chat
user_account.add_correlation(chat.type, chat.get_subtype(r_str=True), chat.id)
# if chat: # TODO Chat---Username correlation ???
# # Chat---Username
# chat.add_correlation(username.type, username.get_subtype(r_str=True), username.id)

7
bin/lib/ail_core.py
View File

@ -13,6 +13,7 @@ from lib.ConfigLoader import ConfigLoader
config_loader = ConfigLoader()
r_serv_db = config_loader.get_db_conn("Kvrocks_DB")
r_object = config_loader.get_db_conn("Kvrocks_Objects")
config_loader = None
AIL_OBJECTS = sorted({'chat', 'cookie-name', 'cve', 'cryptocurrency', 'decoded', 'domain', 'etag', 'favicon', 'hhhash', 'item',
@ -40,9 +41,11 @@ def get_all_objects():
def get_objects_with_subtypes():
return ['chat', 'cryptocurrency', 'pgp', 'username']
def get_object_all_subtypes(obj_type):
def get_object_all_subtypes(obj_type): # TODO Dynamic subtype
if obj_type == 'chat':
return ['discord', 'jabber', 'telegram']
return r_object.smembers(f'all_chat:subtypes')
if obj_type == 'chat-subchannel':
return r_object.smembers(f'all_chat-subchannel:subtypes')
if obj_type == 'cryptocurrency':
return ['bitcoin', 'bitcoin-cash', 'dash', 'ethereum', 'litecoin', 'monero', 'zcash']
if obj_type == 'pgp':

305
bin/lib/chats_viewer.py Executable file
View File

@ -0,0 +1,305 @@
#!/usr/bin/python3
"""
Chats Viewer
===================
"""
import os
import sys
import time
import uuid
sys.path.append(os.environ['AIL_BIN'])
##################################
# Import Project packages
##################################
from lib.ConfigLoader import ConfigLoader
from lib.objects import Chats
from lib.objects import ChatSubChannels
config_loader = ConfigLoader()
r_db = config_loader.get_db_conn("Kvrocks_DB")
r_crawler = config_loader.get_db_conn("Kvrocks_Crawler")
r_cache = config_loader.get_redis_conn("Redis_Cache")
r_obj = config_loader.get_db_conn("Kvrocks_DB") # TEMP new DB ????
# # # # # # # #
# #
# COMMON #
# #
# # # # # # # #
# TODO ChatDefaultPlatform
# CHAT(type=chat, subtype=platform, id= chat_id)
# Channel(type=channel, subtype=platform, id=channel_id)
# Thread(type=thread, subtype=platform, id=thread_id)
# Message(type=message, subtype=platform, id=message_id)
# Protocol/Platform
# class ChatProtocols: # TODO Remove Me
#
# def __init__(self): # name ???? subtype, id ????
# # discord, mattermost, ...
# pass
#
# def get_chat_protocols(self):
# pass
#
# def get_chat_protocol(self, protocol):
# pass
#
# ################################################################
#
# def get_instances(self):
# pass
#
# def get_chats(self):
# pass
#
# def get_chats_by_instance(self, instance):
# pass
#
#
# class ChatNetwork: # uuid or protocol
# def __init__(self, network='default'):
# self.id = network
#
# def get_addresses(self):
# pass
#
#
# class ChatServerAddress: # uuid or protocol + network
# def __init__(self, address='default'):
# self.id = address
# map uuid -> type + field
# TODO option last protocol/ imported messages/chat -> unread mode ????
# # # # # # # # #
# #
# PROTOCOLS # IRC, discord, mattermost, ...
# #
# # # # # # # # # TODO icon => UI explorer by protocol + network + instance
def get_chat_protocols():
return r_obj.smembers(f'chat:protocols')
def get_chat_protocols_meta():
metas = []
for protocol_id in get_chat_protocols():
protocol = ChatProtocol(protocol_id)
metas.append(protocol.get_meta(options={'icon'}))
return metas
class ChatProtocol: # TODO first seen last seen ???? + nb by day ????
def __init__(self, protocol):
self.id = protocol
def exists(self):
return r_db.exists(f'chat:protocol:{self.id}')
def get_networks(self):
return r_db.smembers(f'chat:protocol:{self.id}')
def get_nb_networks(self):
return r_db.scard(f'chat:protocol:{self.id}')
def get_icon(self):
if self.id == 'discord':
icon = {'style': 'fab', 'icon': 'fa-discord'}
elif self.id == 'telegram':
icon = {'style': 'fab', 'icon': 'fa-telegram'}
else:
icon = {}
return icon
def get_meta(self, options=set()):
meta = {'id': self.id}
if 'icon' in options:
meta['icon'] = self.get_icon()
return meta
# def get_addresses(self):
# pass
#
# def get_instances_uuids(self):
# pass
# # # # # # # # # # # # # #
# #
# ChatServiceInstance #
# #
# # # # # # # # # # # # # #
# uuid -> protocol + network + server
class ChatServiceInstance:
def __init__(self, instance_uuid):
self.uuid = instance_uuid
def exists(self):
return r_obj.exists(f'chatSerIns:{self.uuid}')
def get_protocol(self): # return objects ????
return r_obj.hget(f'chatSerIns:{self.uuid}', 'protocol')
def get_network(self): # return objects ????
network = r_obj.hget(f'chatSerIns:{self.uuid}', 'network')
if network:
return network
def get_address(self): # return objects ????
address = r_obj.hget(f'chatSerIns:{self.uuid}', 'address')
if address:
return address
def get_meta(self, options=set()):
meta = {'uuid': self.uuid,
'protocol': self.get_protocol(),
'network': self.get_network(),
'address': self.get_address()}
if 'chats' in options:
meta['chats'] = []
for chat_id in self.get_chats():
meta['chats'].append(Chats.Chat(chat_id, self.uuid).get_meta({'nb_subchannels'}))
return meta
def get_nb_chats(self):
return Chats.Chats().get_nb_ids_by_subtype(self.uuid)
def get_chats(self):
return Chats.Chats().get_ids_by_subtype(self.uuid)
def get_chat_service_instances():
return r_obj.smembers(f'chatSerIns:all')
def get_chat_service_instance_uuid(protocol, network, address):
if not network:
network = ''
if not address:
address = ''
return r_obj.hget(f'map:chatSerIns:{protocol}:{network}', address)
def get_chat_service_instance(protocol, network, address):
instance_uuid = get_chat_service_instance_uuid(protocol, network, address)
if instance_uuid:
return ChatServiceInstance(instance_uuid)
def create_chat_service_instance(protocol, network=None, address=None):
instance_uuid = get_chat_service_instance_uuid(protocol, network, address)
if instance_uuid:
return instance_uuid
else:
if not network:
network = ''
if not address:
address = ''
instance_uuid = str(uuid.uuid5(uuid.NAMESPACE_URL, f'{protocol}|{network}|{address}'))
r_obj.sadd(f'chatSerIns:all', instance_uuid)
# map instance - uuid
r_obj.hset(f'map:chatSerIns:{protocol}:{network}', address, instance_uuid)
r_obj.hset(f'chatSerIns:{instance_uuid}', 'protocol', protocol)
if network:
r_obj.hset(f'chatSerIns:{instance_uuid}', 'network', network)
if address:
r_obj.hset(f'chatSerIns:{instance_uuid}', 'address', address)
# protocols
r_obj.sadd(f'chat:protocols', protocol) # TODO first seen / last seen
# protocol -> network
r_obj.sadd(f'chat:protocol:networks:{protocol}', network)
return instance_uuid
# INSTANCE ===> CHAT IDS
# protocol -> instance_uuids => for protocol->networks -> protocol+network => HGETALL
# protocol+network -> instance_uuids => HGETALL
# protocol -> networks ???default??? or ''
# --------------------------------------------------------
# protocol+network -> addresses => HKEYS
# protocol+network+addresse => HGET
# Chat -> subtype=uuid, id = chat id
# instance_uuid -> chat id
# protocol - uniq ID
# protocol + network -> uuid ????
# protocol + network + address -> uuid
#######################################################################################
def get_subchannels_meta_from_global_id(subchannels):
meta = []
for sub in subchannels:
_, instance_uuid, sub_id = sub.split(':', 2)
subchannel = ChatSubChannels.ChatSubChannel(sub_id, instance_uuid)
meta.append(subchannel.get_meta({'nb_messages'}))
return meta
def api_get_chat_service_instance(chat_instance_uuid):
chat_instance = ChatServiceInstance(chat_instance_uuid)
if not chat_instance.exists():
return {"status": "error", "reason": "Unknown uuid"}, 404
return chat_instance.get_meta({'chats'}), 200
def api_get_chat(chat_id, chat_instance_uuid):
chat = Chats.Chat(chat_id, chat_instance_uuid)
if not chat.exists():
return {"status": "error", "reason": "Unknown chat"}, 404
meta = chat.get_meta({'img', 'subchannels', 'username'})
if meta['subchannels']:
print(meta['subchannels'])
meta['subchannels'] = get_subchannels_meta_from_global_id(meta['subchannels'])
return meta, 200
def api_get_subchannel(chat_id, chat_instance_uuid):
subchannel = ChatSubChannels.ChatSubChannel(chat_id, chat_instance_uuid)
if not subchannel.exists():
return {"status": "error", "reason": "Unknown chat"}, 404
meta = subchannel.get_meta({'img', 'nb_messages'})
meta['messages'], meta['tags_messages'] = subchannel.get_messages()
return meta, 200
# # # # # # # # # # LATER
# #
# ChatCategory #
# #
# # # # # # # # # #
if __name__ == '__main__':
r = get_chat_service_instances()
print(r)
r = ChatServiceInstance(r.pop())
print(r.get_meta({'chats'}))
# r = get_chat_protocols()
# print(r)

153
bin/lib/objects/ChatSubChannels.py Executable file
View File

@ -0,0 +1,153 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
import os
import sys
from datetime import datetime
from flask import url_for
# from pymisp import MISPObject
sys.path.append(os.environ['AIL_BIN'])
##################################
# Import Project packages
##################################
from lib import ail_core
from lib.ConfigLoader import ConfigLoader
from lib.objects.abstract_chat_object import AbstractChatObject, AbstractChatObjects
from lib.data_retention_engine import update_obj_date
from lib.objects import ail_objects
from lib.timeline_engine import Timeline
from lib.correlations_engine import get_correlation_by_correl_type
config_loader = ConfigLoader()
baseurl = config_loader.get_config_str("Notifications", "ail_domain")
r_object = config_loader.get_db_conn("Kvrocks_Objects")
r_cache = config_loader.get_redis_conn("Redis_Cache")
config_loader = None
################################################################################
################################################################################
################################################################################
class ChatSubChannel(AbstractChatObject):
"""
AIL Chat Object. (strings)
"""
# ID -> <CHAT ID>/<SubChannel ID> subtype = chat_instance_uuid
def __init__(self, id, subtype):
super(ChatSubChannel, self).__init__('chat-subchannel', id, subtype)
# def get_ail_2_ail_payload(self):
# payload = {'raw': self.get_gzip_content(b64=True),
# 'compress': 'gzip'}
# return payload
# # WARNING: UNCLEAN DELETE /!\ TEST ONLY /!\
def delete(self):
# # TODO:
pass
def get_link(self, flask_context=False):
if flask_context:
url = url_for('correlation.show_correlation', type=self.type, subtype=self.subtype, id=self.id)
else:
url = f'{baseurl}/correlation/show?type={self.type}&subtype={self.subtype}&id={self.id}'
return url
def get_svg_icon(self): # TODO
# if self.subtype == 'telegram':
# style = 'fab'
# icon = '\uf2c6'
# elif self.subtype == 'discord':
# style = 'fab'
# icon = '\uf099'
# else:
# style = 'fas'
# icon = '\uf007'
style = 'fas'
icon = '\uf086'
return {'style': style, 'icon': icon, 'color': '#4dffff', 'radius': 5}
# TODO TIME LAST MESSAGES
def get_meta(self, options=set()):
meta = self._get_meta(options=options)
meta['tags'] = self.get_tags(r_list=True)
meta['name'] = self.get_name()
if 'img' in options:
meta['sub'] = self.get_img()
if 'nb_messages':
meta['nb_messages'] = self.get_nb_messages()
return meta
def get_misp_object(self):
# obj_attrs = []
# if self.subtype == 'telegram':
# obj = MISPObject('telegram-account', standalone=True)
# obj_attrs.append(obj.add_attribute('username', value=self.id))
#
# elif self.subtype == 'twitter':
# obj = MISPObject('twitter-account', standalone=True)
# obj_attrs.append(obj.add_attribute('name', value=self.id))
#
# else:
# obj = MISPObject('user-account', standalone=True)
# obj_attrs.append(obj.add_attribute('username', value=self.id))
#
# first_seen = self.get_first_seen()
# last_seen = self.get_last_seen()
# if first_seen:
# obj.first_seen = first_seen
# if last_seen:
# obj.last_seen = last_seen
# if not first_seen or not last_seen:
# self.logger.warning(
# f'Export error, None seen {self.type}:{self.subtype}:{self.id}, first={first_seen}, last={last_seen}')
#
# for obj_attr in obj_attrs:
# for tag in self.get_tags():
# obj_attr.add_tag(tag)
# return obj
return
############################################################################
############################################################################
# others optional metas, ... -> # TODO ALL meta in hset
def _get_timeline_name(self):
return Timeline(self.get_global_id(), 'username')
def update_name(self, name, timestamp):
self._get_timeline_name().add_timestamp(timestamp, name)
# TODO # # # # # # # # # # #
def get_users(self):
pass
#### Categories ####
#### Threads ####
#### Messages #### TODO set parents
# def get_last_message_id(self):
#
# return r_object.hget(f'meta:{self.type}:{self.subtype}:{self.id}', 'last:message:id')
class ChatSubChannels(AbstractChatObjects):
def __init__(self):
super().__init__('chat-subchannels')
# if __name__ == '__main__':
# chat = Chat('test', 'telegram')
# r = chat.get_messages()
# print(r)

103
bin/lib/objects/ChatThreads.py Executable file
View File

@ -0,0 +1,103 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
import os
import sys
from datetime import datetime
from flask import url_for
# from pymisp import MISPObject
sys.path.append(os.environ['AIL_BIN'])
##################################
# Import Project packages
##################################
from lib import ail_core
from lib.ConfigLoader import ConfigLoader
from lib.objects.abstract_subtype_object import AbstractSubtypeObject, get_all_id
from lib.data_retention_engine import update_obj_date
from lib.objects import ail_objects
from lib.timeline_engine import Timeline
from lib.correlations_engine import get_correlation_by_correl_type
config_loader = ConfigLoader()
baseurl = config_loader.get_config_str("Notifications", "ail_domain")
r_object = config_loader.get_db_conn("Kvrocks_Objects")
r_cache = config_loader.get_redis_conn("Redis_Cache")
config_loader = None
################################################################################
################################################################################
################################################################################
class Chat(AbstractSubtypeObject): # TODO # ID == username ?????
"""
AIL Chat Object. (strings)
"""
def __init__(self, id, subtype):
super(Chat, self).__init__('chat-thread', id, subtype)
# def get_ail_2_ail_payload(self):
# payload = {'raw': self.get_gzip_content(b64=True),
# 'compress': 'gzip'}
# return payload
# # WARNING: UNCLEAN DELETE /!\ TEST ONLY /!\
def delete(self):
# # TODO:
pass
def get_link(self, flask_context=False):
if flask_context:
url = url_for('correlation.show_correlation', type=self.type, subtype=self.subtype, id=self.id)
else:
url = f'{baseurl}/correlation/show?type={self.type}&subtype={self.subtype}&id={self.id}'
return url
def get_svg_icon(self): # TODO
# if self.subtype == 'telegram':
# style = 'fab'
# icon = '\uf2c6'
# elif self.subtype == 'discord':
# style = 'fab'
# icon = '\uf099'
# else:
# style = 'fas'
# icon = '\uf007'
style = 'fas'
icon = '\uf086'
return {'style': style, 'icon': icon, 'color': '#4dffff', 'radius': 5}
def get_meta(self, options=set()):
meta = self._get_meta(options=options)
meta['id'] = self.id
meta['subtype'] = self.subtype
meta['tags'] = self.get_tags(r_list=True)
if 'username':
meta['username'] = self.get_username()
return meta
def get_misp_object(self):
return
############################################################################
############################################################################
# others optional metas, ... -> # TODO ALL meta in hset
#### Messages #### TODO set parents
# def get_last_message_id(self):
#
# return r_object.hget(f'meta:{self.type}:{self.subtype}:{self.id}', 'last:message:id')
if __name__ == '__main__':
chat = Chat('test', 'telegram')
r = chat.get_messages()
print(r)

123
bin/lib/objects/Chats.py
View File

@ -15,6 +15,9 @@ sys.path.append(os.environ['AIL_BIN'])
##################################
from lib import ail_core
from lib.ConfigLoader import ConfigLoader
from lib.objects.abstract_chat_object import AbstractChatObject, AbstractChatObjects
from lib.objects.abstract_subtype_object import AbstractSubtypeObject, get_all_id
from lib.data_retention_engine import update_obj_date
from lib.objects import ail_objects
@ -33,19 +36,14 @@ config_loader = None
################################################################################
################################################################################
class Chat(AbstractSubtypeObject): # TODO # ID == username ?????
class Chat(AbstractChatObject):
"""
AIL Chat Object. (strings)
AIL Chat Object.
"""
def __init__(self, id, subtype):
super(Chat, self).__init__('chat', id, subtype)
# def get_ail_2_ail_payload(self):
# payload = {'raw': self.get_gzip_content(b64=True),
# 'compress': 'gzip'}
# return payload
# # WARNING: UNCLEAN DELETE /!\ TEST ONLY /!\
def delete(self):
# # TODO:
@ -74,9 +72,16 @@ class Chat(AbstractSubtypeObject): # TODO # ID == username ?????
def get_meta(self, options=set()):
meta = self._get_meta(options=options)
meta['id'] = self.id
meta['subtype'] = self.subtype
meta['name'] = self.get_name()
meta['tags'] = self.get_tags(r_list=True)
if 'img':
meta['icon'] = self.get_img()
if 'username' in options:
meta['username'] = self.get_username()
if 'subchannels' in options:
meta['subchannels'] = self.get_subchannels()
if 'nb_subchannels':
meta['nb_subchannels'] = self.get_nb_subchannels()
return meta
def get_misp_object(self):
@ -112,11 +117,6 @@ class Chat(AbstractSubtypeObject): # TODO # ID == username ?????
############################################################################
############################################################################
# others optional metas, ... -> # TODO ALL meta in hset
def get_name(self): # get username ????
pass
# users that send at least a message else participants/spectator
# correlation created by messages
def get_users(self):
@ -138,22 +138,22 @@ class Chat(AbstractSubtypeObject): # TODO # ID == username ?????
def update_username_timeline(self, username_global_id, timestamp):
self._get_timeline_username().add_timestamp(timestamp, username_global_id)
#### ChatSubChannels ####
#### Categories ####
#### Threads ####
#### Messages #### TODO set parents
# def get_last_message_id(self):
#
# return r_object.hget(f'meta:{self.type}:{self.subtype}:{self.id}', 'last:message:id')
def get_obj_message_id(self, obj_id):
if obj_id.endswith('.gz'):
obj_id = obj_id[:-3]
return int(obj_id.split('_')[-1])
def _get_message_timestamp(self, obj_global_id):
return r_object.zscore(f'messages:{self.type}:{self.subtype}:{self.id}', obj_global_id)
def _get_messages(self):
return r_object.zrange(f'messages:{self.type}:{self.subtype}:{self.id}', 0, -1, withscores=True)
def get_message_meta(self, obj_global_id, parent=True, mess_datetime=None):
obj = ail_objects.get_obj_from_global_id(obj_global_id)
mess_dict = obj.get_meta(options={'content', 'link', 'parent', 'user-account'})
@ -179,26 +179,6 @@ class Chat(AbstractSubtypeObject): # TODO # ID == username ?????
mess_dict['hour'] = mess_datetime.strftime('%H:%M:%S')
return mess_dict
def get_messages(self, start=0, page=1, nb=500): # TODO limit nb returned, # TODO add replies
start = 0
stop = -1
# r_object.delete(f'messages:{self.type}:{self.subtype}:{self.id}')
# TODO chat without username ???? -> chat ID ????
messages = {}
curr_date = None
for message in self._get_messages():
date = datetime.fromtimestamp(message[1])
date_day = date.strftime('%Y/%m/%d')
if date_day != curr_date:
messages[date_day] = []
curr_date = date_day
mess_dict = self.get_message_meta(message[0], parent=True, mess_datetime=date)
messages[date_day].append(mess_dict)
return messages
# Zset with ID ??? id -> item id ??? multiple id == media + text
# id -> media id
# How do we handle reply/thread ??? -> separate with new chats name/id ZSET ???
@ -229,43 +209,12 @@ class Chat(AbstractSubtypeObject): # TODO # ID == username ?????
# domain = get_item_domain(item_id)
# self.add_correlation('domain', '', domain)
# TODO kvrocks exception if key don't exists
def get_obj_by_message_id(self, mess_id):
return r_object.hget(f'messages:ids:{self.type}:{self.subtype}:{self.id}', mess_id)
# importer -> use cache for previous reply SET to_add_id: previously_imported : expire SET key -> 30 mn
def add_message(self, obj_global_id, timestamp, mess_id, reply_id=None):
r_object.hset(f'messages:ids:{self.type}:{self.subtype}:{self.id}', mess_id, obj_global_id)
r_object.zadd(f'messages:{self.type}:{self.subtype}:{self.id}', {obj_global_id: timestamp})
if reply_id:
reply_obj = self.get_obj_by_message_id(reply_id)
if reply_obj:
self.add_obj_children(reply_obj, obj_global_id)
else:
self.add_message_cached_reply(reply_id, mess_id)
# ADD cached replies
for reply_obj in self.get_cached_message_reply(mess_id):
self.add_obj_children(obj_global_id, reply_obj)
def _get_message_cached_reply(self, message_id):
return r_cache.smembers(f'messages:ids:{self.type}:{self.subtype}:{self.id}:{message_id}')
def get_cached_message_reply(self, message_id):
objs_global_id = []
for mess_id in self._get_message_cached_reply(message_id):
obj_global_id = self.get_obj_by_message_id(mess_id)
if obj_global_id:
objs_global_id.append(obj_global_id)
return objs_global_id
def add_message_cached_reply(self, reply_to_id, message_id):
r_cache.sadd(f'messages:ids:{self.type}:{self.subtype}:{self.id}:{reply_to_id}', message_id)
r_cache.expire(f'messages:ids:{self.type}:{self.subtype}:{self.id}:{reply_to_id}', 600)
# TODO nb replies = nb son ???? what if it create a onion item ??? -> need source filtering
class Chats(AbstractChatObjects):
def __init__(self):
super().__init__('chat')
# TODO factorize
def get_all_subtypes():
@ -280,28 +229,6 @@ def get_all():
def get_all_by_subtype(subtype):
return get_all_id('chat', subtype)
# # TODO FILTER NAME + Key + mail
# def sanitize_username_name_to_search(name_to_search, subtype): # TODO FILTER NAME
#
# return name_to_search
#
# def search_usernames_by_name(name_to_search, subtype, r_pos=False):
# usernames = {}
# # for subtype in subtypes:
# r_name = sanitize_username_name_to_search(name_to_search, subtype)
# if not name_to_search or isinstance(r_name, dict):
# # break
# return usernames
# r_name = re.compile(r_name)
# for user_name in get_all_usernames_by_subtype(subtype):
# res = re.search(r_name, user_name)
# if res:
# usernames[user_name] = {}
# if r_pos:
# usernames[user_name]['hl-start'] = res.start()
# usernames[user_name]['hl-end'] = res.end()
# return usernames
if __name__ == '__main__':
chat = Chat('test', 'telegram')

22
bin/lib/objects/Messages.py
View File

@ -43,6 +43,10 @@ config_loader = None
# /!\ handle null chat and message id -> chat = uuid and message = timestamp ???
# ID = <ChatInstance UUID>/<timestamp>/<chat ID>/<message ID> => telegram without channels
# ID = <ChatInstance UUID>/<timestamp>/<chat ID>/<Channel ID>/<message ID>
# ID = <ChatInstance UUID>/<timestamp>/<chat ID>/<Thread ID>/<message ID>
# ID = <ChatInstance UUID>/<timestamp>/<chat ID>/<Channel ID>/<Thread ID>/<message ID>
class Message(AbstractObject):
"""
AIL Message Object. (strings)
@ -86,7 +90,7 @@ class Message(AbstractObject):
return dirs[-2]
def get_message_id(self): # TODO optimize
message_id = self.get_basename().rsplit('_', 1)[1]
message_id = self.get_basename().rsplit('/', 1)[1]
# if message_id.endswith('.gz'):
# message_id = message_id[:-3]
return message_id
@ -97,6 +101,10 @@ class Message(AbstractObject):
# chat_id = chat_id[:-3]
return chat_id
# TODO get Instance ID
# TODO get channel ID
# TODO get thread ID
def get_user_account(self):
user_account = self.get_correlation('user-account')
if user_account.get('user-account'):
@ -255,8 +263,16 @@ class Message(AbstractObject):
def delete(self):
pass
def create_obj_id(source, chat_id, message_id, timestamp):
return f'{source}/{timestamp}/{chat_id}_{message_id}'
def create_obj_id(chat_instance, chat_id, message_id, timestamp, channel_id=None, thread_id=None):
timestamp = int(timestamp)
if channel_id and thread_id:
return f'{chat_instance}/{timestamp}/{chat_id}/{chat_id}/{message_id}' # TODO add thread ID ?????
elif channel_id:
return f'{chat_instance}/{timestamp}/{channel_id}/{chat_id}/{message_id}'
elif thread_id:
return f'{chat_instance}/{timestamp}/{chat_id}/{thread_id}/{message_id}'
else:
return f'{chat_instance}/{timestamp}/{chat_id}/{message_id}'
# TODO Check if already exists
# def create(source, chat_id, message_id, timestamp, content, tags=[]):

241
bin/lib/objects/abstract_chat_object.py Executable file
View File

@ -0,0 +1,241 @@
# -*-coding:UTF-8 -*
"""
Base Class for AIL Objects
"""
##################################
# Import External packages
##################################
import os
import sys
from abc import ABC
from datetime import datetime
# from flask import url_for
sys.path.append(os.environ['AIL_BIN'])
##################################
# Import Project packages
##################################
from lib.objects.abstract_subtype_object import AbstractSubtypeObject
from lib.ail_core import get_object_all_subtypes, zscan_iter ################
from lib.ConfigLoader import ConfigLoader
from lib.objects.Messages import Message
from lib.objects.UsersAccount import UserAccount
from lib.objects.Usernames import Username
from lib.data_retention_engine import update_obj_date
from packages import Date
# LOAD CONFIG
config_loader = ConfigLoader()
r_cache = config_loader.get_redis_conn("Redis_Cache")
r_object = config_loader.get_db_conn("Kvrocks_Objects")
config_loader = None
# # FIXME: SAVE SUBTYPE NAMES ?????
class AbstractChatObject(AbstractSubtypeObject, ABC):
"""
Abstract Subtype Object
"""
def __init__(self, obj_type, id, subtype):
""" Abstract for all the AIL object
:param obj_type: object type (item, ...)
:param id: Object ID
"""
super().__init__(obj_type, id, subtype)
# get useraccount / username
# get users ?
# timeline name ????
# info
# created
# last imported/updated
# TODO get instance
# TODO get protocol
# TODO get network
# TODO get address
def get_chat(self): # require ail object TODO ##
if self.type != 'chat':
parent = self.get_parent()
obj_type, _ = parent.split(':', 1)
if obj_type == 'chat':
return parent
def get_subchannels(self):
subchannels = []
if self.type == 'chat': # category ???
print(self.get_childrens())
for obj_global_id in self.get_childrens():
print(obj_global_id)
obj_type, _ = obj_global_id.split(':', 1)
if obj_type == 'chat-subchannel':
subchannels.append(obj_global_id)
return subchannels
def get_nb_subchannels(self):
nb = 0
if self.type == 'chat':
for obj_global_id in self.get_childrens():
obj_type, _ = obj_global_id.split(':', 1)
if obj_type == 'chat-subchannel':
nb += 1
return nb
def get_threads(self):
threads = []
for obj_global_id in self.get_childrens():
obj_type, _ = obj_global_id.split(':', 1)
if obj_type == 'chat-thread':
threads.append(obj_global_id)
return threads
def get_created_at(self):
return self._get_field('created_at')
def set_created_at(self, timestamp):
self._set_field('created_at', timestamp)
def get_name(self):
name = self._get_field('name')
if not name:
name = ''
return name
def set_name(self, name):
self._set_field('name', name)
def get_img(self):
return self._get_field('img')
def set_img(self, icon):
self._set_field('img', icon)
def get_nb_messages(self):
return r_object.zcard(f'messages:{self.type}:{self.subtype}:{self.id}')
def _get_messages(self): # TODO paginate
return r_object.zrange(f'messages:{self.type}:{self.subtype}:{self.id}', 0, -1, withscores=True)
def get_message_meta(self, message, parent=True, mess_datetime=None): # TODO handle file message
obj = Message(message[9:])
mess_dict = obj.get_meta(options={'content', 'link', 'parent', 'user-account'})
print(mess_dict)
if mess_dict.get('parent') and parent:
mess_dict['reply_to'] = self.get_message_meta(mess_dict['parent'], parent=False)
if mess_dict.get('user-account'):
_, user_account_subtype, user_account_id = mess_dict['user-account'].split(':', 3)
print(mess_dict['user-account'])
user_account = UserAccount(user_account_id, user_account_subtype)
mess_dict['user-account'] = {}
mess_dict['user-account']['type'] = user_account.get_type()
mess_dict['user-account']['subtype'] = user_account.get_subtype(r_str=True)
mess_dict['user-account']['id'] = user_account.get_id()
username = user_account.get_username()
if username:
_, username_account_subtype, username_account_id = username.split(':', 3)
username = Username(username_account_id, username_account_subtype).get_default_meta(link=False)
mess_dict['user-account']['username'] = username # TODO get username at the given timestamp ???
else:
mess_dict['user-account'] = {'id': 'UNKNOWN'}
if not mess_datetime:
obj_mess_id = message.get_timestamp()
mess_datetime = datetime.fromtimestamp(obj_mess_id)
mess_dict['date'] = mess_datetime.isoformat(' ')
mess_dict['hour'] = mess_datetime.strftime('%H:%M:%S')
return mess_dict
def get_messages(self, start=0, page=1, nb=500, unread=False): # threads ????
# TODO return message meta
tags = {}
messages = {}
curr_date = None
for message in self._get_messages():
date = datetime.fromtimestamp(message[1])
date_day = date.strftime('%Y/%m/%d')
if date_day != curr_date:
messages[date_day] = []
curr_date = date_day
mess_dict = self.get_message_meta(message[0], parent=True, mess_datetime=date) # TODO use object
messages[date_day].append(mess_dict)
if mess_dict.get('tags'):
for tag in mess_dict['tags']:
if tag not in tags:
tags[tag] = 0
tags[tag] += 1
return messages, tags
# TODO REWRITE ADD OR ADD MESSAGE ????
# add
# add message
def get_obj_by_message_id(self, message_id):
return r_object.hget(f'messages:ids:{self.type}:{self.subtype}:{self.id}', message_id)
def add_message_cached_reply(self, reply_id, message_id):
r_cache.sadd(f'messages:ids:{self.type}:{self.subtype}:{self.id}:{reply_id}', message_id)
r_cache.expire(f'messages:ids:{self.type}:{self.subtype}:{self.id}:{reply_id}', 600)
def _get_message_cached_reply(self, message_id):
return r_cache.smembers(f'messages:ids:{self.type}:{self.subtype}:{self.id}:{message_id}')
def get_cached_message_reply(self, message_id):
objs_global_id = []
for mess_id in self._get_message_cached_reply(message_id):
obj_global_id = self.get_obj_by_message_id(mess_id) # TODO CATCH EXCEPTION
if obj_global_id:
objs_global_id.append(obj_global_id)
return objs_global_id
def add_message(self, obj_global_id, message_id, timestamp, reply_id=None):
r_object.hset(f'messages:ids:{self.type}:{self.subtype}:{self.id}', message_id, obj_global_id)
r_object.zadd(f'messages:{self.type}:{self.subtype}:{self.id}', {obj_global_id: float(timestamp)})
# MESSAGE REPLY
if reply_id:
reply_obj = self.get_obj_by_message_id(reply_id) # TODO CATCH EXCEPTION
if reply_obj:
self.add_obj_children(reply_obj, obj_global_id)
else:
self.add_message_cached_reply(reply_id, message_id)
# get_messages_meta ????
# TODO move me to abstract subtype
class AbstractChatObjects(ABC):
def __init__(self, type):
self.type = type
def add_subtype(self, subtype):
r_object.sadd(f'all_{self.type}:subtypes', subtype)
def get_subtypes(self):
return r_object.smembers(f'all_{self.type}:subtypes')
def get_nb_ids_by_subtype(self, subtype):
return r_object.zcard(f'{self.type}_all:{subtype}')
def get_ids_by_subtype(self, subtype):
print(subtype)
print(f'{self.type}_all:{subtype}')
return r_object.zrange(f'{self.type}_all:{subtype}', 0, -1)
def get_all_id_iterator_iter(self, subtype):
return zscan_iter(r_object, f'{self.type}_all:{subtype}')
def get_ids(self):
pass
def search(self):
pass

7
bin/lib/objects/abstract_object.py
View File

@ -289,23 +289,24 @@ class AbstractObject(ABC):
def get_parent(self):
return r_object.hget(f'meta:{self.type}:{self.get_subtype(r_str=True)}:{self.id}', 'parent')
def get_children(self):
def get_childrens(self):
return r_object.smembers(f'child:{self.type}:{self.get_subtype(r_str=True)}:{self.id}')
def set_parent(self, obj_type=None, obj_subtype=None, obj_id=None, obj_global_id=None): # TODO ######################
def set_parent(self, obj_type=None, obj_subtype=None, obj_id=None, obj_global_id=None): # TODO # REMOVE ITEM DUP
if not obj_global_id:
if obj_subtype is None:
obj_subtype = ''
obj_global_id = f'{obj_type}:{obj_subtype}:{obj_id}'
r_object.hset(f'meta:{self.type}:{self.get_subtype(r_str=True)}:{self.id}', 'parent', obj_global_id)
def add_children(self, obj_type=None, obj_subtype=None, obj_id=None, obj_global_id=None): # TODO ######################
def add_children(self, obj_type=None, obj_subtype=None, obj_id=None, obj_global_id=None): # TODO # REMOVE ITEM DUP
if not obj_global_id:
if obj_subtype is None:
obj_subtype = ''
obj_global_id = f'{obj_type}:{obj_subtype}:{obj_id}'
r_object.sadd(f'child:{self.type}:{self.get_subtype(r_str=True)}:{self.id}', obj_global_id)
## others objects ##
def add_obj_children(self, parent_global_id, son_global_id):
r_object.sadd(f'child:{parent_global_id}', son_global_id)
r_object.hset(f'meta:{son_global_id}', 'parent', parent_global_id)

7
bin/lib/objects/abstract_subtype_object.py
View File

@ -88,7 +88,9 @@ class AbstractSubtypeObject(AbstractObject, ABC):
def _get_meta(self, options=None):
if options is None:
options = set()
meta = {'first_seen': self.get_first_seen(),
meta = {'id': self.id,
'subtype': self.subtype,
'first_seen': self.get_first_seen(),
'last_seen': self.get_last_seen(),
'nb_seen': self.get_nb_seen()}
if 'icon' in options:
@ -150,8 +152,11 @@ class AbstractSubtypeObject(AbstractObject, ABC):
# => data Retention + efficient search
#
#
def _add_subtype(self):
r_object.sadd(f'all_{self.type}:subtypes', self.subtype)
def add(self, date, obj=None):
self._add_subtype()
self.update_daterange(date)
update_obj_date(date, self.type, self.subtype)
# daily

4
var/www/Flask_server.py
View File

@ -50,7 +50,7 @@ from blueprints.objects_title import objects_title
from blueprints.objects_cookie_name import objects_cookie_name
from blueprints.objects_etag import objects_etag
from blueprints.objects_hhhash import objects_hhhash
from blueprints.objects_chat import objects_chat
from blueprints.chats_explorer import chats_explorer
Flask_dir = os.environ['AIL_FLASK']
@ -108,7 +108,7 @@ app.register_blueprint(objects_title, url_prefix=baseUrl)
app.register_blueprint(objects_cookie_name, url_prefix=baseUrl)
app.register_blueprint(objects_etag, url_prefix=baseUrl)
app.register_blueprint(objects_hhhash, url_prefix=baseUrl)
app.register_blueprint(objects_chat, url_prefix=baseUrl)
app.register_blueprint(chats_explorer, url_prefix=baseUrl)
# ========= =========#

132
var/www/blueprints/chats_explorer.py Normal file
View File

@ -0,0 +1,132 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
'''
Blueprint Flask: crawler splash endpoints: dashboard, onion crawler ...
'''
import os
import sys
import json
from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response, abort, send_file
from flask_login import login_required, current_user
# Import Role_Manager
from Role_Manager import login_admin, login_analyst, login_read_only
sys.path.append(os.environ['AIL_BIN'])
##################################
# Import Project packages
##################################
from lib import chats_viewer
############################################
from lib import ail_core
from lib.objects import ail_objects
from lib import chats_viewer
from lib.objects import Chats
from lib.objects import ChatSubChannels
# ============ BLUEPRINT ============
chats_explorer = Blueprint('chats_explorer', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/chats_explorer'))
# ============ VARIABLES ============
bootstrap_label = ['primary', 'success', 'danger', 'warning', 'info']
def create_json_response(data, status_code):
return Response(json.dumps(data, indent=2, sort_keys=True), mimetype='application/json'), status_code
# ============ FUNCTIONS ============
# ============= ROUTES ==============
@chats_explorer.route("/chats/explorer", methods=['GET'])
@login_required
@login_read_only
def chats_explorer_dashboard():
return
@chats_explorer.route("chats/explorer/protocols", methods=['GET'])
@login_required
@login_read_only
def chats_explorer_protocols():
protocols = chats_viewer.get_chat_protocols_meta()
return render_template('chats_protocols.html', protocols=protocols)
@chats_explorer.route("chats/explorer/instance", methods=['GET'])
@login_required
@login_read_only
def chats_explorer_instance():
intance_uuid = request.args.get('uuid')
chat_instance = chats_viewer.api_get_chat_service_instance(intance_uuid)
if chat_instance[1] != 200:
return create_json_response(chat_instance[0], chat_instance[1])
else:
chat_instance = chat_instance[0]
return render_template('chat_instance.html', chat_instance=chat_instance)
@chats_explorer.route("chats/explorer/chat", methods=['GET'])
@login_required
@login_read_only
def chats_explorer_chat():
chat_id = request.args.get('id')
instance_uuid = request.args.get('uuid')
chat = chats_viewer.api_get_chat(chat_id, instance_uuid)
if chat[1] != 200:
return create_json_response(chat[0], chat[1])
else:
chat = chat[0]
return render_template('chat_viewer.html', chat=chat)
@chats_explorer.route("/chats/explorer/subchannel", methods=['GET'])
@login_required
@login_read_only
def objects_subchannel_messages():
subchannel_id = request.args.get('id')
instance_uuid = request.args.get('uuid')
subchannel = chats_viewer.api_get_subchannel(subchannel_id, instance_uuid)
if subchannel[1] != 200:
return create_json_response(subchannel[0], subchannel[1])
else:
subchannel = subchannel[0]
return render_template('SubChannelMessages.html', subchannel=subchannel)
@chats_explorer.route("/chats/explorer/subchannel", methods=['GET'])
@login_required
@login_read_only
def objects_message():
message_id = request.args.get('id')
message = chats_viewer.api_get_message(message_id)
if message[1] != 200:
return create_json_response(message[0], message[1])
else:
message = message[0]
return render_template('ChatMessage.html', message=message)
#############################################################################################
#############################################################################################
#############################################################################################
@chats_explorer.route("/objects/chat/messages", methods=['GET'])
@login_required
@login_read_only
def objects_dashboard_chat():
chat = request.args.get('id')
subtype = request.args.get('subtype')
chat = Chats.Chat(chat, subtype)
if chat.exists():
messages, mess_tags = chat.get_messages()
print(messages)
print(chat.get_subchannels())
meta = chat.get_meta({'icon', 'username'})
if meta.get('username'):
meta['username'] = ail_objects.get_obj_from_global_id(meta['username']).get_meta()
print(meta)
return render_template('ChatMessages.html', meta=meta, messages=messages, mess_tags=mess_tags, bootstrap_label=bootstrap_label)
else:
return abort(404)

58
var/www/blueprints/objects_chat.py
View File

@ -1,58 +0,0 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
'''
Blueprint Flask: crawler splash endpoints: dashboard, onion crawler ...
'''
import os
import sys
import json
from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response, abort, send_file
from flask_login import login_required, current_user
# Import Role_Manager
from Role_Manager import login_admin, login_analyst, login_read_only
sys.path.append(os.environ['AIL_BIN'])
##################################
# Import Project packages
##################################
from lib import ail_core
from lib.objects import abstract_subtype_object
from lib.objects import ail_objects
from lib.objects import Chats
from packages import Date
# ============ BLUEPRINT ============
objects_chat = Blueprint('objects_chat', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/objects/chat'))
# ============ VARIABLES ============
bootstrap_label = ['primary', 'success', 'danger', 'warning', 'info']
def create_json_response(data, status_code):
return Response(json.dumps(data, indent=2, sort_keys=True), mimetype='application/json'), status_code
# ============ FUNCTIONS ============
# ============= ROUTES ==============
@objects_chat.route("/objects/chat/messages", methods=['GET'])
@login_required
@login_read_only
def objects_dashboard_chat():
chat = request.args.get('id')
subtype = request.args.get('subtype')
chat = Chats.Chat(chat, subtype)
if chat.exists():
messages = chat.get_messages()
meta = chat.get_meta({'icon'})
print(meta)
return render_template('ChatMessages.html', meta=meta, messages=messages, bootstrap_label=bootstrap_label)
else:
return abort(404)

209
var/www/templates/chats_explorer/ChatMessage.html Normal file
View File

@ -0,0 +1,209 @@
<!DOCTYPE html>
<html>
<head>
<title>Chat Messages - AIL</title>
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
<!-- JS -->
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
<style>
.chat-message-left,
.chat-message-right {
display: flex;
flex-shrink: 0;
}
.chat-message-right {
flex-direction: row-reverse;
margin-left: auto
}
</style>
</head>
<body>
{% include 'nav_bar.html' %}
<div class="container-fluid">
<div class="row">
{% include 'sidebars/sidebar_objects.html' %}
<div class="col-12 col-lg-10" id="core_content">
<div class="card my-3">
<div class="card-header" style="background-color:#d9edf7;font-size: 15px">
<h3 class="text-secondary">{{ meta["id"] }} :</h3>
<ul class="list-group mb-2">
<li class="list-group-item py-0">
<div class="row">
<div class="col-md-10">
<table class="table">
<thead>
<tr>
<th>Object subtype</th>
<th>First seen</th>
<th>Last seen</th>
<th>Username</th>
<th>Nb seen</th>
</tr>
</thead>
<tbody>
<tr>
<td>
<svg height="26" width="26">
<g class="nodes">
<circle cx="13" cy="13" r="13" fill="{{ meta["icon"]["color"] }}"></circle>
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon {{ meta["icon"]["style"] }}" font-size="16px">{{ meta["icon"]["icon"] }}</text>
</g>
</svg>
{{ meta["subtype"] }}
</td>
<td>{{ meta['first_seen'] }}</td>
<td>{{ meta['last_seen'] }}</td>
<td>
{% if 'username' in meta %}
{{ meta['username']['id'] }}
{% endif %}
</td>
<td>{{ meta['nb_seen'] }}</td>
</tr>
</tbody>
</table>
</div>
<div class="col-md-1">
<div id="sparkline"></div>
</div>
</div>
</li>
<li class="list-group-item py-0">
<br>
<div class="mb-3">
Tags:
{% for tag in meta['tags'] %}
<button class="btn btn-{{ bootstrap_label[loop.index0 % 5] }}" data-toggle="modal" data-target="#edit_tags_modal"
data-tagid="{{ tag }}" data-objtype="chat" data-objsubtype="{{ meta["subtype"] }}" data-objid="{{ meta["id"] }}">
{{ tag }}
</button>
{% endfor %}
<button type="button" class="btn btn-light" data-toggle="modal" data-target="#add_tags_modal">
<i class="far fa-plus-square"></i>
</button>
</div>
</li>
</ul>
{% with obj_type='chat', obj_id=meta['id'], obj_subtype=meta['subtype'] %}
{% include 'modals/investigations_register_obj.html' %}
{% endwith %}
<button type="button" class="btn btn-primary" data-toggle="modal" data-target="#investigations_register_obj_modal">
<i class="fas fa-microscope"></i> Investigations
</button>
</div>
</div>
{% for tag in mess_tags %}
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }} <span class="badge badge-light">{{ mess_tags[tag] }}</span></span>
{% endfor %}
<div>
<div class="list-group d-inline-block">
{% for date in messages %}
<a class="list-group-item list-group-item-action" href="#date_section_{{ date }}">{{ date }}</a>
{% endfor %}
</div>
</div>
<div class="position-relative">
<div class="chat-messages p-4">
<h2 id="date_section_{{ date }}"><span class="badge badge-secondary mb-2">{{ date }}</span></h2>
<div class="chat-message-left pb-1">
<div>
<img src="{{ url_for('static', filename='image/ail-icon.png') }}" class="rounded-circle mr-1" alt="{{ mess['user-account']['id'] }}" width="40" height="40">
<div class="text-muted small text-nowrap mt-2">{{ mess['hour'] }}</div>
</div>
<div class="flex-shrink-1 bg-light rounded py-2 px-3 ml-4 pb-4" style="overflow-x: auto">
<div class="font-weight-bold mb-1">
{% if mess['user-account']['username'] %}
{{ mess['user-account']['username']['id'] }}
{% else %}
{{ mess['user-account']['id'] }}
{% endif %}
</div>
{% if mess['reply_to'] %}
<div class="flex-shrink-1 border rounded py-2 px-3 ml-4 mb-3" style="overflow-x: auto">
<div class="font-weight-bold mb-1">
{% if mess['reply_to']['user-account']['username'] %}
{{ mess['reply_to']['user-account']['username']['id'] }}
{% else %}
{{ mess['reply_to']['user-account']['id'] }}
{% endif %}
</div>
<pre class="my-0">{{ mess['reply_to']['content'] }}</pre>
{% for tag in mess['reply_to']['tags'] %}
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span>
{% endfor %}
<div class="text-muted small text-nowrap">{{ mess['reply_to']['date'] }}</div>
{# <div class="">#}
{# <a class="btn btn-light btn-sm text-secondary py-0" href="{{ url_for('correlation.show_correlation')}}?type={{ mess['reply_to']['type'] }}&subtype={{ mess['reply_to']['subtype'] }}&id={{ mess['reply_to']['id'] }}"><i class="fas fa-project-diagram"></i></a>#}
{# <a class="btn btn-light btn-sm text-secondary py-0" href="{{ mess['reply_to']['link'] }}"><i class="fas fa-eye"></i></a>#}
{# </div>#}
</div>
{% endif %}
<pre class="my-0">{{ mess['content'] }}</pre>
{% for tag in mess['tags'] %}
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span>
{% endfor %}
<div class="">
<a class="btn btn-light btn-sm text-secondary px-1" href="{{ url_for('correlation.show_correlation')}}?type={{ mess['type'] }}&subtype={{ mess['subtype'] }}&id={{ mess['id'] }}"><i class="fas fa-project-diagram"></i></a>
<a class="btn btn-light btn-sm text-secondary px-1" href="{{ mess['link'] }}"><i class="fas fa-eye"></i></a>
</div>
</div>
</div>
</div>
</div>
</div>
</div>
</div>
<script>
var chart = {};
$(document).ready(function(){
$("#page-Decoded").addClass("active");
$("#nav_chat").addClass("active");
});
function toggle_sidebar(){
if($('#nav_menu').is(':visible')){
$('#nav_menu').hide();
$('#side_menu').removeClass('border-right')
$('#side_menu').removeClass('col-lg-2')
$('#core_content').removeClass('col-lg-10')
}else{
$('#nav_menu').show();
$('#side_menu').addClass('border-right')
$('#side_menu').addClass('col-lg-2')
$('#core_content').addClass('col-lg-10')
}
}
</script>
</body>
</html>

20
var/www/templates/objects/chat/ChatMessages.html → var/www/templates/chats_explorer/ChatMessages.html
View File

@ -58,6 +58,7 @@
<th>Object subtype</th>
<th>First seen</th>
<th>Last seen</th>
<th>Username</th>
<th>Nb seen</th>
</tr>
</thead>
@ -74,6 +75,11 @@
</td>
<td>{{ meta['first_seen'] }}</td>
<td>{{ meta['last_seen'] }}</td>
<td>
{% if 'username' in meta %}
{{ meta['username']['id'] }}
{% endif %}
</td>
<td>{{ meta['nb_seen'] }}</td>
</tr>
</tbody>
@ -111,11 +117,23 @@
</div>
</div>
{% for tag in mess_tags %}
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }} <span class="badge badge-light">{{ mess_tags[tag] }}</span></span>
{% endfor %}
<div>
<div class="list-group d-inline-block">
{% for date in messages %}
<a class="list-group-item list-group-item-action" href="#date_section_{{ date }}">{{ date }}</a>
{% endfor %}
</div>
</div>
<div class="position-relative">
<div class="chat-messages p-4">
{% for date in messages %}
<h2><span class="badge badge-secondary mb-2">{{ date }}</span></h2>
<h2 id="date_section_{{ date }}"><span class="badge badge-secondary mb-2">{{ date }}</span></h2>
{% for mess in messages[date] %}
<div class="chat-message-left pb-1">

211
var/www/templates/chats_explorer/SubChannelMessages.html Normal file
View File

@ -0,0 +1,211 @@
<!DOCTYPE html>
<html>
<head>
<title>Sub-Channel Messages - AIL</title>
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
{# <link href="{{ url_for('static', filename='css/daterangepicker.min.css') }}" rel="stylesheet">#}
<!-- JS -->
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
{# <script src="{{ url_for('static', filename='js/moment.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.daterangepicker.min.js') }}"></script>#}
<script src="{{ url_for('static', filename='js/d3.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/d3/sparklines.js')}}"></script>
<style>
.chat-message-left,
.chat-message-right {
display: flex;
flex-shrink: 0;
}
.chat-message-right {
flex-direction: row-reverse;
margin-left: auto
}
</style>
</head>
<body>
{% include 'nav_bar.html' %}
<div class="container-fluid">
<div class="row">
{% include 'sidebars/sidebar_objects.html' %}
<div class="col-12 col-lg-10" id="core_content">
<div class="card my-3">
<div class="card-header" style="background-color:#d9edf7;font-size: 15px">
<h3 class="text-secondary">{{ subchannel["id"] }} :</h3>
<ul class="list-group mb-2">
<li class="list-group-item py-0">
<div class="row">
<div class="col-md-10">
<table class="table">
<thead>
<tr>
<th>Chat Instance</th>
<th>First seen</th>
<th>Last seen</th>
<th>Username</th>
<th>Nb Messages</th>
</tr>
</thead>
<tbody>
<tr>
<td>
{{ subchannel["subtype"] }}
</td>
<td>{{ subchannel['first_seen'] }}</td>
<td>{{ subchannel['last_seen'] }}</td>
<td>
{% if 'username' in subchannel %}
{{ subchannel['username'] }}
{% endif %}
</td>
<td>{{ subchannel['nb_messages'] }}</td>
</tr>
</tbody>
</table>
</div>
</div>
</li>
<li class="list-group-item py-0">
<br>
<div class="mb-3">
Tags:
{% for tag in subchannel['tags'] %}
<button class="btn btn-{{ bootstrap_label[loop.index0 % 5] }}" data-toggle="modal" data-target="#edit_tags_modal"
data-tagid="{{ tag }}" data-objtype="chat" data-objsubtype="{{ subchannel["subtype"] }}" data-objid="{{ subchannel["id"] }}">
{{ tag }}
</button>
{% endfor %}
<button type="button" class="btn btn-light" data-toggle="modal" data-target="#add_tags_modal">
<i class="far fa-plus-square"></i>
</button>
</div>
</li>
</ul>
{% with obj_type='chat', obj_id=subchannel['id'], obj_subtype=subchannel['subtype'] %}
{% include 'modals/investigations_register_obj.html' %}
{% endwith %}
<button type="button" class="btn btn-primary" data-toggle="modal" data-target="#investigations_register_obj_modal">
<i class="fas fa-microscope"></i> Investigations
</button>
</div>
</div>
{% for tag in mess_tags %}
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }} <span class="badge badge-light">{{ mess_tags[tag] }}</span></span>
{% endfor %}
<div>
<div class="list-group d-inline-block">
{% for date in messages %}
<a class="list-group-item list-group-item-action" href="#date_section_{{ date }}">{{ date }}</a>
{% endfor %}
</div>
</div>
<div class="position-relative">
<div class="chat-messages p-4">
{% for date in subchannel['messages'] %}
<h2 id="date_section_{{ date }}"><span class="badge badge-secondary mb-2">{{ date }}</span></h2>
{% for mess in subchannel['messages'][date] %}
<div class="chat-message-left pb-1">
<div>
<img src="{{ url_for('static', filename='image/ail-icon.png') }}" class="rounded-circle mr-1" alt="{{ mess['user-account']['id'] }}" width="40" height="40">
<div class="text-muted small text-nowrap mt-2">{{ mess['hour'] }}</div>
</div>
<div class="flex-shrink-1 bg-light rounded py-2 px-3 ml-4 pb-4" style="overflow-x: auto">
<div class="font-weight-bold mb-1">
{% if mess['user-account']['username'] %}
{{ mess['user-account']['username']['id'] }}
{% else %}
{{ mess['user-account']['id'] }}
{% endif %}
</div>
{% if mess['reply_to'] %}
<div class="flex-shrink-1 border rounded py-2 px-3 ml-4 mb-3" style="overflow-x: auto">
<div class="font-weight-bold mb-1">
{% if mess['reply_to']['user-account']['username'] %}
{{ mess['reply_to']['user-account']['username']['id'] }}
{% else %}
{{ mess['reply_to']['user-account']['id'] }}
{% endif %}
</div>
<pre class="my-0">{{ mess['reply_to']['content'] }}</pre>
{% for tag in mess['reply_to']['tags'] %}
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span>
{% endfor %}
<div class="text-muted small text-nowrap">{{ mess['reply_to']['date'] }}</div>
{# <div class="">#}
{# <a class="btn btn-light btn-sm text-secondary py-0" href="{{ url_for('correlation.show_correlation')}}?type={{ mess['reply_to']['type'] }}&subtype={{ mess['reply_to']['subtype'] }}&id={{ mess['reply_to']['id'] }}"><i class="fas fa-project-diagram"></i></a>#}
{# <a class="btn btn-light btn-sm text-secondary py-0" href="{{ mess['reply_to']['link'] }}"><i class="fas fa-eye"></i></a>#}
{# </div>#}
</div>
{% endif %}
<pre class="my-0">{{ mess['content'] }}</pre>
{% for tag in mess['tags'] %}
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span>
{% endfor %}
<div class="">
<a class="btn btn-light btn-sm text-secondary px-1" href="{{ url_for('correlation.show_correlation')}}?type={{ mess['type'] }}&subtype={{ mess['subtype'] }}&id={{ mess['id'] }}"><i class="fas fa-project-diagram"></i></a>
<a class="btn btn-light btn-sm text-secondary px-1" href="{{ mess['link'] }}"><i class="fas fa-eye"></i></a>
</div>
</div>
</div>
{% endfor %}
<br>
{% endfor %}
</div>
</div>
</div>
</div>
</div>
<script>
var chart = {};
$(document).ready(function(){
$("#page-Decoded").addClass("active");
$("#nav_chat").addClass("active");
});
function toggle_sidebar(){
if($('#nav_menu').is(':visible')){
$('#nav_menu').hide();
$('#side_menu').removeClass('border-right')
$('#side_menu').removeClass('col-lg-2')
$('#core_content').removeClass('col-lg-10')
}else{
$('#nav_menu').show();
$('#side_menu').addClass('border-right')
$('#side_menu').addClass('col-lg-2')
$('#core_content').addClass('col-lg-10')
}
}
</script>
</body>
</html>

123
var/www/templates/chats_explorer/chat_instance.html Normal file
View File

@ -0,0 +1,123 @@
<!DOCTYPE html>
<html>
<head>
<title>Chats Protocols - AIL</title>
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
<!-- JS -->
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
</head>
<body>
{% include 'nav_bar.html' %}
<div class="container-fluid">
<div class="row">
{% include 'sidebars/sidebar_objects.html' %}
<div class="col-12 col-lg-10" id="core_content">
<div class="card my-3">
<div class="card-header" style="background-color:#d9edf7;font-size: 15px">
<h4 class="text-secondary">{{ chat_instance["protocol"] }} :</h4>
<ul class="list-group mb-2">
<li class="list-group-item py-0">
<table class="table">
<thead>
<tr>
<th>Chat Instance</th>
<th>Network</th>
<th>Address</th>
<th>Nb Chats</th>
</tr>
</thead>
<tbody>
<tr>
<td>
{{ chat_instance["uuid"] }}
</td>
<td>{% if chat_instance["network"] %}{{ chat_instance["network"]}}{% endif %}</td>
<td>{% if chat_instance["address"] %}{{ chat_instance["address"] }}{% endif %}</td>
<td>{{ chat_instance["chats"] | length }}</td>
</tr>
</tbody>
</table>
</li>
</ul>
</div>
</div>
<table id="tablechats" class="table table-striped table-bordered">
<thead class="bg-dark text-white">
<tr>
<th>Icon</th>
<th>Name</th>
<th>ID</th>
<th>First Seen</th>
<th>Last Seen</th>
<th>NB Chats</th>
</tr>
</thead>
<tbody style="font-size: 15px;">
{% for chat in chat_instance["chats"] %}
<tr>
<td></td>
<td>{{ chat['name'] }}</td>
<td><a href="{{ url_for('chats_explorer.chats_explorer_chat') }}?uuid={{ chat_instance['uuid'] }}&id={{ chat['id'] }}">{{ chat['id'] }}</a></td>
<td>{{ chat['first_seen'] }}</td>
<td>{{ chat['last_seen'] }}</td>
<td>{{ chat['nb_subchannels'] }}</td>
</tr>
{% endfor %}
</tbody>
</table>
</div>
</div>
</div>
<script>
$(document).ready(function(){
$("#page-Decoded").addClass("active");
$("#nav_cve").addClass("active");
$('#tablechats').DataTable({
"aLengthMenu": [[5, 10, 15, -1], [5, 10, 15, "All"]],
"iDisplayLength": 10,
"order": [[ 4, "desc" ]]
});
});
function toggle_sidebar(){
if($('#nav_menu').is(':visible')){
$('#nav_menu').hide();
$('#side_menu').removeClass('border-right')
$('#side_menu').removeClass('col-lg-2')
$('#core_content').removeClass('col-lg-10')
}else{
$('#nav_menu').show();
$('#side_menu').addClass('border-right')
$('#side_menu').addClass('col-lg-2')
$('#core_content').addClass('col-lg-10')
}
}
</script>
</body>
</html>

133
var/www/templates/chats_explorer/chat_viewer.html Normal file
View File

@ -0,0 +1,133 @@
<!DOCTYPE html>
<html>
<head>
<title>Chats Protocols - AIL</title>
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
<!-- JS -->
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
</head>
<body>
{% include 'nav_bar.html' %}
<div class="container-fluid">
<div class="row">
{% include 'sidebars/sidebar_objects.html' %}
<div class="col-12 col-lg-10" id="core_content">
<div class="card my-3">
<div class="card-header" style="background-color:#d9edf7;font-size: 15px">
<h4 class="text-secondary">{{ chat["username"] }} {{ chat["id"] }} :</h4>
<ul class="list-group mb-2">
<li class="list-group-item py-0">
<table class="table">
<thead class="bg-dark text-white">
<tr>
<th>Icon</th>
<th>Name</th>
<th>ID</th>
<th>First Seen</th>
<th>Last Seen</th>
<th>NB Sub-Channels</th>
</tr>
</thead>
<tbody style="font-size: 15px;">
<tr>
<td></td>
<td>{{ chat['name'] }}</td>
<td>{{ chat['id'] }}</td>
<td>{{ chat['first_seen'] }}</td>
<td>{{ chat['last_seen'] }}</td>
<td>{{ chat['nb_subchannels'] }}</td>
</tr>
</tbody>
</table>
</li>
</ul>
</div>
</div>
{% if chat['subchannels'] %}
<h4>Sub-Channels:</h4>
<table id="tablesubchannels" class="table">
<thead class="bg-dark text-white">
<tr>
<th>Icon</th>
<th>Name</th>
<th>ID</th>
<th>First Seen</th>
<th>Last Seen</th>
<th>NB Messages</th>
</tr>
</thead>
<tbody style="font-size: 15px;">
{% for meta in chat["subchannels"] %}
<tr>
<td>
<img src="{{ url_for('static', filename='image/ail-icon.png') }}" class="rounded-circle mr-1" alt="{{ meta['id'] }}" width="40" height="40">
</td>
<td><b>{{ meta['name'] }}</b></td>
<td><a href="{{ url_for('chats_explorer.objects_subchannel_messages') }}?uuid={{ meta['subtype'] }}&id={{ meta['id'] }}">{{ meta['id'] }}</a></td>
<td>{{ meta['first_seen'] }}</td>
<td>{{ meta['last_seen'] }}</td>
<td>{{ meta['nb_messages'] }}</td>
</tr>
{% endfor %}
</tbody>
</table>
{% endif %}
</div>
</div>
</div>
<script>
$(document).ready(function(){
$("#page-Decoded").addClass("active");
$("#nav_cve").addClass("active");
{% if chat['subchannels'] %}
$('#tablesubchannels').DataTable({
"aLengthMenu": [[5, 10, 15, -1], [5, 10, 15, "All"]],
"iDisplayLength": 10,
"order": [[ 4, "desc" ]]
});
{% endif %}
});
function toggle_sidebar(){
if($('#nav_menu').is(':visible')){
$('#nav_menu').hide();
$('#side_menu').removeClass('border-right')
$('#side_menu').removeClass('col-lg-2')
$('#core_content').removeClass('col-lg-10')
}else{
$('#nav_menu').show();
$('#side_menu').addClass('border-right')
$('#side_menu').addClass('col-lg-2')
$('#core_content').addClass('col-lg-10')
}
}
</script>
</body>
</html>

80
var/www/templates/chats_explorer/chats_instance.html Normal file
View File

@ -0,0 +1,80 @@
<!DOCTYPE html>
<html>
<head>
<title>Chats Protocols - AIL</title>
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
<!-- JS -->
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
</head>
<body>
{% include 'nav_bar.html' %}
<div class="container-fluid">
<div class="row">
{% include 'sidebars/sidebar_objects.html' %}
<div class="col-12 col-lg-10" id="core_content">
{% if protocols %}
<div class="d-flex justify-content-around my-2">
{% for meta in protocols %}
<a class="btn btn-light border-secondary" style="" href="#">
<div class="card-body">
<h4 class="card-title">
{% if meta['icon'] %}
<i class="{{ meta['icon']['style'] }} {{ meta['icon']['icon'] }}"></i>
{% endif %}
<b>{{ meta['id'] }}</b>
</h4>
</div>
</a>
{% endfor %}
</div>
{% else %}
<h4>No Protocol/Chats Imported</h4>
{% endif %}
</div>
</div>
</div>
<script>
$(document).ready(function(){
$("#page-Decoded").addClass("active");
$("#nav_cve").addClass("active");
});
function toggle_sidebar(){
if($('#nav_menu').is(':visible')){
$('#nav_menu').hide();
$('#side_menu').removeClass('border-right')
$('#side_menu').removeClass('col-lg-2')
$('#core_content').removeClass('col-lg-10')
}else{
$('#nav_menu').show();
$('#side_menu').addClass('border-right')
$('#side_menu').addClass('col-lg-2')
$('#core_content').addClass('col-lg-10')
}
}
</script>
</body>
</html>

80
var/www/templates/chats_explorer/chats_protocols.html Normal file
View File

@ -0,0 +1,80 @@
<!DOCTYPE html>
<html>
<head>
<title>Chats Protocols - AIL</title>
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
<!-- JS -->
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
</head>
<body>
{% include 'nav_bar.html' %}
<div class="container-fluid">
<div class="row">
{% include 'sidebars/sidebar_objects.html' %}
<div class="col-12 col-lg-10" id="core_content">
{% if protocols %}
<div class="d-flex justify-content-around my-2">
{% for meta in protocols %}
<a class="btn btn-light border-secondary" style="" href="#">
<div class="card-body">
<h4 class="card-title">
{% if meta['icon'] %}
<i class="{{ meta['icon']['style'] }} {{ meta['icon']['icon'] }}"></i>
{% endif %}
<b>{{ meta['id'] }}</b>
</h4>
</div>
</a>
{% endfor %}
</div>
{% else %}
<h4>No Protocol/Chats Imported</h4>
{% endif %}
</div>
</div>
</div>
<script>
$(document).ready(function(){
$("#page-Decoded").addClass("active");
$("#nav_cve").addClass("active");
});
function toggle_sidebar(){
if($('#nav_menu').is(':visible')){
$('#nav_menu').hide();
$('#side_menu').removeClass('border-right')
$('#side_menu').removeClass('col-lg-2')
$('#core_content').removeClass('col-lg-10')
}else{
$('#nav_menu').show();
$('#side_menu').addClass('border-right')
$('#side_menu').addClass('col-lg-2')
$('#core_content').addClass('col-lg-10')
}
}
</script>
</body>
</html>