mirror of https://github.com/CIRCL/AIL-framework
chg: [chats] add message file-name object + str emoticon reactions
terrtia
parent
235a913865
commit
f766cbebda
|
|
@ -22,6 +22,8 @@ from lib.objects.Chats import Chat
|
|||
from lib.objects import ChatSubChannels
|
||||
from lib.objects import Images
|
||||
from lib.objects import Messages
|
||||
from lib.objects import FilesNames
|
||||
# from lib.objects import Files
|
||||
from lib.objects import UsersAccount
|
||||
from lib.objects.Usernames import Username
|
||||
from lib import chats_viewer
|
||||
|
|
@ -83,6 +85,12 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
|
|||
def get_message_id(self):
|
||||
return self.json_data['meta']['id']
|
||||
|
||||
def get_media_name(self):
|
||||
return self.json_data['meta'].get('media', {}).get('name')
|
||||
|
||||
def get_reactions(self):
|
||||
return self.json_data['meta'].get('reactions', [])
|
||||
|
||||
def get_message_timestamp(self):
|
||||
return self.json_data['meta']['date']['timestamp'] # TODO CREATE DEFAULT TIMESTAMP
|
||||
# if self.json_data['meta'].get('date'):
|
||||
|
|
@ -223,6 +231,9 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
|
|||
user_account.set_icon(img.get_global_id())
|
||||
new_objs.add(img)
|
||||
|
||||
if meta.get('info'):
|
||||
user_account.set_info(meta['info'])
|
||||
|
||||
return user_account
|
||||
|
||||
# Create abstract class: -> new API endpoint ??? => force field, check if already imported ?
|
||||
|
|
@ -251,11 +262,22 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
|
|||
|
||||
print(self.obj.type)
|
||||
|
||||
# TODO FILES + FILES REF
|
||||
|
||||
# get object by meta object type
|
||||
if self.obj.type == 'message':
|
||||
# Content
|
||||
obj = Messages.create(self.obj.id, self.get_message_content()) # TODO translation
|
||||
|
||||
# FILENAME
|
||||
media_name = self.get_media_name()
|
||||
if media_name:
|
||||
print(media_name)
|
||||
FilesNames.FilesNames().create(media_name, date, obj)
|
||||
|
||||
for reaction in self.get_reactions():
|
||||
obj.add_reaction(reaction['reaction'], int(reaction['count']))
|
||||
|
||||
else:
|
||||
chat_id = self.get_chat_id()
|
||||
message_id = self.get_message_id()
|
||||
|
|
@ -271,6 +293,11 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
|
|||
obj.add(date, message)
|
||||
obj.set_parent(obj_global_id=message.get_global_id())
|
||||
|
||||
# FILENAME
|
||||
media_name = self.get_media_name()
|
||||
if media_name:
|
||||
FilesNames.FilesNames().create(media_name, date, message, file_obj=obj)
|
||||
|
||||
for obj in objs: # TODO PERF avoid parsing metas multiple times
|
||||
|
||||
# CHAT
|
||||
|
|
|
|||
|
|
@ -16,7 +16,8 @@ r_serv_db = config_loader.get_db_conn("Kvrocks_DB")
|
|||
r_object = config_loader.get_db_conn("Kvrocks_Objects")
|
||||
config_loader = None
|
||||
|
||||
AIL_OBJECTS = sorted({'chat', 'cookie-name', 'cve', 'cryptocurrency', 'decoded', 'domain', 'etag', 'favicon', 'hhhash',
|
||||
AIL_OBJECTS = sorted({'chat', 'cookie-name', 'cve', 'cryptocurrency', 'decoded', 'domain', 'etag', 'favicon',
|
||||
'file-name', 'hhhash',
|
||||
'item', 'image', 'message', 'pgp', 'screenshot', 'title', 'user-account', 'username'})
|
||||
|
||||
def get_ail_uuid():
|
||||
|
|
|
|||
|
|
@ -49,10 +49,11 @@ CORRELATION_TYPES_BY_OBJ = {
|
|||
"domain": ["cve", "cookie-name", "cryptocurrency", "decoded", "etag", "favicon", "hhhash", "item", "pgp", "title", "screenshot", "username"],
|
||||
"etag": ["domain"],
|
||||
"favicon": ["domain", "item"], # TODO Decoded
|
||||
"file-name": ["chat", "message"],
|
||||
"hhhash": ["domain"],
|
||||
"image": ["chat", "message", "user-account"],
|
||||
"item": ["cve", "cryptocurrency", "decoded", "domain", "favicon", "pgp", "screenshot", "title", "username"], # chat ???
|
||||
"message": ["cve", "cryptocurrency", "decoded", "image", "pgp", "user-account"], # chat ??
|
||||
"message": ["cve", "cryptocurrency", "decoded", "file-name", "image", "pgp", "user-account"], # chat ??
|
||||
"pgp": ["domain", "item", "message"],
|
||||
"screenshot": ["domain", "item"],
|
||||
"title": ["domain", "item"],
|
||||
|
|
|
|||
|
|
@ -0,0 +1,101 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
from flask import url_for
|
||||
from pymisp import MISPObject
|
||||
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from lib.ConfigLoader import ConfigLoader
|
||||
from lib.objects.abstract_daterange_object import AbstractDaterangeObject, AbstractDaterangeObjects
|
||||
|
||||
config_loader = ConfigLoader()
|
||||
r_object = config_loader.get_db_conn("Kvrocks_Objects")
|
||||
config_loader = None
|
||||
|
||||
|
||||
class FileName(AbstractDaterangeObject):
|
||||
"""
|
||||
AIL FileName Object. (strings)
|
||||
"""
|
||||
|
||||
# ID = SHA256
|
||||
def __init__(self, name):
|
||||
super().__init__('file-name', name)
|
||||
|
||||
# def get_ail_2_ail_payload(self):
|
||||
# payload = {'raw': self.get_gzip_content(b64=True),
|
||||
# 'compress': 'gzip'}
|
||||
# return payload
|
||||
|
||||
# # WARNING: UNCLEAN DELETE /!\ TEST ONLY /!\
|
||||
def delete(self):
|
||||
# # TODO:
|
||||
pass
|
||||
|
||||
def get_link(self, flask_context=False):
|
||||
if flask_context:
|
||||
url = url_for('correlation.show_correlation', type=self.type, id=self.id)
|
||||
else:
|
||||
url = f'{baseurl}/correlation/show?type={self.type}&id={self.id}'
|
||||
return url
|
||||
|
||||
def get_svg_icon(self):
|
||||
return {'style': 'far', 'icon': '\uf249', 'color': '#36F5D5', 'radius': 5}
|
||||
|
||||
def get_misp_object(self):
|
||||
obj_attrs = []
|
||||
obj = MISPObject('file')
|
||||
|
||||
# obj_attrs.append(obj.add_attribute('sha256', value=self.id))
|
||||
# obj_attrs.append(obj.add_attribute('attachment', value=self.id, data=self.get_file_content()))
|
||||
for obj_attr in obj_attrs:
|
||||
for tag in self.get_tags():
|
||||
obj_attr.add_tag(tag)
|
||||
return obj
|
||||
|
||||
def get_meta(self, options=set()):
|
||||
meta = self._get_meta(options=options)
|
||||
meta['id'] = self.id
|
||||
meta['tags'] = self.get_tags(r_list=True)
|
||||
if 'tags_safe' in options:
|
||||
meta['tags_safe'] = self.is_tags_safe(meta['tags'])
|
||||
return meta
|
||||
|
||||
def create(self): # create ALL SET ??????
|
||||
pass
|
||||
|
||||
def add_reference(self, date, src_ail_object, file_obj=None):
|
||||
self.add(date, src_ail_object)
|
||||
if file_obj:
|
||||
self.add_correlation(file_obj.type, file_obj.get_subtype(r_str=True), file_obj.get_id())
|
||||
|
||||
# TODO USE ZSET FOR ALL OBJS IDS ??????
|
||||
|
||||
class FilesNames(AbstractDaterangeObjects):
|
||||
"""
|
||||
CookieName Objects
|
||||
"""
|
||||
def __init__(self):
|
||||
super().__init__('file-name', FileName)
|
||||
|
||||
def sanitize_id_to_search(self, name_to_search):
|
||||
return name_to_search
|
||||
|
||||
# TODO sanitize file name
|
||||
def create(self, name, date, src_ail_object, file_obj=None, limit=500, force=False):
|
||||
if 0 < len(name) <= limit or force or limit < 0:
|
||||
file_name = self.obj_class(name)
|
||||
# if not file_name.exists():
|
||||
# file_name.create()
|
||||
file_name.add_reference(date, src_ail_object, file_obj=file_obj)
|
||||
return file_name
|
||||
|
||||
# if __name__ == '__main__':
|
||||
# name_to_search = '29ba'
|
||||
# print(search_screenshots_by_name(name_to_search))
|
||||
|
|
@ -118,9 +118,24 @@ class Message(AbstractObject):
|
|||
user_account = f'user-account:{user_account["user-account"].pop()}'
|
||||
if meta:
|
||||
_, user_account_subtype, user_account_id = user_account.split(':', 3)
|
||||
user_account = UsersAccount.UserAccount(user_account_id, user_account_subtype).get_meta(options={'username', 'username_meta'})
|
||||
user_account = UsersAccount.UserAccount(user_account_id, user_account_subtype).get_meta(options={'icon', 'username', 'username_meta'})
|
||||
return user_account
|
||||
|
||||
def get_files_names(self):
|
||||
names = []
|
||||
filenames = self.get_correlation('file-name').get('file-name')
|
||||
if filenames:
|
||||
for name in filenames:
|
||||
names.append(name[1:])
|
||||
return names
|
||||
|
||||
def get_reactions(self):
|
||||
return r_object.hgetall(f'meta:reactions:{self.type}::{self.id}')
|
||||
|
||||
# TODO sanitize reactions
|
||||
def add_reaction(self, reactions, nb_reaction):
|
||||
r_object.hset(f'meta:reactions:{self.type}::{self.id}', reactions, nb_reaction)
|
||||
|
||||
# Update value on import
|
||||
# reply to -> parent ?
|
||||
# reply/comment - > children ?
|
||||
|
|
@ -232,6 +247,10 @@ class Message(AbstractObject):
|
|||
meta['chat'] = self.get_chat_id()
|
||||
if 'images' in options:
|
||||
meta['images'] = self.get_images()
|
||||
if 'files-names' in options:
|
||||
meta['files-names'] = self.get_files_names()
|
||||
if 'reactions' in options:
|
||||
meta['reactions'] = self.get_reactions()
|
||||
|
||||
# meta['encoding'] = None
|
||||
return meta
|
||||
|
|
@ -314,8 +333,8 @@ def create_obj_id(chat_instance, chat_id, message_id, timestamp, channel_id=None
|
|||
# def create(source, chat_id, message_id, timestamp, content, tags=[]):
|
||||
def create(obj_id, content, translation=None, tags=[]):
|
||||
message = Message(obj_id)
|
||||
if not message.exists():
|
||||
message.create(content, translation=translation, tags=tags)
|
||||
# if not message.exists():
|
||||
message.create(content, translation=translation, tags=tags)
|
||||
return message
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -91,6 +91,12 @@ class UserAccount(AbstractSubtypeObject):
|
|||
def set_icon(self, icon):
|
||||
self._set_field('icon', icon)
|
||||
|
||||
def get_info(self):
|
||||
return self._get_field('info')
|
||||
|
||||
def set_info(self, info):
|
||||
return self._set_field('info', info)
|
||||
|
||||
def _get_timeline_username(self):
|
||||
return Timeline(self.get_global_id(), 'username')
|
||||
|
||||
|
|
|
|||
|
|
@ -181,7 +181,7 @@ class AbstractChatObject(AbstractSubtypeObject, ABC):
|
|||
|
||||
def get_message_meta(self, message, timestamp=None): # TODO handle file message
|
||||
message = Messages.Message(message[9:])
|
||||
meta = message.get_meta(options={'content', 'images', 'link', 'parent', 'parent_meta', 'user-account'}, timestamp=timestamp)
|
||||
meta = message.get_meta(options={'content', 'files-names', 'images', 'link', 'parent', 'parent_meta', 'reactions', 'user-account'}, timestamp=timestamp)
|
||||
return meta
|
||||
|
||||
def get_messages(self, start=0, page=1, nb=500, unread=False): # threads ???? # TODO ADD last/first message timestamp + return page
|
||||
|
|
@ -189,7 +189,7 @@ class AbstractChatObject(AbstractSubtypeObject, ABC):
|
|||
tags = {}
|
||||
messages = {}
|
||||
curr_date = None
|
||||
for message in self._get_messages(nb=30, page=1):
|
||||
for message in self._get_messages(nb=50, page=1):
|
||||
timestamp = message[1]
|
||||
date_day = datetime.fromtimestamp(timestamp).strftime('%Y/%m/%d')
|
||||
if date_day != curr_date:
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ from lib.objects.Decodeds import Decoded, get_all_decodeds_objects, get_nb_decod
|
|||
from lib.objects.Domains import Domain
|
||||
from lib.objects import Etags
|
||||
from lib.objects.Favicons import Favicon
|
||||
from lib.objects import FilesNames
|
||||
from lib.objects import HHHashs
|
||||
from lib.objects.Items import Item, get_all_items_objects, get_nb_items_objects
|
||||
from lib.objects import Images
|
||||
|
|
@ -69,6 +70,8 @@ def get_object(obj_type, subtype, obj_id):
|
|||
return Etags.Etag(obj_id)
|
||||
elif obj_type == 'favicon':
|
||||
return Favicon(obj_id)
|
||||
elif obj_type == 'file-name':
|
||||
return FilesNames.FileName(obj_id)
|
||||
elif obj_type == 'hhhash':
|
||||
return HHHashs.HHHash(obj_id)
|
||||
elif obj_type == 'image':
|
||||
|
|
|
|||
|
|
@ -63,7 +63,17 @@
|
|||
<img class="message_image mb-1" src="{{ url_for('objects_image.image', filename=message_image)}}">
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
{% if message['files-names'] %}
|
||||
{% for file_name in message['files-names'] %}
|
||||
<div class="flex-shrink-1 bg-white border-primary text-secondary rounded py-2 px-3 ml-4 mb-3" style="overflow-x: auto">
|
||||
<i class="far fa-file fa-3x"></i> {{ file_name }}
|
||||
</div>
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
<pre class="my-0">{{ message['content'] }}</pre>
|
||||
{% for reaction in message['reactions'] %}
|
||||
<span class="border rounded px-1">{{ reaction }} {{ message['reactions'][reaction] }}</span>
|
||||
{% endfor %}
|
||||
{% for tag in message['tags'] %}
|
||||
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span>
|
||||
{% endfor %}
|
||||
|
|
|
|||
Loading…
Reference in New Issue