mirror of https://github.com/CIRCL/AIL-framework
341 lines
12 KiB
Python
Executable File
341 lines
12 KiB
Python
Executable File
# -*-coding:UTF-8 -*
|
|
"""
|
|
Base Class for AIL Objects
|
|
"""
|
|
|
|
##################################
|
|
# Import External packages
|
|
##################################
|
|
import os
|
|
import sys
|
|
import time
|
|
from abc import ABC
|
|
|
|
from datetime import datetime
|
|
# from flask import url_for
|
|
|
|
sys.path.append(os.environ['AIL_BIN'])
|
|
##################################
|
|
# Import Project packages
|
|
##################################
|
|
from lib.objects.abstract_subtype_object import AbstractSubtypeObject
|
|
from lib.ail_core import unpack_correl_objs_id, zscan_iter ################
|
|
from lib.ConfigLoader import ConfigLoader
|
|
from lib.objects import Messages
|
|
from packages import Date
|
|
|
|
# from lib.data_retention_engine import update_obj_date
|
|
|
|
|
|
# LOAD CONFIG
|
|
config_loader = ConfigLoader()
|
|
r_cache = config_loader.get_redis_conn("Redis_Cache")
|
|
r_object = config_loader.get_db_conn("Kvrocks_Objects")
|
|
config_loader = None
|
|
|
|
# # FIXME: SAVE SUBTYPE NAMES ?????
|
|
|
|
class AbstractChatObject(AbstractSubtypeObject, ABC):
|
|
"""
|
|
Abstract Subtype Object
|
|
"""
|
|
|
|
def __init__(self, obj_type, id, subtype):
|
|
""" Abstract for all the AIL object
|
|
|
|
:param obj_type: object type (item, ...)
|
|
:param id: Object ID
|
|
"""
|
|
super().__init__(obj_type, id, subtype)
|
|
|
|
# get useraccount / username
|
|
# get users ?
|
|
# timeline name ????
|
|
# last imported/updated
|
|
|
|
# TODO get instance
|
|
# TODO get protocol
|
|
# TODO get network
|
|
# TODO get address
|
|
|
|
def get_chat(self): # require ail object TODO ##
|
|
if self.type != 'chat':
|
|
parent = self.get_parent()
|
|
if parent:
|
|
obj_type, _ = parent.split(':', 1)
|
|
if obj_type == 'chat':
|
|
return parent
|
|
|
|
def get_subchannels(self):
|
|
subchannels = []
|
|
if self.type == 'chat': # category ???
|
|
for obj_global_id in self.get_childrens():
|
|
obj_type, _ = obj_global_id.split(':', 1)
|
|
if obj_type == 'chat-subchannel':
|
|
subchannels.append(obj_global_id)
|
|
return subchannels
|
|
|
|
def get_nb_subchannels(self):
|
|
nb = 0
|
|
if self.type == 'chat':
|
|
for obj_global_id in self.get_childrens():
|
|
obj_type, _ = obj_global_id.split(':', 1)
|
|
if obj_type == 'chat-subchannel':
|
|
nb += 1
|
|
return nb
|
|
|
|
def get_threads(self):
|
|
threads = []
|
|
for child in self.get_childrens():
|
|
obj_type, obj_subtype, obj_id = child.split(':', 2)
|
|
if obj_type == 'chat-thread':
|
|
threads.append({'type': obj_type, 'subtype': obj_subtype, 'id': obj_id})
|
|
return threads
|
|
|
|
def get_created_at(self, date=False):
|
|
created_at = self._get_field('created_at')
|
|
if date and created_at:
|
|
created_at = datetime.utcfromtimestamp(float(created_at))
|
|
created_at = created_at.isoformat(' ')
|
|
return created_at
|
|
|
|
def set_created_at(self, timestamp):
|
|
self._set_field('created_at', timestamp)
|
|
|
|
def get_name(self):
|
|
name = self._get_field('name')
|
|
if not name:
|
|
name = ''
|
|
return name
|
|
|
|
def set_name(self, name):
|
|
self._set_field('name', name)
|
|
|
|
def get_icon(self):
|
|
icon = self._get_field('icon')
|
|
if icon:
|
|
return icon.rsplit(':', 1)[1]
|
|
|
|
def set_icon(self, icon):
|
|
self._set_field('icon', icon)
|
|
|
|
def get_info(self):
|
|
return self._get_field('info')
|
|
|
|
def set_info(self, info):
|
|
self._set_field('info', info)
|
|
|
|
def get_nb_messages(self):
|
|
return r_object.zcard(f'messages:{self.type}:{self.subtype}:{self.id}')
|
|
|
|
def _get_messages(self, nb=-1, page=-1):
|
|
if nb < 1:
|
|
messages = r_object.zrange(f'messages:{self.type}:{self.subtype}:{self.id}', 0, -1, withscores=True)
|
|
nb_pages = 0
|
|
page = 1
|
|
total = len(messages)
|
|
nb_first = 1
|
|
nb_last = total
|
|
else:
|
|
total = r_object.zcard(f'messages:{self.type}:{self.subtype}:{self.id}')
|
|
nb_pages = total / nb
|
|
if not nb_pages.is_integer():
|
|
nb_pages = int(nb_pages) + 1
|
|
else:
|
|
nb_pages = int(nb_pages)
|
|
if page > nb_pages or page < 1:
|
|
page = nb_pages
|
|
|
|
if page > 1:
|
|
start = (page - 1) * nb
|
|
else:
|
|
start = 0
|
|
messages = r_object.zrange(f'messages:{self.type}:{self.subtype}:{self.id}', start, start+nb-1, withscores=True)
|
|
# if messages:
|
|
# messages = reversed(messages)
|
|
nb_first = start+1
|
|
nb_last = start+nb
|
|
if nb_last > total:
|
|
nb_last = total
|
|
return messages, {'nb': nb, 'page': page, 'nb_pages': nb_pages, 'total': total, 'nb_first': nb_first, 'nb_last': nb_last}
|
|
|
|
def get_timestamp_first_message(self):
|
|
return r_object.zrange(f'messages:{self.type}:{self.subtype}:{self.id}', 0, 0, withscores=True)
|
|
|
|
def get_timestamp_last_message(self):
|
|
return r_object.zrevrange(f'messages:{self.type}:{self.subtype}:{self.id}', 0, 0, withscores=True)
|
|
|
|
def get_first_message(self):
|
|
return r_object.zrange(f'messages:{self.type}:{self.subtype}:{self.id}', 0, 0)
|
|
|
|
def get_last_message(self):
|
|
return r_object.zrevrange(f'messages:{self.type}:{self.subtype}:{self.id}', 0, 0)
|
|
|
|
def get_nb_message_by_hours(self, date_day, nb_day):
|
|
hours = []
|
|
# start=0, end=23
|
|
timestamp = time.mktime(datetime.strptime(date_day, "%Y%m%d").utctimetuple())
|
|
for i in range(24):
|
|
timestamp_end = timestamp + 3600
|
|
nb_messages = r_object.zcount(f'messages:{self.type}:{self.subtype}:{self.id}', timestamp, timestamp_end)
|
|
timestamp = timestamp_end
|
|
hours.append({'date': f'{date_day[0:4]}-{date_day[4:6]}-{date_day[6:8]}', 'day': nb_day, 'hour': i, 'count': nb_messages})
|
|
return hours
|
|
|
|
def get_nb_message_by_week(self, date_day):
|
|
date_day = Date.get_date_week_by_date(date_day)
|
|
week_messages = []
|
|
i = 0
|
|
for date in Date.daterange_add_days(date_day, 6):
|
|
week_messages = week_messages + self.get_nb_message_by_hours(date, i)
|
|
i += 1
|
|
return week_messages
|
|
|
|
def get_nb_message_this_week(self):
|
|
week_date = Date.get_current_week_day()
|
|
return self.get_nb_message_by_week(week_date)
|
|
|
|
def get_nb_week_messages(self):
|
|
week = {}
|
|
# Init
|
|
for day in ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun']:
|
|
week[day] = {}
|
|
for i in range(24):
|
|
week[day][i] = 0
|
|
|
|
# chat
|
|
for mess_t in r_object.zrange(f'messages:{self.type}:{self.subtype}:{self.id}', 0, -1, withscores=True):
|
|
timestamp = datetime.utcfromtimestamp(float(mess_t[1]))
|
|
date_name = timestamp.strftime('%a')
|
|
week[date_name][timestamp.hour] += 1
|
|
|
|
subchannels = self.get_subchannels()
|
|
for gid in subchannels:
|
|
for mess_t in r_object.zrange(f'messages:{gid}', 0, -1, withscores=True):
|
|
timestamp = datetime.utcfromtimestamp(float(mess_t[1]))
|
|
date_name = timestamp.strftime('%a')
|
|
week[date_name][timestamp.hour] += 1
|
|
stats = []
|
|
nb_day = 0
|
|
for day in week:
|
|
for hour in week[day]:
|
|
stats.append({'date': day, 'day': nb_day, 'hour': hour, 'count': week[day][hour]})
|
|
nb_day += 1
|
|
return stats
|
|
|
|
def get_message_meta(self, message, timestamp=None, translation_target='', options=None): # TODO handle file message
|
|
message = Messages.Message(message[9:])
|
|
if not options:
|
|
options = {'content', 'files-names', 'forwarded_from', 'images', 'language', 'link', 'parent', 'parent_meta', 'reactions', 'thread', 'translation', 'user-account'}
|
|
meta = message.get_meta(options=options, timestamp=timestamp, translation_target=translation_target)
|
|
return meta
|
|
|
|
def get_messages(self, start=0, page=-1, nb=500, unread=False, options=None, translation_target='en'): # threads ???? # TODO ADD last/first message timestamp + return page
|
|
# TODO return message meta
|
|
tags = {}
|
|
messages = {}
|
|
curr_date = None
|
|
try:
|
|
nb = int(nb)
|
|
except TypeError:
|
|
nb = 500
|
|
if not page:
|
|
page = -1
|
|
try:
|
|
page = int(page)
|
|
except TypeError:
|
|
page = 1
|
|
mess, pagination = self._get_messages(nb=nb, page=page)
|
|
for message in mess:
|
|
timestamp = message[1]
|
|
date_day = datetime.utcfromtimestamp(timestamp).strftime('%Y/%m/%d')
|
|
if date_day != curr_date:
|
|
messages[date_day] = []
|
|
curr_date = date_day
|
|
mess_dict = self.get_message_meta(message[0], timestamp=timestamp, translation_target=translation_target, options=options)
|
|
messages[date_day].append(mess_dict)
|
|
|
|
if mess_dict.get('tags'):
|
|
for tag in mess_dict['tags']:
|
|
if tag not in tags:
|
|
tags[tag] = 0
|
|
tags[tag] += 1
|
|
return messages, pagination, tags
|
|
|
|
# TODO REWRITE ADD OR ADD MESSAGE ????
|
|
# add
|
|
# add message
|
|
|
|
def get_obj_by_message_id(self, message_id):
|
|
return r_object.hget(f'messages:ids:{self.type}:{self.subtype}:{self.id}', message_id)
|
|
|
|
def add_message_cached_reply(self, reply_id, message_id):
|
|
r_cache.sadd(f'messages:ids:{self.type}:{self.subtype}:{self.id}:{reply_id}', message_id)
|
|
r_cache.expire(f'messages:ids:{self.type}:{self.subtype}:{self.id}:{reply_id}', 600)
|
|
|
|
def _get_message_cached_reply(self, message_id):
|
|
return r_cache.smembers(f'messages:ids:{self.type}:{self.subtype}:{self.id}:{message_id}')
|
|
|
|
def get_cached_message_reply(self, message_id):
|
|
objs_global_id = []
|
|
for mess_id in self._get_message_cached_reply(message_id):
|
|
obj_global_id = self.get_obj_by_message_id(mess_id) # TODO CATCH EXCEPTION
|
|
if obj_global_id:
|
|
objs_global_id.append(obj_global_id)
|
|
return objs_global_id
|
|
|
|
def add_chat_with_messages(self):
|
|
r_object.sadd(f'{self.type}_w_mess:{self.subtype}', self.id)
|
|
|
|
def add_message(self, obj_global_id, message_id, timestamp, reply_id=None):
|
|
r_object.hset(f'messages:ids:{self.type}:{self.subtype}:{self.id}', message_id, obj_global_id)
|
|
r_object.zadd(f'messages:{self.type}:{self.subtype}:{self.id}', {obj_global_id: float(timestamp)})
|
|
|
|
# MESSAGE REPLY
|
|
if reply_id:
|
|
reply_obj = self.get_obj_by_message_id(reply_id) # TODO CATCH EXCEPTION
|
|
if reply_obj:
|
|
self.add_obj_children(reply_obj, obj_global_id)
|
|
else:
|
|
self.add_message_cached_reply(reply_id, message_id)
|
|
# CACHED REPLIES
|
|
for mess_id in self.get_cached_message_reply(message_id):
|
|
self.add_obj_children(obj_global_id, mess_id)
|
|
|
|
# def get_deleted_messages(self, message_id):
|
|
|
|
def get_participants(self):
|
|
return unpack_correl_objs_id('user-account', self.get_correlation('user-account')['user-account'], r_type='dict')
|
|
|
|
def get_nb_participants(self):
|
|
return self.get_nb_correlation('user-account')
|
|
|
|
def get_user_messages(self, user_id):
|
|
return self.get_correlation_iter('user-account', self.subtype, user_id, 'message')
|
|
|
|
# TODO move me to abstract subtype
|
|
class AbstractChatObjects(ABC):
|
|
def __init__(self, type):
|
|
self.type = type
|
|
|
|
def add_subtype(self, subtype):
|
|
r_object.sadd(f'all_{self.type}:subtypes', subtype)
|
|
|
|
def get_subtypes(self):
|
|
return r_object.smembers(f'all_{self.type}:subtypes')
|
|
|
|
def get_nb_ids_by_subtype(self, subtype):
|
|
return r_object.zcard(f'{self.type}_all:{subtype}')
|
|
|
|
def get_ids_by_subtype(self, subtype):
|
|
return r_object.zrange(f'{self.type}_all:{subtype}', 0, -1)
|
|
|
|
def get_all_id_iterator_iter(self, subtype):
|
|
return zscan_iter(r_object, f'{self.type}_all:{subtype}')
|
|
|
|
def get_ids(self):
|
|
pass
|
|
|
|
def search(self):
|
|
pass
|