CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

fix: [user-account] fix last username timestamp

relationships
terrtia 2024-06-06 15:05:40 +02:00
parent 883563be56
commit 424229e28a
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
2 changed files with 16 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
bin/importer/feeders/Default.py
View File

@ -44,6 +44,11 @@ class DefaultFeeder:
def get_date(self): def get_date(self):
return datetime.date.today().strftime("%Y%m%d") return datetime.date.today().strftime("%Y%m%d")
def get_feeder_timestamp(self):
timestamp = self.json_data.get('timestamp')
if timestamp:
return int(timestamp)
def get_json_data(self): def get_json_data(self):
""" """
Return the JSON data, Return the JSON data,

20
bin/importer/feeders/abstract_chats_feeder.py
View File

@ -198,7 +198,7 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
############################################################################################################################## ##############################################################################################################################
def process_chat(self, new_objs, obj, date, timestamp, reply_id=None): def process_chat(self, new_objs, obj, date, timestamp, feeder_timestamp, reply_id=None):
meta = self.json_data['meta']['chat'] # todo replace me by function meta = self.json_data['meta']['chat'] # todo replace me by function
chat = Chat(self.get_chat_id(), self.get_chat_instance_uuid()) chat = Chat(self.get_chat_id(), self.get_chat_instance_uuid())
subchannel = None subchannel = None
@ -224,7 +224,7 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
if meta.get('username'): if meta.get('username'):
username = Username(meta['username'], self.get_chat_protocol()) username = Username(meta['username'], self.get_chat_protocol())
chat.update_username_timeline(username.get_global_id(), timestamp) chat.update_username_timeline(username.get_global_id(), feeder_timestamp)
if meta.get('subchannel'): if meta.get('subchannel'):
subchannel, thread = self.process_subchannel(obj, date, timestamp, reply_id=reply_id) subchannel, thread = self.process_subchannel(obj, date, timestamp, reply_id=reply_id)
@ -293,13 +293,13 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
########################################################################################## ##########################################################################################
def _process_user(self, meta, date, timestamp, new_objs=None): #################33 timestamp def _process_user(self, meta, date, timestamp, new_objs=None):
user_account = UsersAccount.UserAccount(meta['id'], self.get_chat_instance_uuid()) user_account = UsersAccount.UserAccount(meta['id'], self.get_chat_instance_uuid())
if meta.get('username'): if meta.get('username'):
username = Username(meta['username'], self.get_chat_protocol()) username = Username(meta['username'], self.get_chat_protocol())
# TODO timeline or/and correlation ???? # TODO timeline or/and correlation ????
user_account.add_correlation(username.type, username.get_subtype(r_str=True), username.id) user_account.add_correlation(username.type, username.get_subtype(r_str=True), username.id)
user_account.update_username_timeline(username.get_global_id(), timestamp) # TODO time.time !!!! (time when meta are retrieved) user_account.update_username_timeline(username.get_global_id(), timestamp)
# Username---Message # Username---Message
username.add(date) # TODO # correlation message ??? ############################################################### username.add(date) # TODO # correlation message ??? ###############################################################
@ -375,6 +375,9 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
chats_objs = set() chats_objs = set()
date, timestamp = self.get_message_date_timestamp() date, timestamp = self.get_message_date_timestamp()
feeder_timestamp = self.get_feeder_timestamp()
if not feeder_timestamp:
feeder_timestamp = timestamp
# REPLY # REPLY
reply_id = self.get_message_reply_id() reply_id = self.get_message_reply_id()
@ -431,10 +434,10 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
print(obj.id) print(obj.id)
# CHAT # CHAT
curr_chats_objs = self.process_chat(new_objs, obj, date, timestamp, reply_id=reply_id) curr_chats_objs = self.process_chat(new_objs, obj, date, timestamp, feeder_timestamp, reply_id=reply_id)
# SENDER # TODO HANDLE NULL SENDER # SENDER # TODO HANDLE NULL SENDER
user_account = self.process_sender(new_objs, obj, date, timestamp) user_account = self.process_sender(new_objs, obj, date, feeder_timestamp)
if user_account: if user_account:
# UserAccount---ChatObjects # UserAccount---ChatObjects
@ -465,7 +468,7 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
if chat_obj.type == 'chat': if chat_obj.type == 'chat':
chat_fwd.add_relationship(chat_obj.get_global_id(), 'forwarded_to') chat_fwd.add_relationship(chat_obj.get_global_id(), 'forwarded_to')
if meta_fwd.get('user'): if meta_fwd.get('user'):
user_fwd = self._process_user(meta_fwd['user'], date, timestamp, new_objs=new_objs) # TODO date, timestamp ??? user_fwd = self._process_user(meta_fwd['user'], date, feeder_timestamp, new_objs=new_objs) # TODO date, timestamp ???
for chat_obj in chats_objs: for chat_obj in chats_objs:
if chat_obj.type == 'chat': if chat_obj.type == 'chat':
user_fwd.add_relationship(chat_obj.get_global_id(), 'forwarded_to') user_fwd.add_relationship(chat_obj.get_global_id(), 'forwarded_to')
@ -502,7 +505,7 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
obj.add_relationship(chat_obj.get_global_id(), 'in') obj.add_relationship(chat_obj.get_global_id(), 'in')
for mention in self.get_json_meta()['mentions'].get('users', []): for mention in self.get_json_meta()['mentions'].get('users', []):
m_obj = self._process_user(mention, date, timestamp, new_objs=new_objs) # TODO date, timestamp ??? m_obj = self._process_user(mention, date, feeder_timestamp, new_objs=new_objs) # TODO date ???
if m_obj: if m_obj:
for chat_obj in chats_objs: for chat_obj in chats_objs:
if chat_obj.type == 'chat': if chat_obj.type == 'chat':
@ -517,7 +520,6 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
if chat_obj.type == 'chat': if chat_obj.type == 'chat':
obj.add_relationship(chat_obj.get_global_id(), 'in') obj.add_relationship(chat_obj.get_global_id(), 'in')
# -MENTION- # # -MENTION- #
return new_objs | objs return new_objs | objs