CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

chg: [chat + user-account] correlations + usernames timeline

dev
Terrtia 2023-08-28 16:29:38 +02:00
parent 843b2d3134
commit b32f110285
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
17 changed files with 279 additions and 173 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
bin/crawlers/Crawler.py
View File

@ -282,7 +282,7 @@ class Crawler(AbstractModule):
title_content = crawlers.extract_title_from_html(entries['html'])
if title_content:
title = Titles.create_title(title_content)
title.add(item.get_date(), item_id)
title.add(item.get_date(), item)
# SCREENSHOT
if self.screenshot:
@ -306,11 +306,11 @@ class Crawler(AbstractModule):
for cookie_name in crawlers.extract_cookies_names_from_har(entries['har']):
print(cookie_name)
cookie = CookiesNames.create(cookie_name)
cookie.add(self.date.replace('/', ''), self.domain.id)
cookie.add(self.date.replace('/', ''), self.domain)
for etag_content in crawlers.extract_etag_from_har(entries['har']):
print(etag_content)
etag = Etags.create(etag_content)
etag.add(self.date.replace('/', ''), self.domain.id)
etag.add(self.date.replace('/', ''), self.domain)
crawlers.extract_hhhash(entries['har'], self.domain.id, self.date.replace('/', ''))
# Next Children

12
bin/importer/feeders/Jabber.py
View File

@ -17,7 +17,7 @@ sys.path.append(os.environ['AIL_BIN'])
##################################
from importer.feeders.Default import DefaultFeeder
from lib.objects.Usernames import Username
from lib import item_basic
from lib.objects.Items import Item
class JabberFeeder(DefaultFeeder):
@ -36,7 +36,7 @@ class JabberFeeder(DefaultFeeder):
self.item_id = f'{item_id}.gz'
return self.item_id
def process_meta(self):
def process_meta(self): # TODO replace me by message
"""
Process JSON meta field.
"""
@ -44,10 +44,12 @@ class JabberFeeder(DefaultFeeder):
# item_basic.add_map_obj_id_item_id(jabber_id, item_id, 'jabber_id') ##############################################
to = str(self.json_data['meta']['jabber:to'])
fr = str(self.json_data['meta']['jabber:from'])
date = item_basic.get_item_date(item_id)
item = Item(self.item_id)
date = item.get_date()
user_to = Username(to, 'jabber')
user_fr = Username(fr, 'jabber')
user_to.add(date, self.item_id)
user_fr.add(date, self.item_id)
user_to.add(date, item)
user_fr.add(date, item)
return None

46
bin/importer/feeders/Telegram.py
View File

@ -21,7 +21,6 @@ from lib.objects.Chats import Chat
from lib.objects import Messages
from lib.objects import UsersAccount
from lib.objects.Usernames import Username
from lib import item_basic
import base64
import io
@ -57,7 +56,7 @@ class TelegramFeeder(DefaultFeeder):
# date = datetime.date.today().strftime("%Y/%m/%d")
chat_id = str(self.json_data['meta']['chat']['id'])
message_id = str(self.json_data['meta']['id'])
self.item_id = Messages.create_obj_id('telegram', chat_id, message_id, timestamp)
self.item_id = Messages.create_obj_id('telegram', chat_id, message_id, timestamp) # TODO rename self.item_id
return self.item_id
def process_meta(self):
@ -68,7 +67,7 @@ class TelegramFeeder(DefaultFeeder):
meta = self.json_data['meta']
mess_id = self.json_data['meta']['id']
if meta.get('reply_to'):
reply_to_id = meta['reply_to']
reply_to_id = meta['reply_to']['id']
else:
reply_to_id = None
@ -76,25 +75,24 @@ class TelegramFeeder(DefaultFeeder):
date = datetime.datetime.fromtimestamp(timestamp)
date = date.strftime('%Y%m%d')
if self.json_data.get('translation'):
translation = self.json_data['translation']
else:
translation = None
decoded = base64.standard_b64decode(self.json_data['data'])
content = gunzip_bytes_obj(decoded)
message = Messages.create(self.item_id, content, translation=translation)
if meta.get('chat'):
chat = Chat(meta['chat']['id'], 'telegram')
if meta['chat'].get('username'): # SAVE USERNAME
chat_username = meta['chat']['username']
if meta['chat'].get('username'):
chat_username = Username(meta['chat']['username'], 'telegram')
chat.update_username_timeline(chat_username.get_global_id(), timestamp)
# Chat---Message
chat.add(date, self.item_id) # TODO modify to accept file objects
# message meta ????? who is the user if two user ????
if self.json_data.get('translation'):
translation = self.json_data['translation']
else:
translation = None
decoded = base64.standard_b64decode(self.json_data['data'])
content = gunzip_bytes_obj(decoded)
Messages.create(self.item_id, content, translation=translation)
chat.add_message(self.item_id, timestamp, mess_id, reply_id=reply_to_id)
chat.add(date)
chat.add_message(message.get_global_id(), timestamp, mess_id, reply_id=reply_to_id)
else:
chat = None
@ -103,7 +101,7 @@ class TelegramFeeder(DefaultFeeder):
user_id = meta['sender']['id']
user_account = UsersAccount.UserAccount(user_id, 'telegram')
# UserAccount---Message
user_account.add(date, self.item_id)
user_account.add(date, obj=message)
# UserAccount---Chat
user_account.add_correlation(chat.type, chat.get_subtype(r_str=True), chat.id)
@ -116,20 +114,22 @@ class TelegramFeeder(DefaultFeeder):
if meta['sender'].get('username'):
username = Username(meta['sender']['username'], 'telegram')
# TODO timeline or/and correlation ????
user_account.add_correlation(username.type, username.get_subtype(r_str=True), username.id)
# TODO Update user_account<--->username timeline
user_account.update_username_timeline(username.get_global_id(), timestamp)
# Username---Message
username.add(date, self.item_id) # TODO ####################################################################
username.add(date) # TODO # correlation message ???
if chat:
# Chat---Username
chat.add_correlation(username.type, username.get_subtype(r_str=True), username.id)
# if chat: # TODO Chat---Username correlation ???
# # Chat---Username
# chat.add_correlation(username.type, username.get_subtype(r_str=True), username.id)
# if meta.get('fwd_from'):
# if meta['fwd_from'].get('post_author') # user first name
# TODO reply threads ????
# message edit ????
return None

8
bin/importer/feeders/Twitter.py
View File

@ -17,7 +17,7 @@ sys.path.append(os.environ['AIL_BIN'])
##################################
from importer.feeders.Default import DefaultFeeder
from lib.objects.Usernames import Username
from lib import item_basic
from lib.objects.Items import Item
class TwitterFeeder(DefaultFeeder):
@ -40,9 +40,9 @@ class TwitterFeeder(DefaultFeeder):
'''
# tweet_id = str(self.json_data['meta']['twitter:tweet_id'])
# item_basic.add_map_obj_id_item_id(tweet_id, item_id, 'twitter_id') ############################################
date = item_basic.get_item_date(self.item_id)
item = Item(self.item_id)
date = item.get_date()
user = str(self.json_data['meta']['twitter:id'])
username = Username(user, 'twitter')
username.add(date, item_id)
username.add(date, item)
return None

16
bin/lib/correlations_engine.py
View File

@ -41,20 +41,22 @@ config_loader = None
##################################
CORRELATION_TYPES_BY_OBJ = {
"chat": ["item", "username"], # item ???
"chat": ["user-account"], # message or direct correlation like cve, bitcoin, ... ???
"cookie-name": ["domain"],
"cryptocurrency": ["domain", "item"],
"cve": ["domain", "item"],
"decoded": ["domain", "item"],
"cryptocurrency": ["domain", "item", "message"],
"cve": ["domain", "item", "message"],
"decoded": ["domain", "item", "message"],
"domain": ["cve", "cookie-name", "cryptocurrency", "decoded", "etag", "favicon", "hhhash", "item", "pgp", "title", "screenshot", "username"],
"etag": ["domain"],
"favicon": ["domain", "item"], # TODO Decoded
"hhhash": ["domain"],
"item": ["chat", "cve", "cryptocurrency", "decoded", "domain", "favicon", "pgp", "screenshot", "title", "username"],
"pgp": ["domain", "item"],
"item": ["cve", "cryptocurrency", "decoded", "domain", "favicon", "pgp", "screenshot", "title", "username"], # chat ???
"message": ["cve", "cryptocurrency", "decoded", "pgp", "user-account"], # chat ??
"pgp": ["domain", "item", "message"],
"screenshot": ["domain", "item"],
"title": ["domain", "item"],
"username": ["chat", "domain", "item"],
"user-account": ["chat", "message"],
"username": ["domain", "item", "message"], # TODO chat-user/account
}
def get_obj_correl_types(obj_type):

6
bin/lib/crawlers.py
View File

@ -342,7 +342,7 @@ def _reprocess_all_hars_cookie_name():
for cookie_name in extract_cookies_names_from_har(get_har_content(har_id)):
print(domain, date, cookie_name)
cookie = CookiesNames.create(cookie_name)
cookie.add(date, domain)
cookie.add(date, Domain(domain))
def extract_etag_from_har(har): # TODO check response url
etags = set()
@ -365,7 +365,7 @@ def _reprocess_all_hars_etag():
for etag_content in extract_etag_from_har(get_har_content(har_id)):
print(domain, date, etag_content)
etag = Etags.create(etag_content)
etag.add(date, domain)
etag.add(date, Domain(domain))
def extract_hhhash_by_id(har_id, domain, date):
return extract_hhhash(get_har_content(har_id), domain, date)
@ -395,7 +395,7 @@ def extract_hhhash(har, domain, date):
# -----
obj = HHHashs.create(hhhash_header, hhhash)
obj.add(date, domain)
obj.add(date, Domain(domain))
hhhashs.add(hhhash)
urls.add(url)

35
bin/lib/objects/Chats.py
View File

@ -18,7 +18,7 @@ from lib.ConfigLoader import ConfigLoader
from lib.objects.abstract_subtype_object import AbstractSubtypeObject, get_all_id
from lib.data_retention_engine import update_obj_date
from lib.objects import ail_objects
from lib import item_basic
from lib.timeline_engine import Timeline
from lib.correlations_engine import get_correlation_by_correl_type
@ -126,6 +126,18 @@ class Chat(AbstractSubtypeObject): # TODO # ID == username ?????
users.add(account[1:])
return users
def _get_timeline_username(self):
return Timeline(self.get_global_id(), 'username')
def get_username(self):
return self._get_timeline_username().get_last_obj_id()
def get_usernames(self):
return self._get_timeline_username().get_objs_ids()
def update_username_timeline(self, username_global_id, timestamp):
self._get_timeline_username().add_timestamp(timestamp, username_global_id)
# def get_last_message_id(self):
#
@ -144,18 +156,21 @@ class Chat(AbstractSubtypeObject): # TODO # ID == username ?????
def get_message_meta(self, obj_global_id, parent=True, mess_datetime=None):
obj = ail_objects.get_obj_from_global_id(obj_global_id)
mess_dict = obj.get_meta(options={'content', 'link', 'parent'})
mess_dict = obj.get_meta(options={'content', 'link', 'parent', 'user-account'})
if mess_dict.get('parent') and parent:
mess_dict['reply_to'] = self.get_message_meta(mess_dict['parent'], parent=False)
mess_dict['username'] = {}
user = obj.get_correlation('username').get('username')
if user:
subtype, user = user.pop().split(':', 1)
mess_dict['username']['type'] = 'telegram'
mess_dict['username']['subtype'] = subtype
mess_dict['username']['id'] = user
if mess_dict.get('user-account'):
user_account = ail_objects.get_obj_from_global_id(mess_dict['user-account'])
mess_dict['user-account'] = {}
mess_dict['user-account']['type'] = user_account.get_type()
mess_dict['user-account']['subtype'] = user_account.get_subtype(r_str=True)
mess_dict['user-account']['id'] = user_account.get_id()
username = user_account.get_username()
if username:
username = ail_objects.get_obj_from_global_id(username).get_default_meta(link=False)
mess_dict['user-account']['username'] = username # TODO get username at the given timestamp ???
else:
mess_dict['username']['id'] = 'UNKNOWN'
mess_dict['user-account']['id'] = 'UNKNOWN'
if not mess_datetime:
obj_mess_id = self._get_message_timestamp(obj_global_id)

19
bin/lib/objects/Messages.py
View File

@ -27,7 +27,7 @@ from flask import url_for
config_loader = ConfigLoader()
r_cache = config_loader.get_redis_conn("Redis_Cache")
r_object = config_loader.get_db_conn("Kvrocks_Objects")
r_content = config_loader.get_db_conn("Kvrocks_Content")
# r_content = config_loader.get_db_conn("Kvrocks_Content")
baseurl = config_loader.get_config_str("Notifications", "ail_domain")
config_loader = None
@ -61,7 +61,7 @@ class Message(AbstractObject):
"""
Returns source/feeder name
"""
l_source = self.id.split('/')[:-4]
l_source = self.id.split('/')[:-2]
return os.path.join(*l_source)
def get_basename(self):
@ -79,7 +79,7 @@ class Message(AbstractObject):
def get_date(self):
timestamp = self.get_timestamp()
return datetime.fromtimestamp(timestamp).strftime('%Y%m%d')
return datetime.fromtimestamp(float(timestamp)).strftime('%Y%m%d')
def get_timestamp(self):
dirs = self.id.split('/')
@ -92,11 +92,16 @@ class Message(AbstractObject):
return message_id
def get_chat_id(self): # TODO optimize -> use me to tag Chat
chat_id = self.get_basename().rsplit('_', 1)[0]
chat_id = self.get_basename().rsplit('_', 1)[0]
# if chat_id.endswith('.gz'):
# chat_id = chat_id[:-3]
return chat_id
def get_user_account(self):
user_account = self.get_correlation('user-account')
if user_account.get('user-account'):
return f'user-account:{user_account["user-account"].pop()}'
# Update value on import
# reply to -> parent ?
# reply/comment - > children ?
@ -139,7 +144,7 @@ class Message(AbstractObject):
return url
def get_svg_icon(self):
return {'style': 'fas', 'icon': 'fa-comment-dots', 'color': '#4dffff', 'radius': 5}
return {'style': 'fas', 'icon': '\uf4ad', 'color': '#4dffff', 'radius': 5}
def get_misp_object(self): # TODO
obj = MISPObject('instant-message', standalone=True)
@ -181,6 +186,8 @@ class Message(AbstractObject):
meta['investigations'] = self.get_investigations()
if 'link' in options:
meta['link'] = self.get_link(flask_context=True)
if 'user-account' in options:
meta['user-account'] = self.get_user_account()
# meta['encoding'] = None
return meta
@ -238,7 +245,7 @@ class Message(AbstractObject):
def create(self, content, translation, tags):
self._set_field('content', content)
r_content.get(f'content:{self.type}:{self.get_subtype(r_str=True)}:{self.id}', content)
# r_content.get(f'content:{self.type}:{self.get_subtype(r_str=True)}:{self.id}', content)
if translation:
self._set_translation(translation)
for tag in tags:

15
bin/lib/objects/UsersAccount.py
View File

@ -3,7 +3,7 @@
import os
import sys
import re
# import re
from flask import url_for
from pymisp import MISPObject
@ -15,6 +15,7 @@ sys.path.append(os.environ['AIL_BIN'])
from lib import ail_core
from lib.ConfigLoader import ConfigLoader
from lib.objects.abstract_subtype_object import AbstractSubtypeObject, get_all_id
from lib.timeline_engine import Timeline
config_loader = ConfigLoader()
baseurl = config_loader.get_config_str("Notifications", "ail_domain")
@ -80,15 +81,17 @@ class UserAccount(AbstractSubtypeObject):
def set_phone(self, phone):
return self._set_field('phone', phone)
# TODO REWRITE ADD FUNCTION
def _get_timeline_username(self):
return Timeline(self.get_global_id(), 'username')
def get_username(self):
return ''
return self._get_timeline_username().get_last_obj_id()
def get_usernames(self):
usernames = []
# TODO TIMELINE
return usernames
return self._get_timeline_username().get_objs_ids()
def update_username_timeline(self, username_global_id, timestamp):
self._get_timeline_username().add_timestamp(timestamp, username_global_id)
def get_meta(self, options=set()):
meta = self._get_meta(options=options)

4
bin/lib/objects/abstract_object.py
View File

@ -65,12 +65,14 @@ class AbstractObject(ABC):
def get_global_id(self):
return f'{self.get_type()}:{self.get_subtype(r_str=True)}:{self.get_id()}'
def get_default_meta(self, tags=False):
def get_default_meta(self, tags=False, link=False):
dict_meta = {'id': self.get_id(),
'type': self.get_type(),
'subtype': self.get_subtype(r_str=True)}
if tags:
dict_meta['tags'] = self.get_tags()
if link:
dict_meta['link'] = self.get_link()
return dict_meta
def _get_field(self, field):

18
bin/lib/objects/abstract_subtype_object.py
View File

@ -151,7 +151,7 @@ class AbstractSubtypeObject(AbstractObject, ABC):
#
#
def add(self, date, item_id):
def add(self, date, obj=None):
self.update_daterange(date)
update_obj_date(date, self.type, self.subtype)
# daily
@ -162,20 +162,22 @@ class AbstractSubtypeObject(AbstractObject, ABC):
#######################################################################
#######################################################################
# Correlations
self.add_correlation('item', '', item_id)
# domain
if is_crawled(item_id):
domain = get_item_domain(item_id)
self.add_correlation('domain', '', domain)
if obj:
# Correlations
self.add_correlation(obj.type, obj.get_subtype(r_str=True), obj.get_id())
if obj.type == 'item': # TODO same for message->chat ???
item_id = obj.get_id()
# domain
if is_crawled(item_id):
domain = get_item_domain(item_id)
self.add_correlation('domain', '', domain)
# TODO:ADD objects + Stats
def create(self, first_seen, last_seen):
self.set_first_seen(first_seen)
self.set_last_seen(last_seen)
def _delete(self):
pass

6
bin/lib/objects/ail_objects.py
View File

@ -23,9 +23,11 @@ from lib.objects import Etags
from lib.objects.Favicons import Favicon
from lib.objects import HHHashs
from lib.objects.Items import Item, get_all_items_objects, get_nb_items_objects
from lib.objects.Messages import Message
from lib.objects import Pgps
from lib.objects.Screenshots import Screenshot
from lib.objects import Titles
from lib.objects.UsersAccount import UserAccount
from lib.objects import Usernames
config_loader = ConfigLoader()
@ -68,6 +70,8 @@ def get_object(obj_type, subtype, id):
return Favicon(id)
elif obj_type == 'hhhash':
return HHHashs.HHHash(id)
elif obj_type == 'message':
return Message(id)
elif obj_type == 'screenshot':
return Screenshot(id)
elif obj_type == 'cryptocurrency':
@ -76,6 +80,8 @@ def get_object(obj_type, subtype, id):
return Pgps.Pgp(id, subtype)
elif obj_type == 'title':
return Titles.Title(id)
elif obj_type == 'user-account':
return UserAccount(id, subtype)
elif obj_type == 'username':
return Usernames.Username(id, subtype)

231
bin/lib/timeline_engine.py
View File

@ -46,112 +46,167 @@ config_loader = None
# return []
# return correl_types
# TODO rename all function + add missing parameters
class Timeline:
def get_bloc_obj_global_id(bloc):
return r_meta.hget('hset:key', bloc)
def __init__(self, global_id, name):
self.id = global_id
self.name = name
def set_bloc_obj_global_id(bloc, global_id):
return r_meta.hset('hset:key', bloc, global_id)
def _get_block_obj_global_id(self, block):
return r_meta.hget(f'block:{self.id}:{self.name}', block)
def get_bloc_timestamp(bloc, position):
return r_meta.zscore('key', f'{position}:{bloc}')
def _set_block_obj_global_id(self, block, global_id):
return r_meta.hset(f'block:{self.id}:{self.name}', block, global_id)
def add_bloc(global_id, timestamp, end=None):
if end:
timestamp_end = end
else:
timestamp_end = timestamp
new_bloc = str(uuid4())
r_meta.zadd('key', {f'start:{new_bloc}': timestamp, f'end:{new_bloc}': timestamp_end})
set_bloc_obj_global_id(new_bloc, global_id)
return new_bloc
def _get_block_timestamp(self, block, position):
return r_meta.zscore(f'line:{self.id}:{self.name}', f'{position}:{block}')
def _update_bloc(bloc, position, timestamp):
r_meta.zadd('key', {f'{position}:{bloc}': timestamp})
def _get_nearest_bloc_inf(self, timestamp):
inf = r_meta.zrevrangebyscore(f'line:{self.id}:{self.name}', float(timestamp), 0, start=0, num=1, withscores=True)
if inf:
inf, score = inf[0]
if inf.startswith('end'):
inf_key = f'start:{inf[4:]}'
inf_score = r_meta.zscore(f'line:{self.id}:{self.name}', inf_key)
if inf_score == score:
inf = inf_key
return inf
else:
return None
# score = timestamp
def get_nearest_bloc_inf(timestamp):
return r_meta.zrevrangebyscore('key', timestamp, 0, num=1)
def _get_nearest_bloc_sup(self, timestamp):
sup = r_meta.zrangebyscore(f'line:{self.id}:{self.name}', float(timestamp), '+inf', start=0, num=1, withscores=True)
if sup:
sup, score = sup[0]
if sup.startswith('start'):
sup_key = f'end:{sup[6:]}'
sup_score = r_meta.zscore(f'line:{self.id}:{self.name}', sup_key)
if score == sup_score:
sup = sup_key
return sup
else:
return None
def get_nearest_bloc_sup(timestamp):
return r_meta.zrangebyscore('key', timestamp, 0, num=1)
def get_first_obj_id(self):
first = r_meta.zrange(f'line:{self.id}:{self.name}', 0, 0)
if first: # start:block
first = first[0]
if first.startswith('start:'):
first = first[6:]
else:
first = first[4:]
return self._get_block_obj_global_id(first)
#######################################################################################
def get_last_obj_id(self):
last = r_meta.zrevrange(f'line:{self.id}:{self.name}', 0, 0)
if last: # end:block
last = last[0]
if last.startswith('end:'):
last = last[4:]
else:
last = last[6:]
return self._get_block_obj_global_id(last)
def add_timestamp(timestamp, obj_global_id):
inf = get_nearest_bloc_inf(timestamp)
sup = get_nearest_bloc_sup(timestamp)
if not inf and not sup:
# create new bloc
new_bloc = add_bloc(obj_global_id, timestamp)
def get_objs_ids(self):
objs = set()
for block in r_meta.zrange(f'line:{self.id}:{self.name}', 0, -1):
if block:
if block.startswith('start:'):
objs.add(self._get_block_obj_global_id(block[6:]))
return objs
# def get_objs_ids(self):
# objs = {}
# last_obj_id = None
# for block, timestamp in r_meta.zrange(f'line:{self.id}:{self.name}', 0, -1, withscores=True):
# if block:
# if block.startswith('start:'):
# last_obj_id = self._get_block_obj_global_id(block[6:])
# objs[last_obj_id] = {'first_seen': timestamp}
# else:
# objs[last_obj_id]['last_seen'] = timestamp
# return objs
def _update_bloc(self, block, position, timestamp):
r_meta.zadd(f'line:{self.id}:{self.name}', {f'{position}:{block}': timestamp})
def _add_bloc(self, obj_global_id, timestamp, end=None):
if end:
timestamp_end = end
else:
timestamp_end = timestamp
new_bloc = str(uuid4())
r_meta.zadd(f'line:{self.id}:{self.name}', {f'start:{new_bloc}': timestamp, f'end:{new_bloc}': timestamp_end})
self._set_block_obj_global_id(new_bloc, obj_global_id)
return new_bloc
# timestamp < first_seen
elif not inf:
sup_pos, sup_id = inf.split(':')
sup_obj = get_bloc_obj_global_id(sup_pos)
if sup_obj == obj_global_id:
_update_bloc(sup_id, 'start', timestamp)
# create new bloc
else:
new_bloc = add_bloc(obj_global_id, timestamp)
def add_timestamp(self, timestamp, obj_global_id):
inf = self._get_nearest_bloc_inf(timestamp)
sup = self._get_nearest_bloc_sup(timestamp)
if not inf and not sup:
# create new bloc
new_bloc = self._add_bloc(obj_global_id, timestamp)
return new_bloc
# timestamp > first_seen
elif not sup:
inf_pos, inf_id = inf.split(':')
inf_obj = get_bloc_obj_global_id(inf_id)
if inf_obj == obj_global_id:
_update_bloc(inf_id, 'end', timestamp)
# create new bloc
else:
new_bloc = add_bloc(obj_global_id, timestamp)
return new_bloc
else:
inf_pos, inf_id = inf.split(':')
sup_pos, sup_id = inf.split(':')
inf_obj = get_bloc_obj_global_id(inf_id)
if inf_id == sup_id:
# reduce bloc + create two new bloc
if obj_global_id != inf_obj:
# get end timestamp
sup_timestamp = get_bloc_timestamp(sup_id, 'end')
# reduce original bloc
_update_bloc(inf_id, 'end', timestamp - 1)
# Insert new bloc
new_bloc = add_bloc(obj_global_id, timestamp)
# Recreate end of the first bloc by a new bloc
add_bloc(inf_obj, timestamp + 1, end=sup_timestamp)
# timestamp < first_seen
elif not inf:
sup_pos, sup_id = sup.split(':')
sup_obj = self._get_block_obj_global_id(sup_id)
if sup_obj == obj_global_id:
self._update_bloc(sup_id, 'start', timestamp)
# create new bloc
else:
new_bloc = self._add_bloc(obj_global_id, timestamp)
return new_bloc
# timestamp in existing bloc
else:
return inf_id
# different blocs: expend sup/inf bloc or create a new bloc if
elif inf_pos == 'end' and sup_pos == 'start':
# Extend inf bloc
if obj_global_id == inf_obj:
_update_bloc(inf_id, 'end', timestamp)
return inf_id
sup_obj = get_bloc_obj_global_id(sup_pos)
# Extend sup bloc
if obj_global_id == sup_obj:
_update_bloc(sup_id, 'start', timestamp)
return sup_id
# timestamp > first_seen
elif not sup:
inf_pos, inf_id = inf.split(':')
inf_obj = self._get_block_obj_global_id(inf_id)
if inf_obj == obj_global_id:
self._update_bloc(inf_id, 'end', timestamp)
# create new bloc
new_bloc = add_bloc(obj_global_id, timestamp)
return new_bloc
else:
new_bloc = self._add_bloc(obj_global_id, timestamp)
return new_bloc
# inf_pos == 'start' and sup_pos == 'end'
# else raise error ???
else:
inf_pos, inf_id = inf.split(':')
sup_pos, sup_id = sup.split(':')
inf_obj = self._get_block_obj_global_id(inf_id)
if inf_id == sup_id:
# reduce bloc + create two new bloc
if obj_global_id != inf_obj:
# get end timestamp
sup_timestamp = self._get_block_timestamp(sup_id, 'end')
# reduce original bloc
self._update_bloc(inf_id, 'end', timestamp - 1)
# Insert new bloc
new_bloc = self._add_bloc(obj_global_id, timestamp)
# Recreate end of the first bloc by a new bloc
self._add_bloc(inf_obj, timestamp + 1, end=sup_timestamp)
return new_bloc
# timestamp in existing bloc
else:
return inf_id
# different blocs: expend sup/inf bloc or create a new bloc if
elif inf_pos == 'end' and sup_pos == 'start':
# Extend inf bloc
if obj_global_id == inf_obj:
self._update_bloc(inf_id, 'end', timestamp)
return inf_id
sup_obj = self._get_block_obj_global_id(sup_id)
# Extend sup bloc
if obj_global_id == sup_obj:
self._update_bloc(sup_id, 'start', timestamp)
return sup_id
# create new bloc
new_bloc = self._add_bloc(obj_global_id, timestamp)
return new_bloc
# inf_pos == 'start' and sup_pos == 'end'
# else raise error ???

2
bin/modules/Cryptocurrencies.py
View File

@ -130,7 +130,7 @@ class Cryptocurrencies(AbstractModule, ABC):
if crypto.is_valid_address():
# print(address)
is_valid_address = True
crypto.add(date, item_id)
crypto.add(date, item)
# Check private key
if is_valid_address:

6
bin/modules/PgpDump.py
View File

@ -210,18 +210,18 @@ class PgpDump(AbstractModule):
date = item.get_date()
for key in self.keys:
pgp = Pgps.Pgp(key, 'key')
pgp.add(date, self.item_id)
pgp.add(date, item)
print(f' key: {key}')
for name in self.names:
pgp = Pgps.Pgp(name, 'name')
pgp.add(date, self.item_id)
pgp.add(date, item)
print(f' name: {name}')
self.tracker_term.compute(name, obj_type='pgp', subtype='name')
self.tracker_regex.compute(name, obj_type='pgp', subtype='name')
self.tracker_yara.compute(name, obj_type='pgp', subtype='name')
for mail in self.mails:
pgp = Pgps.Pgp(mail, 'mail')
pgp.add(date, self.item_id)
pgp.add(date, item)
print(f' mail: {mail}')
self.tracker_term.compute(mail, obj_type='pgp', subtype='mail')
self.tracker_regex.compute(mail, obj_type='pgp', subtype='mail')

4
bin/modules/Telegram.py
View File

@ -58,7 +58,7 @@ class Telegram(AbstractModule):
user_id = dict_url.get('username')
if user_id:
username = Username(user_id, 'telegram')
username.add(item_date, item.id)
username.add(item_date, item)
print(f'username: {user_id}')
invite_hash = dict_url.get('invite_hash')
if invite_hash:
@ -73,7 +73,7 @@ class Telegram(AbstractModule):
user_id = dict_url.get('username')
if user_id:
username = Username(user_id, 'telegram')
username.add(item_date, item.id)
username.add(item_date, item)
print(f'username: {user_id}')
invite_hash = dict_url.get('invite_hash')
if invite_hash:

18
var/www/templates/objects/chat/ChatMessages.html
View File

@ -120,14 +120,26 @@
<div class="chat-message-left pb-0">
<div>
<img src="{{ url_for('static', filename='image/ail-icon.png') }}" class="rounded-circle mr-1" alt="{{ mess['username']['id'] }}" width="40" height="40">
<img src="{{ url_for('static', filename='image/ail-icon.png') }}" class="rounded-circle mr-1" alt="{{ mess['user-account']['id'] }}" width="40" height="40">
<div class="text-muted small text-nowrap mt-2">{{ mess['hour'] }}</div>
</div>
<div class="flex-shrink-1 bg-light rounded py-2 px-3 ml-4 pb-4" style="overflow-x: auto">
<div class="font-weight-bold mb-1">{{ mess['username']['id'] }}</div>
<div class="font-weight-bold mb-1">
{% if mess['user-account']['username'] %}
{{ mess['user-account']['username']['id'] }}
{% else %}
{{ mess['user-account']['id'] }}
{% endif %}
</div>
{% if mess['reply_to'] %}
<div class="flex-shrink-1 border rounded py-2 px-3 ml-4 mb-3" style="overflow-x: auto">
<div class="font-weight-bold mb-1">{{ mess['reply_to']['username']['id'] }}</div>
<div class="font-weight-bold mb-1">
{% if mess['reply_to']['user-account']['username'] %}
{{ mess['reply_to']['user-account']['username']['id'] }}
{% else %}
{{ mess['reply_to']['user-account']['id'] }}
{% endif %}
</div>
<pre class="my-0">{{ mess['reply_to']['content'] }}</pre>
{% for tag in mess['reply_to']['tags'] %}
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span>