CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

chg: [chats] factorise heatmap + chat icon

dev
terrtia 2023-11-24 15:05:19 +01:00
parent 9fbd3f4bb6
commit 2b8e9b43f3
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
16 changed files with 103 additions and 246 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
bin/importer/FeederImporter.py
View File

@ -89,17 +89,26 @@ class FeederImporter(AbstractImporter):
feeder_name = feeder.get_name()
print(f'importing: {feeder_name} feeder')
obj = feeder.get_obj() # TODO replace by a list of objects to import ????
# Get Data object:
data_obj = feeder.get_obj()
# process meta
if feeder.get_json_meta():
feeder.process_meta()
objs = feeder.process_meta()
if objs is None:
objs = set()
else:
objs = set()
if obj.type == 'item': # object save on disk as file (Items)
gzip64_content = feeder.get_gzip64_content()
return obj, f'{feeder_name} {gzip64_content}'
else: # Messages save on DB
if obj.exists():
return obj, f'{feeder_name}'
objs.add(data_obj)
for obj in objs:
if obj.type == 'item': # object save on disk as file (Items)
gzip64_content = feeder.get_gzip64_content()
return obj, f'{feeder_name} {gzip64_content}'
else: # Messages save on DB
if obj.exists():
return obj, f'{feeder_name}'
class FeederModuleImporter(AbstractModule):

1
bin/importer/feeders/BgpMonitor.py
View File

@ -33,3 +33,4 @@ class BgpMonitorFeeder(DefaultFeeder):
tag = 'infoleak:automatic-detection=bgp_monitor'
item = Item(self.get_item_id())
item.add_tag(tag)
return set()

2
bin/importer/feeders/Default.py
View File

@ -84,4 +84,4 @@ class DefaultFeeder:
Process JSON meta filed.
"""
# meta = self.get_json_meta()
pass
return set()

38
bin/importer/feeders/Discord.py Executable file
View File

@ -0,0 +1,38 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
"""
The Telegram Feeder Importer Module
================
Process Telegram JSON
"""
import os
import sys
import datetime
sys.path.append(os.environ['AIL_BIN'])
##################################
# Import Project packages
##################################
from importer.feeders.abstract_chats_feeder import AbstractChatFeeder
from lib.ConfigLoader import ConfigLoader
from lib.objects import ail_objects
from lib.objects.Chats import Chat
from lib.objects import Messages
from lib.objects import UsersAccount
from lib.objects.Usernames import Username
import base64
class DiscordFeeder(AbstractChatFeeder):
def __init__(self, json_data):
super().__init__('discord', json_data)
# def get_obj(self):.
# obj_id = Messages.create_obj_id('telegram', chat_id, message_id, timestamp)
# obj_id = f'message:telegram:{obj_id}'
# self.obj = ail_objects.get_obj_from_global_id(obj_id)
# return self.obj

2
bin/importer/feeders/Jabber.py
View File

@ -52,4 +52,4 @@ class JabberFeeder(DefaultFeeder):
user_fr = Username(fr, 'jabber')
user_to.add(date, item)
user_fr.add(date, item)
return None
return set()

2
bin/importer/feeders/Twitter.py
View File

@ -45,4 +45,4 @@ class TwitterFeeder(DefaultFeeder):
user = str(self.json_data['meta']['twitter:id'])
username = Username(user, 'twitter')
username.add(date, item)
return None
return set()

2
bin/importer/feeders/Urlextract.py
View File

@ -56,3 +56,5 @@ class UrlextractFeeder(DefaultFeeder):
item = Item(self.item_id)
item.set_parent(parent_id)
return set()

13
bin/importer/feeders/abstract_chats_feeder.py
View File

@ -131,7 +131,7 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
self.obj = Messages.Message(obj_id)
return self.obj
def process_chat(self, obj, date, timestamp, reply_id=None): # TODO threads
def process_chat(self, new_objs, obj, date, timestamp, reply_id=None): # TODO threads
meta = self.json_data['meta']['chat'] # todo replace me by function
chat = Chat(self.get_chat_id(), self.get_chat_instance_uuid())
@ -147,6 +147,12 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
if meta.get('date'): # TODO check if already exists
chat.set_created_at(int(meta['date']['timestamp']))
if meta.get('icon'):
img = Images.create(meta['icon'], b64=True)
img.add(date, chat)
chat.set_icon(img.get_global_id())
new_objs.add(img)
if meta.get('username'):
username = Username(meta['username'], self.get_chat_protocol())
chat.update_username_timeline(username.get_global_id(), timestamp)
@ -228,6 +234,7 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
objs = set()
if self.obj:
objs.add(self.obj)
new_objs = set()
date, timestamp = self.get_message_date_timestamp()
@ -261,7 +268,7 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
for obj in objs: # TODO PERF avoid parsing metas multiple times
# CHAT
chat = self.process_chat(obj, date, timestamp, reply_id=reply_id)
chat = self.process_chat(new_objs, obj, date, timestamp, reply_id=reply_id)
# SENDER # TODO HANDLE NULL SENDER
user_account = self.process_sender(obj, date, timestamp)
@ -279,6 +286,8 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
# -> subchannel ?
# -> thread id ?
return new_objs | objs

6
bin/lib/chats_viewer.py
View File

@ -175,7 +175,7 @@ class ChatServiceInstance:
if 'chats' in options:
meta['chats'] = []
for chat_id in self.get_chats():
meta['chats'].append(Chats.Chat(chat_id, self.uuid).get_meta({'created_at', 'nb_subchannels'}))
meta['chats'].append(Chats.Chat(chat_id, self.uuid).get_meta({'created_at', 'icon', 'nb_subchannels'}))
return meta
def get_nb_chats(self):
@ -304,7 +304,7 @@ def api_get_chat(chat_id, chat_instance_uuid):
chat = Chats.Chat(chat_id, chat_instance_uuid)
if not chat.exists():
return {"status": "error", "reason": "Unknown chat"}, 404
meta = chat.get_meta({'created_at', 'img', 'info', 'subchannels', 'username'})
meta = chat.get_meta({'created_at', 'icon', 'info', 'subchannels', 'username'})
if meta['username']:
meta['username'] = get_username_meta_from_global_id(meta['username'])
if meta['subchannels']:
@ -325,7 +325,7 @@ def api_get_subchannel(chat_id, chat_instance_uuid):
subchannel = ChatSubChannels.ChatSubChannel(chat_id, chat_instance_uuid)
if not subchannel.exists():
return {"status": "error", "reason": "Unknown chat"}, 404
meta = subchannel.get_meta({'chat', 'created_at', 'img', 'nb_messages'})
meta = subchannel.get_meta({'chat', 'created_at', 'icon', 'nb_messages'})
if meta['chat']:
meta['chat'] = get_chat_meta_from_global_id(meta['chat'])
if meta.get('username'):

4
bin/lib/objects/Chats.py
View File

@ -74,8 +74,8 @@ class Chat(AbstractChatObject):
meta = self._get_meta(options=options)
meta['name'] = self.get_name()
meta['tags'] = self.get_tags(r_list=True)
if 'img':
meta['icon'] = self.get_img()
if 'icon':
meta['icon'] = self.get_icon()
if 'info':
meta['info'] = self.get_info()
if 'username' in options:

12
bin/lib/objects/abstract_chat_object.py
View File

@ -113,11 +113,13 @@ class AbstractChatObject(AbstractSubtypeObject, ABC):
def set_name(self, name):
self._set_field('name', name)
def get_img(self):
return self._get_field('img')
def get_icon(self):
icon = self._get_field('icon')
if icon:
return icon.rsplit(':', 1)[1]
def set_img(self, icon):
self._set_field('img', icon)
def set_icon(self, icon):
self._set_field('icon', icon)
def get_info(self):
return self._get_field('info')
@ -187,7 +189,7 @@ class AbstractChatObject(AbstractSubtypeObject, ABC):
tags = {}
messages = {}
curr_date = None
for message in self._get_messages(nb=10, page=3):
for message in self._get_messages(nb=30, page=1):
timestamp = message[1]
date_day = datetime.fromtimestamp(timestamp).strftime('%Y/%m/%d')
if date_day != curr_date:

2
bin/modules/Exif.py
View File

@ -42,6 +42,8 @@ class Exif(AbstractModule):
img_exif = img.getexif()
print(img_exif)
if img_exif:
gps = img_exif.get(34853)
print(gps)
for key, val in img_exif.items():
if key in ExifTags.TAGS:
print(f'{ExifTags.TAGS[key]}:{val}')

2
var/www/blueprints/objects_image.py
View File

@ -40,7 +40,7 @@ def image(filename):
abort(404)
filename = filename.replace('/', '')
image = Images.Image(filename)
return send_from_directory(Images.IMAGE_FOLDER, image.get_rel_path(), as_attachment=True)
return send_from_directory(Images.IMAGE_FOLDER, image.get_rel_path(), as_attachment=False, mimetype='image')
@objects_image.route("/objects/images", methods=['GET'])

4
var/www/templates/chats_explorer/block_message.html
View File

@ -58,7 +58,9 @@
</div>
{% endif %}
{% if message['images'] %}
<img class="message_image mb-1" src="{{ url_for('objects_image.image', filename=message['images'][0])}}">
{% for message_image in message['images'] %}
<img class="message_image mb-1" src="{{ url_for('objects_image.image', filename=message_image)}}">
{% endfor %}
{% endif %}
<pre class="my-0">{{ message['content'] }}</pre>
{% for tag in message['tags'] %}

3
var/www/templates/chats_explorer/chat_instance.html
View File

@ -77,7 +77,8 @@
{% for chat in chat_instance["chats"] %}
<tr>
<td>
<img src="{{ url_for('static', filename='image/ail-icon.png') }}" class="rounded-circle mr-1" alt="{{ chat['id'] }}" width="40" height="40">
<img src="{% if chat['icon'] %}{{ url_for('objects_image.image', filename=chat['icon'])}}{% else %}{{ url_for('static', filename='image/ail-icon.png') }}{% endif %}"
class="rounded-circle mr-1" alt="{{ chat['id'] }}" width="40" height="40">
</td>
<td><b>{{ chat['name'] }}</b></td>
<td><a href="{{ url_for('chats_explorer.chats_explorer_chat') }}?uuid={{ chat_instance['uuid'] }}&id={{ chat['id'] }}">{{ chat['id'] }}</a></td>

231
var/www/templates/chats_explorer/chat_viewer.html
View File

@ -17,6 +17,7 @@
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/d3.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/d3/heatmap_week_hour.js')}}"></script>
<style>
.chat-message-left,
@ -56,7 +57,10 @@
<div class="card my-3">
<div class="card-header" style="background-color:#d9edf7;font-size: 15px">
<h4 class="text-secondary">{% if chat['username'] %}{{ chat["username"]["id"] }} {% else %} {{ chat['name'] }}{% endif %} :</h4> {{ chat["id"] }}
<h4 class="text-secondary">{% if chat['username'] %}{{ chat["username"]["id"] }} {% else %} {{ chat['name'] }}{% endif %} :</h4>
{% if chat['icon'] %}
<div><img src="{{ url_for('objects_image.image', filename=chat['icon'])}}" class="mb-2" alt="{{ chat['id'] }}" width="200" height="200"></div>
{% endif %}
<ul class="list-group mb-2">
<li class="list-group-item py-0">
<table class="table">
@ -218,6 +222,12 @@ function toggle_sidebar(){
<script>
d3.json("{{ url_for('chats_explorer.chats_explorer_messages_stats_week') }}?uuid={{ chat['subtype'] }}&id={{ chat['id'] }}")
.then(function(data) {
create_heatmap_week_hour('#heatmapweekhour', data);
})
/*
// set the dimensions and margins of the graph
@ -323,225 +333,6 @@ d3.csv("").then( function(data) {
*/
</script>
<script>
// based on gist nbremer/62cf60e116ae821c06602793d265eaf6
d3.json("{{ url_for('chats_explorer.chats_explorer_messages_stats_week') }}?uuid={{ chat['subtype'] }}&id={{ chat['id'] }}")
.then(function(data) {
var days = ["Mo", "Tu", "We", "Th", "Fr", "Sa", "Su"],
times = d3.range(24);
var margin = {
top: 80,
right: 50,
bottom: 20,
left: 50
};
var width = Math.max(Math.min(window.innerWidth, 1000), 500) - margin.left - margin.right - 20,
gridSize = Math.floor(width / times.length),
height = gridSize * (days.length + 2);
var heatmap_font_size = width * 62.5 / 900;
//SVG container
var svg = d3.select('#heatmapweekhour')
.append("svg")
.attr("width", width + margin.left + margin.right)
.attr("height", height + margin.top + margin.bottom)
.append("g")
.attr("transform", "translate(" + margin.left + "," + margin.top + ")");
// create a tooltip
const tooltip = d3.select("#heatmapweekhour")
.append("div")
.style("opacity", 0)
.attr("class", "tooltip")
.style("background-color", "white")
.style("border", "solid")
.style("border-width", "2px")
.style("border-radius", "5px")
.style("padding", "5px")
// Three function that change the tooltip when user hover / move / leave a cell
const mouseover = function(d) {
tooltip.style("opacity", 1)
d3.select(this)
.style("stroke", "black")
//.style("stroke-opacity", 1)
tooltip.html(d.date + " " + d.hour + "-" + (d.hour + 1) + "h: <b>" + d.count + "</b> messages")
}
const mouseleave = function(d) {
tooltip.style("opacity", 0)
d3.select(this)
.style("stroke", "white")
//.style("stroke-opacity", 0.8)
}
///////////////////////////////////////////////////////////////////////////
//////////////////////////// Draw Heatmap /////////////////////////////////
///////////////////////////////////////////////////////////////////////////
var colorScale = d3.scaleLinear()
.domain([0, d3.max(data, function (d) {
return d.count;
}) / 2, d3.max(data, function (d) {
return d.count;
})])
.range(["#FFFFF6", "#3E9583", "#1F2D86"])
//.interpolate(d3.interpolateHcl);
var dayLabels = svg.selectAll(".dayLabel")
.data(days)
.enter().append("text")
.text(function (d) {
return d;
})
.attr("x", 0)
.attr("y", function (d, i) {
return i * gridSize;
})
.style("text-anchor", "end")
.attr("transform", "translate(-36, -11)")
.attr("class", function (d, i) {
return ((i >= 0 && i <= 4) ? "dayLabel mono axis axis-workweek" : "dayLabel mono axis");
})
.style("font-size", heatmap_font_size + "%");
var timeLabels = svg.selectAll(".timeLabel")
.data(times)
.enter().append("text")
.text(function (d) {
return d;
})
.attr("x", function (d, i) {
return i * gridSize;
})
.attr("y", 0)
.style("text-anchor", "middle")
.attr("transform", "translate(-" + gridSize / 2 + ", -36)")
.attr("class", function (d, i) {
return ((i >= 8 && i <= 17) ? "timeLabel mono axis axis-worktime" : "timeLabel mono axis");
})
.style("font-size", heatmap_font_size + "%");
var heatMap = svg.selectAll(".hour")
.data(data)
.enter().append("rect")
.attr("x", function (d) {
return (d.hour - 1) * gridSize;
})
.attr("y", function (d) {
return (d.day - 1) * gridSize;
})
.attr("class", "hour bordered")
.attr("width", gridSize)
.attr("height", gridSize)
.style("stroke", "white")
.style("stroke-opacity", 0.6)
.style("fill", function (d) {
return colorScale(d.count);
})
.on("mouseover", mouseover)
.on("mouseleave", mouseleave);
//Append title to the top
svg.append("text")
.attr("class", "title")
.attr("x", width / 2)
.attr("y", -60)
.style("text-anchor", "middle")
.text("Chat Messages");
///////////////////////////////////////////////////////////////////////////
//////////////// Create the gradient for the legend ///////////////////////
///////////////////////////////////////////////////////////////////////////
//Extra scale since the color scale is interpolated
var countScale = d3.scaleLinear()
.domain([0, d3.max(data, function (d) {
return d.count;
})])
.range([0, width])
//Calculate the variables for the temp gradient
var numStops = 10;
countRange = countScale.domain();
countRange[2] = countRange[1] - countRange[0];
countPoint = [];
for (var i = 0; i < numStops; i++) {
countPoint.push(i * countRange[2] / (numStops - 1) + countRange[0]);
}//for i
//Create the gradient
svg.append("defs")
.append("linearGradient")
.attr("id", "legend-heatmap")
.attr("x1", "0%").attr("y1", "0%")
.attr("x2", "100%").attr("y2", "0%")
.selectAll("stop")
.data(d3.range(numStops))
.enter().append("stop")
.attr("offset", function (d, i) {
return countScale(countPoint[i]) / width;
})
.attr("stop-color", function (d, i) {
return colorScale(countPoint[i]);
});
///////////////////////////////////////////////////////////////////////////
////////////////////////// Draw the legend ////////////////////////////////
///////////////////////////////////////////////////////////////////////////
var legendWidth = Math.min(width * 0.8, 400);
//Color Legend container
var legendsvg = svg.append("g")
.attr("class", "legendWrapper")
.attr("transform", "translate(" + (width / 2) + "," + (gridSize * days.length) + ")"); // 319
//Draw the Rectangle
legendsvg.append("rect")
.attr("class", "legendRect")
.attr("x", -legendWidth / 2)
.attr("y", 0)
//.attr("rx", hexRadius*1.25/2)
.attr("width", legendWidth)
.attr("height", 10)
.style("fill", "url(#legend-heatmap)");
//Append title
legendsvg.append("text")
.attr("class", "legendTitle")
.attr("x", 0)
.attr("y", -10)
.style("text-anchor", "middle")
.text("Number of Messages");
//Set scale for x-axis
var xScale = d3.scaleLinear()
.range([-legendWidth / 2, legendWidth / 2])
.domain([0, d3.max(data, function (d) {
return d.count;
})]);
//Define x-axis
var xAxis = d3.axisBottom(xScale)
//.orient("bottom")
.ticks(5);
//.tickFormat(formatPercent)
//Set up X axis
legendsvg.append("g")
.attr("class", "axis")
.attr("transform", "translate(0," + (10) + ")")
.call(xAxis);
})
</script>
</body>