mirror of https://github.com/CIRCL/AIL-framework
chg: [chats] factorise heatmap + chat icon
terrtia
parent
9fbd3f4bb6
commit
2b8e9b43f3
|
|
@ -89,11 +89,20 @@ class FeederImporter(AbstractImporter):
|
|||
feeder_name = feeder.get_name()
|
||||
print(f'importing: {feeder_name} feeder')
|
||||
|
||||
obj = feeder.get_obj() # TODO replace by a list of objects to import ????
|
||||
# Get Data object:
|
||||
data_obj = feeder.get_obj()
|
||||
|
||||
# process meta
|
||||
if feeder.get_json_meta():
|
||||
feeder.process_meta()
|
||||
objs = feeder.process_meta()
|
||||
if objs is None:
|
||||
objs = set()
|
||||
else:
|
||||
objs = set()
|
||||
|
||||
objs.add(data_obj)
|
||||
|
||||
for obj in objs:
|
||||
if obj.type == 'item': # object save on disk as file (Items)
|
||||
gzip64_content = feeder.get_gzip64_content()
|
||||
return obj, f'{feeder_name} {gzip64_content}'
|
||||
|
|
|
|||
|
|
@ -33,3 +33,4 @@ class BgpMonitorFeeder(DefaultFeeder):
|
|||
tag = 'infoleak:automatic-detection=bgp_monitor'
|
||||
item = Item(self.get_item_id())
|
||||
item.add_tag(tag)
|
||||
return set()
|
||||
|
|
|
|||
|
|
@ -84,4 +84,4 @@ class DefaultFeeder:
|
|||
Process JSON meta filed.
|
||||
"""
|
||||
# meta = self.get_json_meta()
|
||||
pass
|
||||
return set()
|
||||
|
|
|
|||
|
|
@ -0,0 +1,38 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
"""
|
||||
The Telegram Feeder Importer Module
|
||||
================
|
||||
|
||||
Process Telegram JSON
|
||||
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
import datetime
|
||||
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from importer.feeders.abstract_chats_feeder import AbstractChatFeeder
|
||||
from lib.ConfigLoader import ConfigLoader
|
||||
from lib.objects import ail_objects
|
||||
from lib.objects.Chats import Chat
|
||||
from lib.objects import Messages
|
||||
from lib.objects import UsersAccount
|
||||
from lib.objects.Usernames import Username
|
||||
|
||||
import base64
|
||||
|
||||
class DiscordFeeder(AbstractChatFeeder):
|
||||
|
||||
def __init__(self, json_data):
|
||||
super().__init__('discord', json_data)
|
||||
|
||||
# def get_obj(self):.
|
||||
# obj_id = Messages.create_obj_id('telegram', chat_id, message_id, timestamp)
|
||||
# obj_id = f'message:telegram:{obj_id}'
|
||||
# self.obj = ail_objects.get_obj_from_global_id(obj_id)
|
||||
# return self.obj
|
||||
|
||||
|
|
@ -52,4 +52,4 @@ class JabberFeeder(DefaultFeeder):
|
|||
user_fr = Username(fr, 'jabber')
|
||||
user_to.add(date, item)
|
||||
user_fr.add(date, item)
|
||||
return None
|
||||
return set()
|
||||
|
|
|
|||
|
|
@ -45,4 +45,4 @@ class TwitterFeeder(DefaultFeeder):
|
|||
user = str(self.json_data['meta']['twitter:id'])
|
||||
username = Username(user, 'twitter')
|
||||
username.add(date, item)
|
||||
return None
|
||||
return set()
|
||||
|
|
|
|||
|
|
@ -56,3 +56,5 @@ class UrlextractFeeder(DefaultFeeder):
|
|||
item = Item(self.item_id)
|
||||
item.set_parent(parent_id)
|
||||
|
||||
return set()
|
||||
|
||||
|
|
|
|||
|
|
@ -131,7 +131,7 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
|
|||
self.obj = Messages.Message(obj_id)
|
||||
return self.obj
|
||||
|
||||
def process_chat(self, obj, date, timestamp, reply_id=None): # TODO threads
|
||||
def process_chat(self, new_objs, obj, date, timestamp, reply_id=None): # TODO threads
|
||||
meta = self.json_data['meta']['chat'] # todo replace me by function
|
||||
chat = Chat(self.get_chat_id(), self.get_chat_instance_uuid())
|
||||
|
||||
|
|
@ -147,6 +147,12 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
|
|||
if meta.get('date'): # TODO check if already exists
|
||||
chat.set_created_at(int(meta['date']['timestamp']))
|
||||
|
||||
if meta.get('icon'):
|
||||
img = Images.create(meta['icon'], b64=True)
|
||||
img.add(date, chat)
|
||||
chat.set_icon(img.get_global_id())
|
||||
new_objs.add(img)
|
||||
|
||||
if meta.get('username'):
|
||||
username = Username(meta['username'], self.get_chat_protocol())
|
||||
chat.update_username_timeline(username.get_global_id(), timestamp)
|
||||
|
|
@ -228,6 +234,7 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
|
|||
objs = set()
|
||||
if self.obj:
|
||||
objs.add(self.obj)
|
||||
new_objs = set()
|
||||
|
||||
date, timestamp = self.get_message_date_timestamp()
|
||||
|
||||
|
|
@ -261,7 +268,7 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
|
|||
for obj in objs: # TODO PERF avoid parsing metas multiple times
|
||||
|
||||
# CHAT
|
||||
chat = self.process_chat(obj, date, timestamp, reply_id=reply_id)
|
||||
chat = self.process_chat(new_objs, obj, date, timestamp, reply_id=reply_id)
|
||||
|
||||
# SENDER # TODO HANDLE NULL SENDER
|
||||
user_account = self.process_sender(obj, date, timestamp)
|
||||
|
|
@ -279,6 +286,8 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
|
|||
# -> subchannel ?
|
||||
# -> thread id ?
|
||||
|
||||
return new_objs | objs
|
||||
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -175,7 +175,7 @@ class ChatServiceInstance:
|
|||
if 'chats' in options:
|
||||
meta['chats'] = []
|
||||
for chat_id in self.get_chats():
|
||||
meta['chats'].append(Chats.Chat(chat_id, self.uuid).get_meta({'created_at', 'nb_subchannels'}))
|
||||
meta['chats'].append(Chats.Chat(chat_id, self.uuid).get_meta({'created_at', 'icon', 'nb_subchannels'}))
|
||||
return meta
|
||||
|
||||
def get_nb_chats(self):
|
||||
|
|
@ -304,7 +304,7 @@ def api_get_chat(chat_id, chat_instance_uuid):
|
|||
chat = Chats.Chat(chat_id, chat_instance_uuid)
|
||||
if not chat.exists():
|
||||
return {"status": "error", "reason": "Unknown chat"}, 404
|
||||
meta = chat.get_meta({'created_at', 'img', 'info', 'subchannels', 'username'})
|
||||
meta = chat.get_meta({'created_at', 'icon', 'info', 'subchannels', 'username'})
|
||||
if meta['username']:
|
||||
meta['username'] = get_username_meta_from_global_id(meta['username'])
|
||||
if meta['subchannels']:
|
||||
|
|
@ -325,7 +325,7 @@ def api_get_subchannel(chat_id, chat_instance_uuid):
|
|||
subchannel = ChatSubChannels.ChatSubChannel(chat_id, chat_instance_uuid)
|
||||
if not subchannel.exists():
|
||||
return {"status": "error", "reason": "Unknown chat"}, 404
|
||||
meta = subchannel.get_meta({'chat', 'created_at', 'img', 'nb_messages'})
|
||||
meta = subchannel.get_meta({'chat', 'created_at', 'icon', 'nb_messages'})
|
||||
if meta['chat']:
|
||||
meta['chat'] = get_chat_meta_from_global_id(meta['chat'])
|
||||
if meta.get('username'):
|
||||
|
|
|
|||
|
|
@ -74,8 +74,8 @@ class Chat(AbstractChatObject):
|
|||
meta = self._get_meta(options=options)
|
||||
meta['name'] = self.get_name()
|
||||
meta['tags'] = self.get_tags(r_list=True)
|
||||
if 'img':
|
||||
meta['icon'] = self.get_img()
|
||||
if 'icon':
|
||||
meta['icon'] = self.get_icon()
|
||||
if 'info':
|
||||
meta['info'] = self.get_info()
|
||||
if 'username' in options:
|
||||
|
|
|
|||
|
|
@ -113,11 +113,13 @@ class AbstractChatObject(AbstractSubtypeObject, ABC):
|
|||
def set_name(self, name):
|
||||
self._set_field('name', name)
|
||||
|
||||
def get_img(self):
|
||||
return self._get_field('img')
|
||||
def get_icon(self):
|
||||
icon = self._get_field('icon')
|
||||
if icon:
|
||||
return icon.rsplit(':', 1)[1]
|
||||
|
||||
def set_img(self, icon):
|
||||
self._set_field('img', icon)
|
||||
def set_icon(self, icon):
|
||||
self._set_field('icon', icon)
|
||||
|
||||
def get_info(self):
|
||||
return self._get_field('info')
|
||||
|
|
@ -187,7 +189,7 @@ class AbstractChatObject(AbstractSubtypeObject, ABC):
|
|||
tags = {}
|
||||
messages = {}
|
||||
curr_date = None
|
||||
for message in self._get_messages(nb=10, page=3):
|
||||
for message in self._get_messages(nb=30, page=1):
|
||||
timestamp = message[1]
|
||||
date_day = datetime.fromtimestamp(timestamp).strftime('%Y/%m/%d')
|
||||
if date_day != curr_date:
|
||||
|
|
|
|||
|
|
@ -42,6 +42,8 @@ class Exif(AbstractModule):
|
|||
img_exif = img.getexif()
|
||||
print(img_exif)
|
||||
if img_exif:
|
||||
gps = img_exif.get(34853)
|
||||
print(gps)
|
||||
for key, val in img_exif.items():
|
||||
if key in ExifTags.TAGS:
|
||||
print(f'{ExifTags.TAGS[key]}:{val}')
|
||||
|
|
|
|||
|
|
@ -40,7 +40,7 @@ def image(filename):
|
|||
abort(404)
|
||||
filename = filename.replace('/', '')
|
||||
image = Images.Image(filename)
|
||||
return send_from_directory(Images.IMAGE_FOLDER, image.get_rel_path(), as_attachment=True)
|
||||
return send_from_directory(Images.IMAGE_FOLDER, image.get_rel_path(), as_attachment=False, mimetype='image')
|
||||
|
||||
|
||||
@objects_image.route("/objects/images", methods=['GET'])
|
||||
|
|
|
|||
|
|
@ -58,7 +58,9 @@
|
|||
</div>
|
||||
{% endif %}
|
||||
{% if message['images'] %}
|
||||
<img class="message_image mb-1" src="{{ url_for('objects_image.image', filename=message['images'][0])}}">
|
||||
{% for message_image in message['images'] %}
|
||||
<img class="message_image mb-1" src="{{ url_for('objects_image.image', filename=message_image)}}">
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
<pre class="my-0">{{ message['content'] }}</pre>
|
||||
{% for tag in message['tags'] %}
|
||||
|
|
|
|||
|
|
@ -77,7 +77,8 @@
|
|||
{% for chat in chat_instance["chats"] %}
|
||||
<tr>
|
||||
<td>
|
||||
<img src="{{ url_for('static', filename='image/ail-icon.png') }}" class="rounded-circle mr-1" alt="{{ chat['id'] }}" width="40" height="40">
|
||||
<img src="{% if chat['icon'] %}{{ url_for('objects_image.image', filename=chat['icon'])}}{% else %}{{ url_for('static', filename='image/ail-icon.png') }}{% endif %}"
|
||||
class="rounded-circle mr-1" alt="{{ chat['id'] }}" width="40" height="40">
|
||||
</td>
|
||||
<td><b>{{ chat['name'] }}</b></td>
|
||||
<td><a href="{{ url_for('chats_explorer.chats_explorer_chat') }}?uuid={{ chat_instance['uuid'] }}&id={{ chat['id'] }}">{{ chat['id'] }}</a></td>
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@
|
|||
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/d3.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/d3/heatmap_week_hour.js')}}"></script>
|
||||
|
||||
<style>
|
||||
.chat-message-left,
|
||||
|
|
@ -56,7 +57,10 @@
|
|||
<div class="card my-3">
|
||||
|
||||
<div class="card-header" style="background-color:#d9edf7;font-size: 15px">
|
||||
<h4 class="text-secondary">{% if chat['username'] %}{{ chat["username"]["id"] }} {% else %} {{ chat['name'] }}{% endif %} :</h4> {{ chat["id"] }}
|
||||
<h4 class="text-secondary">{% if chat['username'] %}{{ chat["username"]["id"] }} {% else %} {{ chat['name'] }}{% endif %} :</h4>
|
||||
{% if chat['icon'] %}
|
||||
<div><img src="{{ url_for('objects_image.image', filename=chat['icon'])}}" class="mb-2" alt="{{ chat['id'] }}" width="200" height="200"></div>
|
||||
{% endif %}
|
||||
<ul class="list-group mb-2">
|
||||
<li class="list-group-item py-0">
|
||||
<table class="table">
|
||||
|
|
@ -218,6 +222,12 @@ function toggle_sidebar(){
|
|||
|
||||
|
||||
<script>
|
||||
|
||||
d3.json("{{ url_for('chats_explorer.chats_explorer_messages_stats_week') }}?uuid={{ chat['subtype'] }}&id={{ chat['id'] }}")
|
||||
.then(function(data) {
|
||||
create_heatmap_week_hour('#heatmapweekhour', data);
|
||||
})
|
||||
|
||||
/*
|
||||
|
||||
// set the dimensions and margins of the graph
|
||||
|
|
@ -323,225 +333,6 @@ d3.csv("").then( function(data) {
|
|||
*/
|
||||
</script>
|
||||
|
||||
<script>
|
||||
|
||||
// based on gist nbremer/62cf60e116ae821c06602793d265eaf6
|
||||
d3.json("{{ url_for('chats_explorer.chats_explorer_messages_stats_week') }}?uuid={{ chat['subtype'] }}&id={{ chat['id'] }}")
|
||||
.then(function(data) {
|
||||
|
||||
var days = ["Mo", "Tu", "We", "Th", "Fr", "Sa", "Su"],
|
||||
times = d3.range(24);
|
||||
|
||||
var margin = {
|
||||
top: 80,
|
||||
right: 50,
|
||||
bottom: 20,
|
||||
left: 50
|
||||
};
|
||||
|
||||
var width = Math.max(Math.min(window.innerWidth, 1000), 500) - margin.left - margin.right - 20,
|
||||
gridSize = Math.floor(width / times.length),
|
||||
height = gridSize * (days.length + 2);
|
||||
|
||||
var heatmap_font_size = width * 62.5 / 900;
|
||||
|
||||
//SVG container
|
||||
var svg = d3.select('#heatmapweekhour')
|
||||
.append("svg")
|
||||
.attr("width", width + margin.left + margin.right)
|
||||
.attr("height", height + margin.top + margin.bottom)
|
||||
.append("g")
|
||||
.attr("transform", "translate(" + margin.left + "," + margin.top + ")");
|
||||
|
||||
// create a tooltip
|
||||
const tooltip = d3.select("#heatmapweekhour")
|
||||
.append("div")
|
||||
.style("opacity", 0)
|
||||
.attr("class", "tooltip")
|
||||
.style("background-color", "white")
|
||||
.style("border", "solid")
|
||||
.style("border-width", "2px")
|
||||
.style("border-radius", "5px")
|
||||
.style("padding", "5px")
|
||||
|
||||
// Three function that change the tooltip when user hover / move / leave a cell
|
||||
const mouseover = function(d) {
|
||||
tooltip.style("opacity", 1)
|
||||
d3.select(this)
|
||||
.style("stroke", "black")
|
||||
//.style("stroke-opacity", 1)
|
||||
|
||||
tooltip.html(d.date + " " + d.hour + "-" + (d.hour + 1) + "h: <b>" + d.count + "</b> messages")
|
||||
}
|
||||
const mouseleave = function(d) {
|
||||
tooltip.style("opacity", 0)
|
||||
d3.select(this)
|
||||
.style("stroke", "white")
|
||||
//.style("stroke-opacity", 0.8)
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////
|
||||
//////////////////////////// Draw Heatmap /////////////////////////////////
|
||||
///////////////////////////////////////////////////////////////////////////
|
||||
|
||||
var colorScale = d3.scaleLinear()
|
||||
.domain([0, d3.max(data, function (d) {
|
||||
return d.count;
|
||||
}) / 2, d3.max(data, function (d) {
|
||||
return d.count;
|
||||
})])
|
||||
.range(["#FFFFF6", "#3E9583", "#1F2D86"])
|
||||
//.interpolate(d3.interpolateHcl);
|
||||
|
||||
var dayLabels = svg.selectAll(".dayLabel")
|
||||
.data(days)
|
||||
.enter().append("text")
|
||||
.text(function (d) {
|
||||
return d;
|
||||
})
|
||||
.attr("x", 0)
|
||||
.attr("y", function (d, i) {
|
||||
return i * gridSize;
|
||||
})
|
||||
.style("text-anchor", "end")
|
||||
.attr("transform", "translate(-36, -11)")
|
||||
.attr("class", function (d, i) {
|
||||
return ((i >= 0 && i <= 4) ? "dayLabel mono axis axis-workweek" : "dayLabel mono axis");
|
||||
})
|
||||
.style("font-size", heatmap_font_size + "%");
|
||||
|
||||
var timeLabels = svg.selectAll(".timeLabel")
|
||||
.data(times)
|
||||
.enter().append("text")
|
||||
.text(function (d) {
|
||||
return d;
|
||||
})
|
||||
.attr("x", function (d, i) {
|
||||
return i * gridSize;
|
||||
})
|
||||
.attr("y", 0)
|
||||
.style("text-anchor", "middle")
|
||||
.attr("transform", "translate(-" + gridSize / 2 + ", -36)")
|
||||
.attr("class", function (d, i) {
|
||||
return ((i >= 8 && i <= 17) ? "timeLabel mono axis axis-worktime" : "timeLabel mono axis");
|
||||
})
|
||||
.style("font-size", heatmap_font_size + "%");
|
||||
|
||||
var heatMap = svg.selectAll(".hour")
|
||||
.data(data)
|
||||
.enter().append("rect")
|
||||
.attr("x", function (d) {
|
||||
return (d.hour - 1) * gridSize;
|
||||
})
|
||||
.attr("y", function (d) {
|
||||
return (d.day - 1) * gridSize;
|
||||
})
|
||||
.attr("class", "hour bordered")
|
||||
.attr("width", gridSize)
|
||||
.attr("height", gridSize)
|
||||
.style("stroke", "white")
|
||||
.style("stroke-opacity", 0.6)
|
||||
.style("fill", function (d) {
|
||||
return colorScale(d.count);
|
||||
})
|
||||
.on("mouseover", mouseover)
|
||||
.on("mouseleave", mouseleave);
|
||||
|
||||
//Append title to the top
|
||||
svg.append("text")
|
||||
.attr("class", "title")
|
||||
.attr("x", width / 2)
|
||||
.attr("y", -60)
|
||||
.style("text-anchor", "middle")
|
||||
.text("Chat Messages");
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////
|
||||
//////////////// Create the gradient for the legend ///////////////////////
|
||||
///////////////////////////////////////////////////////////////////////////
|
||||
|
||||
//Extra scale since the color scale is interpolated
|
||||
var countScale = d3.scaleLinear()
|
||||
.domain([0, d3.max(data, function (d) {
|
||||
return d.count;
|
||||
})])
|
||||
.range([0, width])
|
||||
|
||||
//Calculate the variables for the temp gradient
|
||||
var numStops = 10;
|
||||
countRange = countScale.domain();
|
||||
countRange[2] = countRange[1] - countRange[0];
|
||||
countPoint = [];
|
||||
for (var i = 0; i < numStops; i++) {
|
||||
countPoint.push(i * countRange[2] / (numStops - 1) + countRange[0]);
|
||||
}//for i
|
||||
|
||||
//Create the gradient
|
||||
svg.append("defs")
|
||||
.append("linearGradient")
|
||||
.attr("id", "legend-heatmap")
|
||||
.attr("x1", "0%").attr("y1", "0%")
|
||||
.attr("x2", "100%").attr("y2", "0%")
|
||||
.selectAll("stop")
|
||||
.data(d3.range(numStops))
|
||||
.enter().append("stop")
|
||||
.attr("offset", function (d, i) {
|
||||
return countScale(countPoint[i]) / width;
|
||||
})
|
||||
.attr("stop-color", function (d, i) {
|
||||
return colorScale(countPoint[i]);
|
||||
});
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////
|
||||
////////////////////////// Draw the legend ////////////////////////////////
|
||||
///////////////////////////////////////////////////////////////////////////
|
||||
|
||||
var legendWidth = Math.min(width * 0.8, 400);
|
||||
//Color Legend container
|
||||
var legendsvg = svg.append("g")
|
||||
.attr("class", "legendWrapper")
|
||||
.attr("transform", "translate(" + (width / 2) + "," + (gridSize * days.length) + ")"); // 319
|
||||
|
||||
//Draw the Rectangle
|
||||
legendsvg.append("rect")
|
||||
.attr("class", "legendRect")
|
||||
.attr("x", -legendWidth / 2)
|
||||
.attr("y", 0)
|
||||
//.attr("rx", hexRadius*1.25/2)
|
||||
.attr("width", legendWidth)
|
||||
.attr("height", 10)
|
||||
.style("fill", "url(#legend-heatmap)");
|
||||
|
||||
//Append title
|
||||
legendsvg.append("text")
|
||||
.attr("class", "legendTitle")
|
||||
.attr("x", 0)
|
||||
.attr("y", -10)
|
||||
.style("text-anchor", "middle")
|
||||
.text("Number of Messages");
|
||||
|
||||
//Set scale for x-axis
|
||||
var xScale = d3.scaleLinear()
|
||||
.range([-legendWidth / 2, legendWidth / 2])
|
||||
.domain([0, d3.max(data, function (d) {
|
||||
return d.count;
|
||||
})]);
|
||||
|
||||
//Define x-axis
|
||||
var xAxis = d3.axisBottom(xScale)
|
||||
//.orient("bottom")
|
||||
.ticks(5);
|
||||
//.tickFormat(formatPercent)
|
||||
|
||||
//Set up X axis
|
||||
legendsvg.append("g")
|
||||
.attr("class", "axis")
|
||||
.attr("transform", "translate(0," + (10) + ")")
|
||||
.call(xAxis);
|
||||
|
||||
|
||||
})
|
||||
|
||||
</script>
|
||||
|
||||
</body>
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue