chg: [chats] factorise heatmap + chat icon

bin/importer/FeederImporter.py

@@ -89,17 +89,26 @@ class FeederImporter(AbstractImporter):
         feeder_name = feeder.get_name()
         print(f'importing: {feeder_name} feeder')
 
-        obj = feeder.get_obj()  # TODO replace by a list of objects to import ????
+        # Get Data object:
+        data_obj = feeder.get_obj()
+
         # process meta
         if feeder.get_json_meta():
-            feeder.process_meta()
+            objs = feeder.process_meta()
+            if objs is None:
+                objs = set()
+        else:
+            objs = set()
 
-        if obj.type == 'item':  # object save on disk as file (Items)
+        objs.add(data_obj)
-            gzip64_content = feeder.get_gzip64_content()
+
-            return obj, f'{feeder_name} {gzip64_content}'
+        for obj in objs:
-        else:  # Messages save on DB
+            if obj.type == 'item':  # object save on disk as file (Items)
-            if obj.exists():
+                gzip64_content = feeder.get_gzip64_content()
-                return obj, f'{feeder_name}'
+                return obj, f'{feeder_name} {gzip64_content}'
+            else:  # Messages save on DB
+                if obj.exists():
+                    return obj, f'{feeder_name}'
 
 
 class FeederModuleImporter(AbstractModule):

bin/importer/feeders/BgpMonitor.py

@@ -33,3 +33,4 @@ class BgpMonitorFeeder(DefaultFeeder):
         tag = 'infoleak:automatic-detection=bgp_monitor'
         item = Item(self.get_item_id())
         item.add_tag(tag)
+        return set()

bin/importer/feeders/Default.py

@@ -84,4 +84,4 @@ class DefaultFeeder:
         Process JSON meta filed.
         """
         # meta = self.get_json_meta()
-        pass
+        return set()

bin/importer/feeders/Discord.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+"""
+The Telegram Feeder Importer Module
+================
+
+Process Telegram JSON
+
+"""
+import os
+import sys
+import datetime
+
+sys.path.append(os.environ['AIL_BIN'])
+##################################
+# Import Project packages
+##################################
+from importer.feeders.abstract_chats_feeder import AbstractChatFeeder
+from lib.ConfigLoader import ConfigLoader
+from lib.objects import ail_objects
+from lib.objects.Chats import Chat
+from lib.objects import Messages
+from lib.objects import UsersAccount
+from lib.objects.Usernames import Username
+
+import base64
+
+class DiscordFeeder(AbstractChatFeeder):
+
+    def __init__(self, json_data):
+        super().__init__('discord', json_data)
+
+    # def get_obj(self):.
+    #     obj_id = Messages.create_obj_id('telegram', chat_id, message_id, timestamp)
+    #     obj_id = f'message:telegram:{obj_id}'
+    #     self.obj = ail_objects.get_obj_from_global_id(obj_id)
+    #     return self.obj
+

bin/importer/feeders/Jabber.py

@@ -52,4 +52,4 @@ class JabberFeeder(DefaultFeeder):
         user_fr = Username(fr, 'jabber')
         user_to.add(date, item)
         user_fr.add(date, item)
-        return None
+        return set()

bin/importer/feeders/Twitter.py

@@ -45,4 +45,4 @@ class TwitterFeeder(DefaultFeeder):
         user = str(self.json_data['meta']['twitter:id'])
         username = Username(user, 'twitter')
         username.add(date, item)
-        return None
+        return set()

bin/importer/feeders/Urlextract.py

@@ -56,3 +56,5 @@ class UrlextractFeeder(DefaultFeeder):
             item = Item(self.item_id)
             item.set_parent(parent_id)
 
+        return set()
+

bin/importer/feeders/abstract_chats_feeder.py

@@ -131,7 +131,7 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
             self.obj = Messages.Message(obj_id)
         return self.obj
 
-    def process_chat(self, obj, date, timestamp, reply_id=None):  # TODO threads
+    def process_chat(self, new_objs, obj, date, timestamp, reply_id=None):  # TODO threads
         meta = self.json_data['meta']['chat'] # todo replace me by function
         chat = Chat(self.get_chat_id(), self.get_chat_instance_uuid())
 
@@ -147,6 +147,12 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
         if meta.get('date'): # TODO check if already exists
             chat.set_created_at(int(meta['date']['timestamp']))
 
+        if meta.get('icon'):
+            img = Images.create(meta['icon'], b64=True)
+            img.add(date, chat)
+            chat.set_icon(img.get_global_id())
+            new_objs.add(img)
+
         if meta.get('username'):
             username = Username(meta['username'], self.get_chat_protocol())
             chat.update_username_timeline(username.get_global_id(), timestamp)
@@ -228,6 +234,7 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
         objs = set()
         if self.obj:
             objs.add(self.obj)
+        new_objs = set()
 
         date, timestamp = self.get_message_date_timestamp()
 
@@ -261,7 +268,7 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
         for obj in objs:  # TODO PERF avoid parsing metas multiple times
 
             # CHAT
-            chat = self.process_chat(obj, date, timestamp, reply_id=reply_id)
+            chat = self.process_chat(new_objs, obj, date, timestamp, reply_id=reply_id)
 
             # SENDER # TODO HANDLE NULL SENDER
             user_account = self.process_sender(obj, date, timestamp)
@@ -279,6 +286,8 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
                 #       -> subchannel ?
                 #       -> thread id ?
 
+        return new_objs | objs
+
 
 
 

bin/lib/chats_viewer.py

@@ -175,7 +175,7 @@ class ChatServiceInstance:
         if 'chats' in options:
             meta['chats'] = []
             for chat_id in self.get_chats():
-                meta['chats'].append(Chats.Chat(chat_id, self.uuid).get_meta({'created_at', 'nb_subchannels'}))
+                meta['chats'].append(Chats.Chat(chat_id, self.uuid).get_meta({'created_at', 'icon', 'nb_subchannels'}))
         return meta
 
     def get_nb_chats(self):
@@ -304,7 +304,7 @@ def api_get_chat(chat_id, chat_instance_uuid):
     chat = Chats.Chat(chat_id, chat_instance_uuid)
     if not chat.exists():
         return {"status": "error", "reason": "Unknown chat"}, 404
-    meta = chat.get_meta({'created_at', 'img', 'info', 'subchannels', 'username'})
+    meta = chat.get_meta({'created_at', 'icon', 'info', 'subchannels', 'username'})
     if meta['username']:
         meta['username'] = get_username_meta_from_global_id(meta['username'])
     if meta['subchannels']:
@@ -325,7 +325,7 @@ def api_get_subchannel(chat_id, chat_instance_uuid):
     subchannel = ChatSubChannels.ChatSubChannel(chat_id, chat_instance_uuid)
     if not subchannel.exists():
         return {"status": "error", "reason": "Unknown chat"}, 404
-    meta = subchannel.get_meta({'chat', 'created_at', 'img', 'nb_messages'})
+    meta = subchannel.get_meta({'chat', 'created_at', 'icon', 'nb_messages'})
     if meta['chat']:
         meta['chat'] = get_chat_meta_from_global_id(meta['chat'])
     if meta.get('username'):

bin/lib/objects/Chats.py

@@ -74,8 +74,8 @@ class Chat(AbstractChatObject):
         meta = self._get_meta(options=options)
         meta['name'] = self.get_name()
         meta['tags'] = self.get_tags(r_list=True)
-        if 'img':
+        if 'icon':
-            meta['icon'] = self.get_img()
+            meta['icon'] = self.get_icon()
         if 'info':
             meta['info'] = self.get_info()
         if 'username' in options:

bin/lib/objects/abstract_chat_object.py

@@ -113,11 +113,13 @@ class AbstractChatObject(AbstractSubtypeObject, ABC):
     def set_name(self, name):
         self._set_field('name', name)
 
-    def get_img(self):
+    def get_icon(self):
-        return self._get_field('img')
+        icon = self._get_field('icon')
+        if icon:
+            return icon.rsplit(':', 1)[1]
 
-    def set_img(self, icon):
+    def set_icon(self, icon):
-        self._set_field('img', icon)
+        self._set_field('icon', icon)
 
     def get_info(self):
         return self._get_field('info')
@@ -187,7 +189,7 @@ class AbstractChatObject(AbstractSubtypeObject, ABC):
         tags = {}
         messages = {}
         curr_date = None
-        for message in self._get_messages(nb=10, page=3):
+        for message in self._get_messages(nb=30, page=1):
             timestamp = message[1]
             date_day = datetime.fromtimestamp(timestamp).strftime('%Y/%m/%d')
             if date_day != curr_date:

bin/modules/Exif.py

@@ -42,6 +42,8 @@ class Exif(AbstractModule):
         img_exif = img.getexif()
         print(img_exif)
         if img_exif:
+            gps = img_exif.get(34853)
+            print(gps)
             for key, val in img_exif.items():
                 if key in ExifTags.TAGS:
                     print(f'{ExifTags.TAGS[key]}:{val}')

var/www/blueprints/objects_image.py

@@ -40,7 +40,7 @@ def image(filename):
         abort(404)
     filename = filename.replace('/', '')
     image = Images.Image(filename)
-    return send_from_directory(Images.IMAGE_FOLDER, image.get_rel_path(), as_attachment=True)
+    return send_from_directory(Images.IMAGE_FOLDER, image.get_rel_path(), as_attachment=False, mimetype='image')
 
 
 @objects_image.route("/objects/images", methods=['GET'])

var/www/templates/chats_explorer/block_message.html

@@ -58,7 +58,9 @@
             </div>
         {% endif %}
         {% if message['images'] %}
-            <img class="message_image mb-1" src="{{ url_for('objects_image.image', filename=message['images'][0])}}">
+            {% for message_image in message['images'] %}
+                <img class="message_image mb-1" src="{{ url_for('objects_image.image', filename=message_image)}}">
+            {%  endfor %}
         {% endif %}
         <pre class="my-0">{{ message['content'] }}</pre>
         {% for tag in message['tags'] %}

var/www/templates/chats_explorer/chat_instance.html

@@ -77,7 +77,8 @@
                     {% for chat in chat_instance["chats"] %}
                         <tr>
                             <td>
-                                <img src="{{ url_for('static', filename='image/ail-icon.png') }}" class="rounded-circle mr-1" alt="{{ chat['id'] }}" width="40" height="40">
+                                <img src="{% if chat['icon'] %}{{ url_for('objects_image.image', filename=chat['icon'])}}{% else %}{{ url_for('static', filename='image/ail-icon.png') }}{% endif %}"
+                                     class="rounded-circle mr-1" alt="{{ chat['id'] }}" width="40" height="40">
                             </td>
                             <td><b>{{ chat['name'] }}</b></td>
                             <td><a href="{{ url_for('chats_explorer.chats_explorer_chat') }}?uuid={{ chat_instance['uuid'] }}&id={{ chat['id'] }}">{{ chat['id'] }}</a></td>

var/www/templates/chats_explorer/chat_viewer.html

@@ -17,6 +17,7 @@
 	<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
 	<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
     <script src="{{ url_for('static', filename='js/d3.min.js')}}"></script>
+    <script src="{{ url_for('static', filename='js/d3/heatmap_week_hour.js')}}"></script>
 
     <style>
         .chat-message-left,
@@ -56,7 +57,10 @@
                 <div class="card my-3">
 
                     <div class="card-header" style="background-color:#d9edf7;font-size: 15px">
-                        <h4 class="text-secondary">{% if chat['username'] %}{{ chat["username"]["id"] }} {% else %} {{ chat['name'] }}{% endif %} :</h4> {{ chat["id"] }}
+                        <h4 class="text-secondary">{% if chat['username'] %}{{ chat["username"]["id"] }} {% else %} {{ chat['name'] }}{% endif %} :</h4>
+                        {% if chat['icon'] %}
+                            <div><img src="{{ url_for('objects_image.image', filename=chat['icon'])}}" class="mb-2" alt="{{ chat['id'] }}" width="200" height="200"></div>
+                        {% endif %}
                         <ul class="list-group mb-2">
                             <li class="list-group-item py-0">
                                 <table class="table">
@@ -218,6 +222,12 @@ function toggle_sidebar(){
 
 
 <script>
+
+d3.json("{{ url_for('chats_explorer.chats_explorer_messages_stats_week') }}?uuid={{ chat['subtype'] }}&id={{ chat['id'] }}")
+.then(function(data) {
+    create_heatmap_week_hour('#heatmapweekhour', data);
+})
+
 /*
 
 // set the dimensions and margins of the graph
@@ -323,225 +333,6 @@ d3.csv("").then( function(data) {
  */
 </script>
 
-<script>
-
-// based on gist nbremer/62cf60e116ae821c06602793d265eaf6
-d3.json("{{ url_for('chats_explorer.chats_explorer_messages_stats_week') }}?uuid={{ chat['subtype'] }}&id={{ chat['id'] }}")
-.then(function(data) {
-
-    var days = ["Mo", "Tu", "We", "Th", "Fr", "Sa", "Su"],
-        times = d3.range(24);
-
-    var margin = {
-        top: 80,
-        right: 50,
-        bottom: 20,
-        left: 50
-    };
-
-    var width = Math.max(Math.min(window.innerWidth, 1000), 500) - margin.left - margin.right - 20,
-        gridSize = Math.floor(width / times.length),
-        height = gridSize * (days.length + 2);
-
-    var heatmap_font_size = width * 62.5 / 900;
-
-//SVG container
-    var svg = d3.select('#heatmapweekhour')
-        .append("svg")
-        .attr("width", width + margin.left + margin.right)
-        .attr("height", height + margin.top + margin.bottom)
-        .append("g")
-        .attr("transform", "translate(" + margin.left + "," + margin.top + ")");
-
-    // create a tooltip
-    const tooltip = d3.select("#heatmapweekhour")
-        .append("div")
-        .style("opacity", 0)
-        .attr("class", "tooltip")
-        .style("background-color", "white")
-        .style("border", "solid")
-        .style("border-width", "2px")
-        .style("border-radius", "5px")
-        .style("padding", "5px")
-
-    // Three function that change the tooltip when user hover / move / leave a cell
-    const mouseover = function(d) {
-        tooltip.style("opacity", 1)
-        d3.select(this)
-            .style("stroke", "black")
-            //.style("stroke-opacity", 1)
-
-        tooltip.html(d.date + " " + d.hour + "-" + (d.hour + 1) + "h: <b>" + d.count + "</b> messages")
-    }
-    const mouseleave = function(d) {
-        tooltip.style("opacity", 0)
-        d3.select(this)
-            .style("stroke", "white")
-            //.style("stroke-opacity", 0.8)
-    }
-
-///////////////////////////////////////////////////////////////////////////
-//////////////////////////// Draw Heatmap /////////////////////////////////
-///////////////////////////////////////////////////////////////////////////
-
-    var colorScale = d3.scaleLinear()
-        .domain([0, d3.max(data, function (d) {
-            return d.count;
-        }) / 2, d3.max(data, function (d) {
-            return d.count;
-        })])
-        .range(["#FFFFF6", "#3E9583", "#1F2D86"])
-    //.interpolate(d3.interpolateHcl);
-
-    var dayLabels = svg.selectAll(".dayLabel")
-        .data(days)
-        .enter().append("text")
-        .text(function (d) {
-            return d;
-        })
-        .attr("x", 0)
-        .attr("y", function (d, i) {
-            return i * gridSize;
-        })
-        .style("text-anchor", "end")
-        .attr("transform", "translate(-36, -11)")
-        .attr("class", function (d, i) {
-            return ((i >= 0 && i <= 4) ? "dayLabel mono axis axis-workweek" : "dayLabel mono axis");
-        })
-        .style("font-size", heatmap_font_size + "%");
-
-    var timeLabels = svg.selectAll(".timeLabel")
-        .data(times)
-        .enter().append("text")
-        .text(function (d) {
-            return d;
-        })
-        .attr("x", function (d, i) {
-            return i * gridSize;
-        })
-        .attr("y", 0)
-        .style("text-anchor", "middle")
-        .attr("transform", "translate(-" + gridSize / 2 + ", -36)")
-        .attr("class", function (d, i) {
-            return ((i >= 8 && i <= 17) ? "timeLabel mono axis axis-worktime" : "timeLabel mono axis");
-        })
-        .style("font-size", heatmap_font_size + "%");
-
-    var heatMap = svg.selectAll(".hour")
-        .data(data)
-        .enter().append("rect")
-        .attr("x", function (d) {
-            return (d.hour - 1) * gridSize;
-        })
-        .attr("y", function (d) {
-            return (d.day - 1) * gridSize;
-        })
-        .attr("class", "hour bordered")
-        .attr("width", gridSize)
-        .attr("height", gridSize)
-        .style("stroke", "white")
-        .style("stroke-opacity", 0.6)
-        .style("fill", function (d) {
-            return colorScale(d.count);
-        })
-        .on("mouseover", mouseover)
-        .on("mouseleave", mouseleave);
-
-//Append title to the top
-    svg.append("text")
-        .attr("class", "title")
-        .attr("x", width / 2)
-        .attr("y", -60)
-        .style("text-anchor", "middle")
-        .text("Chat Messages");
-
-///////////////////////////////////////////////////////////////////////////
-//////////////// Create the gradient for the legend ///////////////////////
-///////////////////////////////////////////////////////////////////////////
-
-//Extra scale since the color scale is interpolated
-    var countScale = d3.scaleLinear()
-        .domain([0, d3.max(data, function (d) {
-            return d.count;
-        })])
-        .range([0, width])
-
-//Calculate the variables for the temp gradient
-    var numStops = 10;
-    countRange = countScale.domain();
-    countRange[2] = countRange[1] - countRange[0];
-    countPoint = [];
-    for (var i = 0; i < numStops; i++) {
-        countPoint.push(i * countRange[2] / (numStops - 1) + countRange[0]);
-    }//for i
-
-//Create the gradient
-    svg.append("defs")
-        .append("linearGradient")
-        .attr("id", "legend-heatmap")
-        .attr("x1", "0%").attr("y1", "0%")
-        .attr("x2", "100%").attr("y2", "0%")
-        .selectAll("stop")
-        .data(d3.range(numStops))
-        .enter().append("stop")
-        .attr("offset", function (d, i) {
-            return countScale(countPoint[i]) / width;
-        })
-        .attr("stop-color", function (d, i) {
-            return colorScale(countPoint[i]);
-        });
-
-///////////////////////////////////////////////////////////////////////////
-////////////////////////// Draw the legend ////////////////////////////////
-///////////////////////////////////////////////////////////////////////////
-
-    var legendWidth = Math.min(width * 0.8, 400);
-//Color Legend container
-    var legendsvg = svg.append("g")
-        .attr("class", "legendWrapper")
-        .attr("transform", "translate(" + (width / 2) + "," + (gridSize * days.length) + ")");  // 319
-
-//Draw the Rectangle
-    legendsvg.append("rect")
-        .attr("class", "legendRect")
-        .attr("x", -legendWidth / 2)
-        .attr("y", 0)
-        //.attr("rx", hexRadius*1.25/2)
-        .attr("width", legendWidth)
-        .attr("height", 10)
-        .style("fill", "url(#legend-heatmap)");
-
-//Append title
-    legendsvg.append("text")
-        .attr("class", "legendTitle")
-        .attr("x", 0)
-        .attr("y", -10)
-        .style("text-anchor", "middle")
-        .text("Number of Messages");
-
-//Set scale for x-axis
-    var xScale = d3.scaleLinear()
-        .range([-legendWidth / 2, legendWidth / 2])
-        .domain([0, d3.max(data, function (d) {
-            return d.count;
-        })]);
-
-//Define x-axis
-    var xAxis = d3.axisBottom(xScale)
-        //.orient("bottom")
-        .ticks(5);
-    //.tickFormat(formatPercent)
-
-//Set up X axis
-    legendsvg.append("g")
-        .attr("class", "axis")
-        .attr("transform", "translate(0," + (10) + ")")
-        .call(xAxis);
-
-
-})
-
-</script>
 
 </body>