chg: [doc] add overview

HOWTO.md

@@ -29,6 +29,8 @@ Contributions are welcome! Fork the repository, experiment with the code, and su
 
 AIL supports crawling of websites and Tor hidden services. Ensure your Tor client's proxy configuration is correct, especially the SOCKS5 proxy settings.
 
+![Crawler](./doc/screenshots/ail-lacus.png?raw=true "AIL framework Crawler")
+
 ### Installation
 
 [Install Lacus](https://github.com/ail-project/lacus)

README.md

@@ -29,6 +29,8 @@ AIL framework - Framework for Analysis of Information Leaks
 
 AIL is a modular framework to analyse potential information leaks from unstructured data sources like pastes from Pastebin or similar services or unstructured data streams. AIL framework is flexible and can be extended to support other functionalities to mine or process sensitive information (e.g. data leak prevention).
 
+![Overview](./doc/screenshots/ail-overview.png?raw=true "AIL framework Overview")
+
 ![Dashboard](./doc/screenshots/dashboard0.png?raw=true "AIL framework dashboard")
 
 
@@ -55,6 +57,8 @@ Allow easy creation and customization by extending an abstract class.
 
 ## Features
 
+![Internal](./doc/screenshots/ail-internal.png?raw=true "AIL framework Internal")
+
 - Modular architecture to handle streams of unstructured or structured information
 - Default support for external ZMQ feeds, such as provided by CIRCL or other providers
 - Multiple Importers and feeds support

bin/importer/FeederImporter.py

@@ -109,6 +109,9 @@ class FeederImporter(AbstractImporter):
                 gzip64_content = feeder.get_gzip64_content()
                 relay_message = f'{feeder_name} {gzip64_content}'
                 objs_messages.append({'obj': obj, 'message': relay_message})
+            elif obj.type == 'image':
+                date = feeder.get_date()
+                objs_messages.append({'obj': obj, 'message': f'{feeder_name} {date}'})
             else:  # Messages save on DB
                 if obj.exists() and obj.type != 'chat':
                     objs_messages.append({'obj': obj, 'message': feeder_name})

bin/importer/feeders/Default.py

@@ -41,6 +41,9 @@ class DefaultFeeder:
     def get_source(self):
         return self.json_data.get('source')
 
+    def get_date(self):
+        return datetime.date.today().strftime("%Y%m%d")
+
     def get_json_data(self):
         """
         Return the JSON data,

bin/importer/feeders/abstract_chats_feeder.py

@@ -92,6 +92,14 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
     def get_reactions(self):
         return self.json_data['meta'].get('reactions', [])
 
+    def get_date(self):
+        if self.json_data['meta'].get('date'):
+            date = datetime.datetime.fromtimestamp( self.json_data['meta']['date']['timestamp'])
+            date = date.strftime('%Y%m%d')
+        else:
+            date = datetime.date.today().strftime("%Y%m%d")
+        return date
+
     def get_message_timestamp(self):
         if not self.json_data['meta'].get('date'):
             return None

bin/lib/objects/Ocrs.py

@@ -15,7 +15,7 @@ sys.path.append(os.environ['AIL_BIN'])
 ##################################
 # Import Project packages
 ##################################
-from lib.objects.abstract_object import AbstractObject
+from lib.objects.abstract_daterange_object import AbstractDaterangeObject, AbstractDaterangeObjects
 from lib.ConfigLoader import ConfigLoader
 from packages import Date
 # from lib import Language
@@ -32,7 +32,7 @@ config_loader = None
 
 # SET x1,y1:x2,y2:x3,y3:x4,y4:extracted_text
 
-class Ocr(AbstractObject):
+class Ocr(AbstractDaterangeObject):
     """
     AIL Message Object. (strings)
     """
@@ -147,7 +147,7 @@ class Ocr(AbstractObject):
         """
         if options is None:
             options = set()
-        meta = self.get_default_meta(tags=True)
+        meta = self._get_meta(options=options)
         meta['content'] = self.get_content()
 
         # optional meta fields
@@ -218,17 +218,19 @@ class Ocr(AbstractObject):
             coords.append((f'{x1},{y1},{x2},{y2},{x3},{y3},{x4},{y4}', extract[4]))
         return coords
 
-    def edit(self, coordinates, text, new_text, new_coordinates=None):
+    def edit_text(self, coordinates, text, new_text, new_coordinates=None):
         pass
 
-    def add(self, coordinates, text):
+    def add_text(self, coordinates, text):
         val = f'{coordinates}:{text}'
         return r_object.sadd(f'ocr:{self.id}', val)
 
-    def remove(self, val):
+    def remove_text(self, val):
         return r_object.srem(f'ocr:{self.id}', val)
 
-    def update_correlation(self):
+    def update_correlation(self, date=None):
+        if date:
+            self.add(date, None)
         image_correl = self.get_obj_correlations('image', '', self.id)
         for obj_type in image_correl:
             if obj_type != 'ocr':
@@ -237,19 +239,24 @@ class Ocr(AbstractObject):
                     self.add_correlation(obj_type, obj_subtype, obj_id)
 
     def create(self, extracted_texts, tags=[]):
-        r_object.sadd(f'{self.type}:all', self.id)
+        # r_object.sadd(f'{self.type}:all', self.id)
+        created = False
         for extracted in extracted_texts:
             bbox, text = extracted
             if len(text) > 1:
                 str_coords = self.create_coord_str(bbox)
-                self.add(str_coords, text)
+                self.add_text(str_coords, text)
+                created = True
 
-        # Correlations
-        self.update_correlation()
-        self.add_correlation('image', '', self.id)
+        if created:
+            # Correlations
+            self._copy_from('image', self.id)
+            self.update_correlation()
+            self.add_correlation('image', '', self.id)
 
-        for tag in tags:
-            self.add_tag(tag)
+            for tag in tags:
+                self.add_tag(tag)
+            return self.id
 
     # # WARNING: UNCLEAN DELETE /!\ TEST ONLY /!\
     def delete(self):
@@ -273,9 +280,9 @@ class Ocr(AbstractObject):
 def create(obj_id, detections, tags=[]):
     obj = Ocr(obj_id)
     if not obj.exists():
-        obj.create(detections, tags=tags)
-    # TODO Edit
-    return obj
+        obj_id = obj.create(detections, tags=tags)
+        if obj_id:
+            return obj
 
 # TODO preload languages
 def extract_text(image_path, languages, threshold=0.2):
@@ -298,6 +305,16 @@ def get_all_ocrs_objects(filters={}):
     for obj_id in get_ids():
         yield Ocr(obj_id)
 
+class Ocrs(AbstractDaterangeObjects):
+    """
+        OCR Objects
+    """
+    def __init__(self):
+        super().__init__('ocr', Ocr)
+
+    def sanitize_id_to_search(self, name_to_search):
+        return name_to_search  # TODO
+
 
 #### API ####
 def api_get_ocr(obj_id, translation_target=None):
@@ -306,6 +323,3 @@ def api_get_ocr(obj_id, translation_target=None):
         return {"status": "error", "reason": "Unknown ocr"}, 404
     meta = ocr.get_meta({'content', 'icon', 'img', 'language', 'link', 'map', 'translation'}, translation_target=translation_target)
     return meta, 200
-
-
-

bin/lib/objects/abstract_daterange_object.py

@@ -71,7 +71,7 @@ class AbstractDaterangeObject(AbstractObject, ABC):
         else:
             return last_seen
 
-    def get_nb_seen(self): # TODO REPLACE ME -> correlation image
+    def get_nb_seen(self): # TODO REPLACE ME -> correlation image chats
         return self.get_nb_correlation('item') + self.get_nb_correlation('message')
 
     def get_nb_seen_by_date(self, date):
@@ -127,6 +127,20 @@ class AbstractDaterangeObject(AbstractObject, ABC):
     def _add_create(self):
         r_object.sadd(f'{self.type}:all', self.id)
 
+    def _copy_from(self, obj_type, obj_id):
+        first_seen = r_object.hget(f'meta:{obj_type}:{obj_id}', 'first_seen')
+        last_seen = r_object.hget(f'meta:{obj_type}:{obj_id}', 'last_seen')
+        if first_seen and last_seen:
+            for date in Date.get_daterange(first_seen, last_seen):
+                nb = r_object.zscore(f'{obj_type}:date:{date}', self.id)
+                if nb:
+                    r_object.zincrby(f'{self.type}:date:{date}', nb, self.id)
+            update_obj_date(first_seen, self.type)
+            update_obj_date(last_seen, self.type)
+            self._add_create()
+            self.set_first_seen(first_seen)
+            self.set_last_seen(last_seen)
+
     def _add(self, date, obj): # TODO OBJ=None
         if not self.exists():
             self._add_create()

bin/modules/Global.py

@@ -132,7 +132,7 @@ class Global(AbstractModule):
             # TODO send to specific object queue => image, ...
             self.add_message_to_queue(obj=self.obj, queue='Item')
         elif self.obj.type == 'image':
-            self.add_message_to_queue(obj=self.obj, queue='Image')
+            self.add_message_to_queue(obj=self.obj, queue='Image', message=message)
         else:
             self.logger.critical(f"Empty obj: {self.obj} {message} not processed")
 

bin/modules/Mixer.py

@@ -218,7 +218,7 @@ class Mixer(AbstractModule):
             if self.obj.type == 'item':
                 self.add_message_to_queue(obj=self.obj, message=gzip64encoded)
             else:
-                self.add_message_to_queue(obj=self.obj)
+                self.add_message_to_queue(obj=self.obj, message=gzip64encoded)
 
 
 if __name__ == "__main__":

bin/modules/OcrExtractor.py

@@ -83,7 +83,7 @@ class OcrExtractor(AbstractModule):
 
     def compute(self, message):
         image = self.get_obj()
-        print(image.id)
+        date = message
 
         ocr = Ocrs.Ocr(image.id)
         if self.is_cached():
@@ -96,19 +96,24 @@ class OcrExtractor(AbstractModule):
         if not ocr.exists():
             path = image.get_filepath()
             languages = get_model_languages(image)
-            print(languages)
+            print(image.id, languages)
             texts = Ocrs.extract_text(path, languages)
             if texts:
                 print('create')
                 ocr = Ocrs.create(image.id, texts)
-                self.add_message_to_queue(ocr)
+                if ocr:
+                    self.add_message_to_queue(ocr)
+                else:
+                    print('no text')
+                    self.add_to_cache()
             # Save in cache
             else:
                 print('no text detected')
                 self.add_to_cache()
         else:
-            print('update correlation')
-            ocr.update_correlation()
+            # print(image.id)
+            # print('update correlation', date)
+            ocr.update_correlation(date=date)
 
 
 if __name__ == '__main__':

var/www/blueprints/objects_ocr.py

@@ -25,6 +25,8 @@ from lib import Language
 from lib import Tag
 from lib.objects import Ocrs
 
+from packages import Date
+
 # ============ BLUEPRINT ============
 objects_ocr = Blueprint('objects_ocr', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/objects/ocr'))
 
@@ -49,6 +51,48 @@ def ocr_image(filename):
     return send_file(BytesIO(ocr.draw_bounding_boxs()), mimetype='image/png')
 
 
+@objects_ocr.route("/objects/ocrs", methods=['GET'])
+@login_required
+@login_read_only
+def objects_ocrs():
+    date_from = request.args.get('date_from')
+    date_to = request.args.get('date_to')
+    show_objects = request.args.get('show_objects')
+    date = Date.sanitise_date_range(date_from, date_to)
+    date_from = date['date_from']
+    date_to = date['date_to']
+
+    if show_objects:
+        dict_objects = Ocrs.Ocrs().api_get_meta_by_daterange(date_from, date_to)
+    else:
+        dict_objects = {}
+
+    return render_template("OcrDaterange.html", date_from=date_from, date_to=date_to,
+                           dict_objects=dict_objects, show_objects=show_objects)
+
+
+@objects_ocr.route("/objects/ocrs/post", methods=['POST'])
+@login_required
+@login_read_only
+def objects_ocrs_post():
+    date_from = request.form.get('date_from')
+    date_to = request.form.get('date_to')
+    show_objects = request.form.get('show_objects')
+    return redirect(url_for('objects_ocr.objects_ocrs', date_from=date_from, date_to=date_to, show_objects=show_objects))
+
+
+@objects_ocr.route("/objects/ocrs/range/json", methods=['GET'])
+@login_required
+@login_read_only
+def objects_ocrs_range_json():
+    date_from = request.args.get('date_from')
+    date_to = request.args.get('date_to')
+    date = Date.sanitise_date_range(date_from, date_to)
+    date_from = date['date_from']
+    date_to = date['date_to']
+    return jsonify(Ocrs.Ocrs().api_get_chart_nb_by_daterange(date_from, date_to))
+
+
 @objects_ocr.route("/objects/ocr", methods=['GET'])
 @login_required
 @login_read_only

var/www/templates/objects/ocr/OcrDaterange.html

@@ -0,0 +1,602 @@
+<!DOCTYPE html>
+<html>
+
+<head>
+  <title>Ocrs - AIL</title>
+	<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
+
+  <!-- Core CSS -->
+	<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/daterangepicker.min.css') }}" rel="stylesheet">
+
+  <!-- JS -->
+	<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/moment.min.js') }}"></script>
+	<script src="{{ url_for('static', filename='js/jquery.daterangepicker.min.js') }}"></script>
+	<script src="{{ url_for('static', filename='js/d3.min.js') }}"></script>
+    <script src="{{ url_for('static', filename='js/d3/sparklines.js')}}"></script>
+
+	<style>
+		.input-group .form-control {
+			position: unset;
+		}
+		.line {
+  		fill: none;
+  		stroke: #000;
+  		stroke-width: 2.0px;
+		}
+		.bar {
+      fill: steelblue;
+    }
+    .bar:hover{
+      fill: brown;
+			cursor: pointer;
+    }
+		.bar_stack:hover{
+			cursor: pointer;
+    }
+		.pie_path:hover{
+			cursor: pointer;
+		}
+		.svgText {
+  		pointer-events: none;
+		}
+		div.tooltip {
+			position: absolute;
+			text-align: center;
+			padding: 2px;
+			font: 12px sans-serif;
+			background: #ebf4fb;
+			border: 2px solid #b7ddf2;
+			border-radius: 8px;
+			pointer-events: none;
+			color: #000000;
+		}
+	</style>
+</head>
+<body>
+
+	{% include 'nav_bar.html' %}
+
+	<div class="container-fluid">
+		<div class="row">
+
+			{% include 'sidebars/sidebar_objects.html' %}
+
+			<div class="col-12 col-lg-10" id="core_content">
+
+				<div class="row">
+					<div class="col-xl-10">
+						<div class="mt-1" id="barchart_type"></div>
+
+{#                        {% include 'image/block_images_search.html' %}#}
+
+                    </div>
+
+
+					<div class="col-xl-2">
+
+						<div class="card mb-3 mt-2" style="background-color:#d9edf7;">
+						  <div class="card-body text-center py-2">
+						    <h6 class="card-title" style="color:#286090;">Select a date range :</h6>
+								<form action="{{ url_for('objects_ocr.objects_ocrs_post') }}" id="hash_selector_form" method='post'>
+									<div class="input-group" id="date-range-from">
+										<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
+										<input class="form-control" id="date-range-from-input" placeholder="yyyy-mm-dd" value="{{ date_from }}" name="date_from" autocomplete="off">
+									</div>
+									<div class="input-group" id="date-range-to">
+										<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
+										<input class="form-control" id="date-range-to-input" placeholder="yyyy-mm-dd" value="{{ date_to }}" name="date_to" autocomplete="off">
+									</div>
+									<div class="form-check my-1">
+										<input class="form-check-input" type="checkbox" id="checkbox-input-show" name="show_objects" value="True" {% if show_objects %}checked{% endif %}>
+										<label class="form-check-label" for="checkbox-input-show">
+											<span style="color:#286090; font-size: 14px;">
+												Show Ocrs <i class="fas fa-key"></i>
+											</span>
+										</label>
+									</div>
+									<button class="btn btn-primary" style="text-align:center;">
+										<i class="fas fa-copy"></i> Search
+									</button>
+								</form>
+						  </div>
+						</div>
+
+						<div id="pie_chart_encoded">
+						</div>
+						<div id="pie_chart_top5_types">
+						</div>
+					</div>
+				</div>
+
+				{% if dict_objects %}
+                    {% if date_from|string == date_to|string %}
+                        <h3> {{ date_from }} Ocrs Name: </h3>
+                    {% else %}
+                        <h3> {{ date_from }} to {{ date_to }} Ocrs Name: </h3>
+                    {% endif %}
+                    <table id="tableb64" class="table table-striped table-bordered">
+                        <thead class="bg-dark text-white">
+                        <tr>
+                            <th></th>
+                            <th>First Seen</th>
+                            <th>Last Seen</th>
+                            <th>Total</th>
+                            <th>Last days</th>
+                        </tr>
+                        </thead>
+						<tbody style="font-size: 15px;">
+                        {% for obj_id in dict_objects %}
+                            <tr>
+                                <td><a target="_blank" href="{{ url_for('correlation.show_correlation') }}?type=ocr&id={{ obj_id }}">{{ dict_objects[obj_id]['id'] }}</a></td>
+                                <td>{{ dict_objects[obj_id]['first_seen'] }}</td>
+                                <td>{{ dict_objects[obj_id]['last_seen'] }}</td>
+                                <td>{{ dict_objects[obj_id]['nb_seen'] }}</td>
+                                <td id="sparklines_{{ obj_id }}" style="text-align:center;"></td>
+                            </tr>
+                        {% endfor %}
+                        </tbody>
+                    </table>
+
+
+                {% else %}
+                    {% if show_objects %}
+                        {% if date_from|string == date_to|string %}
+							<h3> {{ date_from }}, No OCR</h3>
+                        {% else %}
+							<h3> {{ date_from }} to {{ date_to }}, No OCR</h3>
+                        {% endif %}
+                    {% endif %}
+                {% endif %}
+            </div>
+
+        </div>
+    </div>
+
+<script>
+    var chart = {};
+    $(document).ready(function(){
+        $("#page-Decoded").addClass("active");
+        $("#nav_ocr").addClass("active");
+
+        $('#date-range-from').dateRangePicker({
+            separator : ' to ',
+            getValue: function()
+            {
+                if ($('#date-range-from-input').val() && $('#date-range-to').val() )
+                    return $('#date-range-from-input').val() + ' to ' + $('#date-range-to').val();
+                else
+                    return '';
+            },
+            setValue: function(s,s1,s2)
+            {
+                $('#date-range-from-input').val(s1);
+                $('#date-range-to-input').val(s2);
+            },
+        });
+        $('#date-range-to').dateRangePicker({
+            separator : ' to ',
+            getValue: function()
+            {
+                if ($('#date-range-from-input').val() && $('#date-range-to').val() )
+                    return $('#date-range-from-input').val() + ' to ' + $('#date-range-to').val();
+                else
+                    return '';
+            },
+            setValue: function(s,s1,s2)
+            {
+                $('#date-range-from-input').val(s1);
+                $('#date-range-to-input').val(s2);
+            },
+        });
+
+        $('#date-range-from').data('dateRangePicker').setDateRange('{{date_from}}','{{date_to}}');
+        $('#date-range-to').data('dateRangePicker').setDateRange('{{date_from}}','{{date_to}}');
+
+        $('#tableb64').DataTable({
+            "aLengthMenu": [[5, 10, 15, -1], [5, 10, 15, "All"]],
+            "iDisplayLength": 10,
+            "order": [[ 3, "desc" ]]
+        });
+
+
+        chart.stackBarChart = barchart_type_stack("{{ url_for('objects_ocr.objects_ocrs_range_json') }}?date_from={{date_from}}&date_to={{date_to}}", 'id');
+
+
+        chart.onResize();
+        $(window).on("resize", function() {
+            chart.onResize();
+        });
+  });
+
+function toggle_sidebar(){
+	if($('#nav_menu').is(':visible')){
+		$('#nav_menu').hide();
+		$('#side_menu').removeClass('border-right')
+		$('#side_menu').removeClass('col-lg-2')
+		$('#core_content').removeClass('col-lg-10')
+	}else{
+		$('#nav_menu').show();
+		$('#side_menu').addClass('border-right')
+		$('#side_menu').addClass('col-lg-2')
+		$('#core_content').addClass('col-lg-10')
+	}
+}
+</script>
+
+<script>
+	{% for obj_id in dict_objects %}
+    sparkline("sparklines_{{ obj_id }}", {{ dict_objects[obj_id]['sparkline'] }}, {});
+	{% endfor %}
+</script>
+
+<script>
+var margin = {top: 20, right: 100, bottom: 55, left: 45},
+				width  = 1000 - margin.left - margin.right,
+				height = 500  - margin.top  - margin.bottom;
+var x = d3.scaleBand().rangeRound([0, width]).padding(0.1);
+
+var y = d3.scaleLinear().rangeRound([height, 0]);
+
+var xAxis = d3.axisBottom(x);
+
+var yAxis = d3.axisLeft(y);
+
+var color = d3.scaleOrdinal(d3.schemeSet3);
+
+var svg = d3.select("#barchart_type").append("svg")
+				.attr("id", "thesvg")
+				.attr("viewBox", "0 0 1000 500")
+				.attr("width",  width  + margin.left + margin.right)
+				.attr("height", height + margin.top  + margin.bottom)
+			.append("g")
+				.attr("transform", "translate(" + margin.left + "," + margin.top + ")");
+
+function barchart_type_stack(url, id) {
+
+  d3.json(url)
+    .then(function(data){
+
+			var labelVar = 'date';  //A
+		  var varNames = d3.keys(data[0])
+		      .filter(function (key) { return key !== labelVar;}); //B
+
+		  data.forEach(function (d) { //D
+		    var y0 = 0;
+		    d.mapping = varNames.map(function (name) {
+		      return {
+		        name: name,
+		        label: d[labelVar],
+		        y0: y0,
+		        y1: y0 += +d[name]
+		      };
+		    });
+		    d.total = d.mapping[d.mapping.length - 1].y1;
+		  });
+
+		  x.domain(data.map(function (d) { return (d.date); })); //E
+		  y.domain([0, d3.max(data, function (d) { return d.total; })]);
+
+			svg.append("g")
+        .attr("class", "x axis")
+        .attr("transform", "translate(0," + height + ")")
+        .call(xAxis)
+				.selectAll("text")
+					.attr("class", "bar")
+					{% if  date_from|string == date_to|string and type is none %}
+						.on("click", function (d) { window.location.href = "{{ url_for('objects_ocr.objects_ocrs') }}?date_from={{date_from}}&date_to={{date_to}}&type_id="+d })
+						.attr("transform", "rotate(-18)" )
+					{% elif  date_from|string == date_to|string and type is not none %}
+						.on("click", function (d) { window.location.href = "{{ url_for('objects_ocr.objects_ocrs') }}?date_from="+d+'&date_to='+d })
+						.attr("transform", "rotate(-18)" )
+					{% else %}
+						.on("click", function (d) { window.location.href = "{{ url_for('objects_ocr.objects_ocrs') }}?date_from="+d+'&date_to='+d })
+						.attr("transform", "rotate(-40)" )
+					{% endif %}
+					.style("text-anchor", "end");
+
+    	svg.append("g")
+        .attr("class", "y axis")
+        .call(yAxis)
+      	.append("text")
+        	.attr("transform", "rotate(-90)")
+        	.attr("y", 6)
+        	.attr("dy", ".71em")
+        	.style("text-anchor", "end");
+
+			var selection = svg.selectAll(".series")
+		    .data(data)
+		    .enter().append("g")
+		      .attr("class", "series")
+		      .attr("transform", function (d) { return "translate(" + x((d.date)) + ",0)"; });
+
+			selection.selectAll("rect")
+	      .data(function (d) { return d.mapping; })
+	    .enter().append("rect")
+				.attr("class", "bar_stack")
+	      .attr("width", x.bandwidth())
+	      .attr("y", function (d) { return y(d.y1); })
+	      .attr("height", function (d) { return y(d.y0) - y(d.y1); })
+	      .style("fill", function (d) { return color(d.name); })
+	      .style("stroke", "grey")
+	      .on("mouseover", function (d) { showPopover.call(this, d); })
+				.on("mouseout", function (d) { removePopovers(); })
+				{% if  date_from|string == date_to|string and type is none %}
+					.on("click", function(d){ window.location.href = "{{ url_for('objects_ocr.objects_ocrs') }}" +'?date_from={{date_from}}&date_to={{date_to}}&type_id='+d.label+'&encoding='+d.name; });
+				{% elif  date_from|string == date_to|string and type is not none %}
+					.on("click", function(d){ window.location.href = "{{ url_for('objects_ocr.objects_ocrs') }}" +'?type_id={{type_id}}&date_from='+d.label+'&date_to='+d.label+'&encoding='+d.name; });
+				{% else %}
+					.on("click", function(d){ window.location.href = "{{ url_for('objects_ocr.objects_ocrs') }}" +'?type_id='+ d.name +'&date_from='+d.label+'&date_to='+d.label; });
+				{% endif %}
+
+				data.forEach(function(d) {
+					if(d.total !== 0){
+						svg.append("text")
+								.attr("class", "bar")
+								.attr("dy", "-.35em")
+								.attr('x', x(d.date) + x.bandwidth()/2)
+								.attr('y', y(d.total))
+								{% if  date_from|string == date_to|string and type is none %}
+									.on("click", function () {window.location.href = "{{ url_for('objects_ocr.objects_ocrs') }}"+'?date_from={{date_from}}&date_to={{date_to}}&type_id='+d.date })
+								{% elif  date_from|string == date_to|string and type is not none %}
+									.on("click", function () {window.location.href = "{{ url_for('objects_ocr.objects_ocrs') }}?type_id={{type_id}}&date_from="+d.date+'&date_to='+d.date })
+								{% else %}
+									.on("click", function () {window.location.href = "{{ url_for('objects_ocr.objects_ocrs') }}"+'?date_from='+d.date+'&date_to='+d.date })
+								{% endif %}
+								.style("text-anchor", "middle")
+								.text(d.total);
+					}
+				});
+
+			drawLegend(varNames);
+    });
+
+}
+
+function drawLegend (varNames) {
+    var legend = svg.selectAll(".legend")
+        .data(varNames.slice().reverse())
+      .enter().append("g")
+        .attr("class", "legend")
+        .attr("transform", function (d, i) { return "translate(0," + i * 20 + ")"; });
+
+    legend.append("rect")
+        .attr("x", 943)
+        .attr("width", 10)
+        .attr("height", 10)
+        .style("fill", color)
+        .style("stroke", "grey");
+
+    legend.append("text")
+				.attr("class", "svgText")
+        .attr("x", 941)
+        .attr("y", 6)
+        .attr("dy", ".35em")
+        .style("text-anchor", "end")
+        .text(function (d) { return d; });
+}
+
+function removePopovers () {
+    $('.popover').each(function() {
+      $(this).remove();
+    });
+  }
+
+function showPopover (d) {
+  $(this).popover({
+    title: "<b><span id='tooltip-id-name-bar'></span></b>",
+    placement: 'top',
+    container: 'body',
+    trigger: 'manual',
+    html : true,
+    content: function() {
+      return "<span id='tooltip-id-label'></span>" +
+            "<br/>num: <span id='tooltip-id-value-bar'></span>"; }
+  });
+  $(this).popover('show');
+  $("#tooltip-id-name-bar").text(d.name);
+  $("#tooltip-id-label").text(d.label);
+  $("#tooltip-id-value-bar").text(d3.format(",")(d.value ? d.value: d.y1 - d.y0));
+}
+
+chart.onResize = function () {
+    var aspect = 1000 / 500, chart = $("#thesvg");
+    var targetWidth = chart.parent().width();
+    chart.attr("width", targetWidth);
+    chart.attr("height", targetWidth / aspect);
+  }
+
+window.chart = chart;
+
+</script>
+
+<script>
+function draw_pie_chart(id, url_json, pie_on_click_url) {
+
+		var width_pie = 200;
+		var height_pie = 200;
+		var padding_pie = 10;
+		var opacity_pie = .8;
+
+		var radius_pie = Math.min(width_pie - padding_pie, height_pie - padding_pie) / 2;
+		//var color_pie = d3.scaleOrdinal(d3.schemeCategory10);
+		var color_pie = d3.scaleOrdinal(d3.schemeSet3);
+
+		var div_pie = d3.select("body").append("div")
+				.attr("class", "tooltip")
+				.style("opacity", 0);
+
+		var svg_pie = d3.select("#"+id)
+				.append('svg')
+				.attr("width", '100%')
+				.attr("height", '100%')
+				.attr('viewBox','0 0 '+Math.min(width_pie,height_pie) +' '+Math.min(width_pie,height_pie) )
+				.attr('preserveAspectRatio','xMinYMin')
+
+
+		var g_pie = svg_pie.append('g')
+				.attr('transform', 'translate(' + (width_pie/2) + ',' + (height_pie/2) + ')');
+
+		var arc_pie = d3.arc()
+				.innerRadius(0)
+				.outerRadius(radius_pie);
+
+		d3.json(url_json)
+			.then(function(data){
+
+			var pie_pie = d3.pie()
+					.value(function(d) { return d.value; })
+					.sort(null);
+
+			var path_pie = g_pie.selectAll('path')
+				  .data(pie_pie(data))
+				  .enter()
+				  .append("g")
+				  .append('path')
+				  .attr('d', arc_pie)
+				  .attr('fill', (d,i) => color_pie(i))
+					.attr('class', 'pie_path')
+					.on("mouseover", mouseovered_pie)
+					.on("mouseout", mouseouted_pie)
+					.on("click", function (d) {window.location.href = pie_on_click_url+d.data.name })
+				  .style('opacity', opacity_pie)
+				  .style('stroke', 'white');
+			});
+
+
+		function mouseovered_pie(d) {
+      //remove old content
+      $("#tooltip-id-name").remove();
+      $("#tooltip-id-value").remove();
+
+			// tooltip
+			var content;
+
+			content = "<b><span id='tooltip-id-name'></span></b><br/>"+
+								"<br/>"+
+								"<i>Decoded</i>: <span id='tooltip-id-value'></span><br/>"
+
+			div_pie.transition()
+				.duration(200)
+				.style("opacity", .9);
+			div_pie.html(content)
+				.style("left", (d3.event.pageX) + "px")
+				.style("top", (d3.event.pageY - 28) + "px");
+
+      $("#tooltip-id-name").text(d.data.name);
+      $("#tooltip-id-value").text(d.data.value);
+		}
+
+		function mouseouted_pie() {
+			div_pie.transition()
+				.duration(500)
+				.style("opacity", 0);
+		}
+}
+
+</script>
+
+
+
+<script>
+function barchart_type(url, id) {
+
+
+  var margin = {top: 20, right: 20, bottom: 70, left: 40};
+
+  var width = 960 - margin.left - margin.right;
+  var height = 500 - margin.top - margin.bottom;
+
+  var x = d3.scaleBand().rangeRound([0, width]).padding(0.1);
+  var y = d3.scaleLinear().rangeRound([height, 0]);
+
+  var xAxis = d3.axisBottom(x)
+    //.tickFormat(d3.time.format("%Y-%m"));
+
+  var yAxis = d3.axisLeft(y)
+    .ticks(10);
+
+/*var svg = d3.select(id).append("svg")
+      .attr("width", width + margin.left + margin.right)
+      .attr("height", height + margin.top + margin.bottom)
+			.attr("id", "thesvg")
+      .append("g")
+        .attr("transform",
+          "translate(" + margin.left + "," + margin.top + ")");*/
+
+
+  d3.json(url)
+    .then(function(data){
+
+        data.forEach(function(d) {
+          d.value = +d.value;
+        });
+
+				x.domain(data.map(function(d) { return d.date; }));
+        y.domain([0, d3.max(data, function(d) { return d.value; })]);
+
+        var label = svg.append("g")
+            .attr("class", "x axis")
+            .attr("transform", "translate(0," + height + ")")
+            .call(xAxis)
+          .selectAll("text")
+            .style("text-anchor", "end")
+            .attr("dx", "-.8em")
+            .attr("dy", "-.55em")
+						{% if  daily_type_chart %}
+						.attr("transform", "rotate(-20)" );
+						{% else %}
+						.attr("transform", "rotate(-70)" )
+						.attr("class", "bar")
+						{% endif %}
+
+        svg.append("g")
+            .attr("class", "y axis")
+            .call(yAxis)
+          .append("text")
+            .attr("transform", "rotate(-90)")
+            .attr("y", 6)
+            .attr("dy", ".71em")
+            .style("text-anchor", "end")
+            .text("Value ($)");
+
+        var bar = svg.selectAll("bar")
+            .data(data)
+          .enter().append("rect")
+            .attr("class", "bar")
+            //.style("fill", "steelblue")
+						.attr("x", function(d) { return x(d.date); })
+            .attr("width", x.bandwidth())
+            .attr("y", function(d) { return y(d.value); })
+            .attr("height", function(d) { return height - y(d.value); })
+
+
+        data.forEach(function(d) {
+          if(d.value != 0){
+            svg.append("text")
+                .attr("class", "bar")
+                .attr("dy", "-.35em")
+                //.text(function(d) { return d.value; });
+                .text(d.value)
+								.style("text-anchor", "middle")
+								.attr('x', x(d.date) + x.bandwidth()/2)
+                .attr('y', y(d.value));
+          }
+        });
+
+    });
+
+}
+</script>
+
+
+</body>
+
+</html>

var/www/templates/sidebars/sidebar_objects.html

@@ -82,6 +82,12 @@
                       <span>Image</span>
                   </a>
               </li>
+              <li class="nav-item">
+                  <a class="nav-link" href="{{url_for('objects_ocr.objects_ocrs')}}" id="nav_ocr">
+                      <i class="fas fa-expand"></i>
+                      <span>OCR</span>
+                  </a>
+              </li>
               <li class="nav-item">
                   <a class="nav-link" href="{{url_for('objects_title.objects_titles')}}" id="nav_title">
                       <i class="fas fa-heading"></i>