chg: [username] add username search

bin/lib/objects/Usernames.py

@@ -13,7 +13,7 @@ sys.path.append(os.environ['AIL_BIN'])
 # Import Project packages
 ##################################
 from lib.ConfigLoader import ConfigLoader
-from lib.objects.abstract_subtype_object import AbstractSubtypeObject, get_all_id
+from lib.objects.abstract_subtype_object import AbstractSubtypeObject, AbstractSubtypeObjects, get_all_id
 
 config_loader = ConfigLoader()
 baseurl = config_loader.get_config_str("Notifications", "ail_domain")
@@ -65,7 +65,7 @@ class Username(AbstractSubtypeObject):
         meta = self._get_meta(options=options)
         meta['id'] = self.id
         meta['subtype'] = self.subtype
-        meta['tags'] = self.get_tags(r_list=True)
+        meta['tags'] = self.get_tags(r_list=True) # TODO NB Chats
         return meta
 
     def get_misp_object(self):
@@ -136,7 +136,20 @@ def search_usernames_by_name(name_to_search, subtype, r_pos=False):
     return usernames
 
 
+class Usernames(AbstractSubtypeObjects):
+    """
+        Usernames Objects
+    """
+    def __init__(self):
+        super().__init__('username', Username)
+
+    def sanitize_id_to_search(self, subtypes, name_to_search):
+        return name_to_search
+
+
 # if __name__ == '__main__':
 #     name_to_search = 'co'
-#     subtype = 'telegram'
+#     subtypes = ['telegram']
-#     print(search_usernames_by_name(name_to_search, subtype))
+#     u = Usernames()
+#     r = u.search_by_id(name_to_search, subtypes, r_pos=True)
+#     print(r)

bin/lib/objects/abstract_subtype_object.py

@@ -7,8 +7,9 @@ Base Class for AIL Objects
 # Import External packages
 ##################################
 import os
+import re
 import sys
-from abc import ABC
+from abc import ABC, abstractmethod
 
 # from flask import url_for
 
@@ -192,6 +193,62 @@ class AbstractSubtypeObject(AbstractObject, ABC):
     def _delete(self):
         pass
 
+
+class AbstractSubtypeObjects(ABC):
+    """
+    Abstract Subtype Objects
+    """
+    def __init__(self, obj_type, obj_class):
+        self.type = obj_type
+        self.obj_class = obj_class
+
+    def get_ids(self): # TODO FORMAT
+        ids = []
+        for subtype in get_object_all_subtypes(self.type):
+            ids += r_object.zrange(f'{self.type}_all:{subtype}', 0, -1)
+        return ids
+
+    def get_id_iterators_by_subtype(self, subtype):
+        return zscan_iter(r_object, f'{self.type}_all:{subtype}')
+
+    def get_metas(self, subtype, obj_ids, options=set()):
+        dict_obj = {}
+        for obj_id in obj_ids:
+            obj = self.obj_class(obj_id, subtype)
+            dict_obj[obj_id] = obj.get_meta(options=options)
+        return dict_obj
+
+    @abstractmethod
+    def sanitize_id_to_search(self, subtypes, id_to_search):
+        return id_to_search
+
+    # TODO
+    def search_by_id(self, name_to_search, subtypes=[], r_pos=False, case_sensitive=True):
+        objs = {}
+        if case_sensitive:
+            flags = 0
+        else:
+            flags = re.IGNORECASE
+        # for subtype in subtypes:
+        r_name = self.sanitize_id_to_search(subtypes, name_to_search)
+        if not name_to_search or isinstance(r_name, dict):
+            return objs
+        r_name = re.compile(r_name, flags=flags)
+        for subtype in subtypes:
+            for obj_id in self.get_id_iterators_by_subtype(subtype):
+                obj_id = obj_id[0]
+                res = re.search(r_name, obj_id)
+                if res:
+                    objs[obj_id] = {}
+                    if r_pos:
+                        objs[obj_id]['hl-start'] = res.start()
+                        objs[obj_id]['hl-end'] = res.end()
+        return objs
+
+########################################################################
+########################################################################
+########################################################################
+
 def get_all_id(obj_type, subtype):
     return r_object.zrange(f'{obj_type}_all:{subtype}', 0, -1)
 
@@ -246,3 +303,5 @@ def get_subtypes_objs_range_json(obj_type, date_from, date_to):
                 objs_range.append(day_dict)
 
     return objs_range
+
+

var/www/blueprints/chats_explorer.py

@@ -49,7 +49,7 @@ def chats_explorer_dashboard():
 @login_read_only
 def chats_explorer_protocols():
     protocols = chats_viewer.get_chat_protocols_meta()
-    return render_template('chats_protocols.html', protocols=protocols)
+    return render_template('chats_protocols.html', protocols=protocols, username_subtypes=ail_core.get_object_all_subtypes('username'))
 
 @chats_explorer.route("chats/explorer/networks", methods=['GET'])
 @login_required

var/www/blueprints/objects_subtypes.py

@@ -23,6 +23,7 @@ from lib import ail_core
 from lib.objects import abstract_subtype_object
 from lib.objects import ail_objects
 from lib.objects import CryptoCurrencies
+from lib.objects import Usernames
 from packages import Date
 
 # ============ BLUEPRINT ============
@@ -117,6 +118,43 @@ def objects_dashboard_pgp():
 def objects_dashboard_username():
     return subtypes_objects_dashboard('username', request)
 
+@objects_subtypes.route("/objects/usernames/search", methods=['GET', 'POST'])
+@login_required
+@login_read_only
+def objects_username_search():
+    if request.method == 'POST':
+        to_search = request.form.get('to_search')
+        subtype = request.form.get('search_subtype')
+        page = request.form.get('page', 1)
+        try:
+            page = int(page)
+        except (TypeError, ValueError):
+            page = 1
+        return redirect(url_for('objects_subtypes.objects_username_search', search=to_search, page=page, subtype=subtype))
+    else:
+        to_search = request.args.get('search')
+        subtype = request.args.get('subtype')  # TODO sanityze
+        page = request.args.get('page', 1)
+        try:
+            page = int(page)
+        except (TypeError, ValueError):
+            page = 1
+
+        usernames = Usernames.Usernames()
+        search_result = usernames.search_by_id(to_search, [subtype], page)
+
+        if search_result:
+            ids = sorted(search_result.keys())
+            dict_page = ail_core.paginate_iterator(ids, nb_obj=500, page=page)
+            dict_objects = usernames.get_metas(subtype, dict_page['list_elem'], options={'icon', 'sparkline'})  # TODO OPTIONS
+        else:
+            dict_objects = {}
+            dict_page = {}
+
+        return render_template("username/search_usernames_result.html", dict_objects=dict_objects, search_result=search_result,
+                               dict_page=dict_page, subtypes=ail_core.get_object_all_subtypes('username'),
+                               to_search=to_search, subtype=subtype)
+
 @objects_subtypes.route("/objects/user-accounts", methods=['GET'])
 @login_required
 @login_read_only

var/www/templates/chats_explorer/chats_protocols.html

@@ -70,6 +70,12 @@
                     </div>
                 </div>
 
+                <br>
+                <br>
+                {% with subtypes=username_subtypes %}
+                    {% include 'username/block_usernames_search.html' %}
+                {% endwith %}
+
             </div>
 
         </div>

var/www/templates/objects/subtypes_objs_dashboard.html

@@ -76,6 +76,12 @@
 						<div class="mt-1" id="barchart_type">
 						</div>
 
+                        {% if obj_type == 'username' %}
+                            {% with subtypes=subtypes %}
+                                {% include 'username/block_usernames_search.html' %}
+                            {% endwith %}
+                        {% else %}
+
                             <div class="card border-secondary my-2">
                                 <div class="card-body text-dark">
                                     <h5 class="card-title">Search {{obj_type}} by name:</h5>
@@ -98,6 +104,7 @@
                                 </form>
                               </div>
                             </div>
+                        {% endif %}
 
 					</div>
 

var/www/templates/objects/username/block_usernames_search.html

@@ -0,0 +1,21 @@
+<div class="card border-secondary my-2">
+    <div class="card-body text-dark">
+        <h5 class="card-title">Usernames Search:</h5>
+        <form action="{{ url_for('objects_subtypes.objects_username_search') }}" id="search_subtype_onj" method='post'>
+            <div class="input-group mb-1">
+                <input type="text" name="page" value="{% if page %}{{ page }}{% else %}1{% endif %}" hidden="">
+                <select class="custom-select col-2" name="search_subtype" required>
+                    {% for s in subtypes %}
+                        <option value="{{ s }}" {% if s == subtype %}selected{% endif %}>{{ s }}</option>
+                    {% endfor %}
+                </select>
+                <input type="text" class="form-control col-8" name="to_search" value="{% if to_search %}{{ to_search }}{% endif %}" placeholder="Username to Search" required>
+                <button class="btn btn-primary input-group-addon search-obj col-2"><i class="fas fa-search"></i></button>
+            </div>
+            <div class="custom-control custom-switch mt-1">
+                <input class="custom-control-input" type="checkbox" name="case_sensitive" id="case_sensitive" {% if type_to_search %}{% if case_sensitive %}value="True" checked{% else %}value="False"{% endif %}{% else %}value="True" checked{% endif %}>
+                <label class="custom-control-label" for="case_sensitive">Case Sensitive</label>
+            </div>
+        </form>
+    </div>
+</div>

var/www/templates/objects/username/search_usernames_result.html

@@ -0,0 +1,130 @@
+<!DOCTYPE html>
+<html>
+
+<head>
+  <title>Usernames Search - AIL</title>
+	<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
+
+  <!-- Core CSS -->
+	<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
+    <link href="{{ url_for('static', filename='css/ail-project.css') }}" rel="stylesheet">
+
+  <!-- JS -->
+	<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/d3.min.js') }}"></script>
+    <script src="{{ url_for('static', filename='js/d3/sparklines.js')}}"></script>
+
+</head>
+<body>
+
+	{% include 'nav_bar.html' %}
+
+	<div class="container-fluid">
+		<div class="row">
+
+			{% include 'sidebars/sidebar_objects.html' %}
+
+			<div class="col-12 col-lg-10" id="core_content">
+
+                {% with page=dict_page['page'], subtypes=subtypes, subtype=subtype %}
+				    {% include 'username/block_usernames_search.html' %}
+                {% endwith %}
+
+
+				<table id="table_objects" class="table table-striped table-bordered">
+                    <thead class="bg-dark text-white">
+                    <tr>
+                        <th>Subtype</th>
+                        <th>ID</th>
+                        <th>First Seen</th>
+                        <th>Last Seen</th>
+                        <th>NB Seen</th>
+                        <th>Sparkline</th>
+                    </tr>
+                    </thead>
+                    <tbody style="font-size: 15px;">
+                    {% for obj_id in dict_objects %}
+                        <tr>
+                            <td>
+                                {% with style=dict_objects[obj_id]['icon']['style'], icon=dict_objects[obj_id]['icon']['icon'] , color=dict_objects[obj_id]['icon']['color'] %}
+                                    {% include 'objects/obj_svg_block.html' %}
+                                {% endwith %}
+                                 {{ dict_objects[obj_id]['subtype'] }}
+                            </td>
+                            <td>
+                                <a target="_blank" href="{{ url_for('correlation.show_correlation') }}?type=username&subtype=telegram&id={{ obj_id }}">
+                                    <span>{{ dict_objects[obj_id]['id'][:search_result[obj_id]['hl-start']] }}</span><span class="hg-text">{{dict_objects[obj_id]['id'][search_result[obj_id]['hl-start']:search_result[obj_id]['hl-end']]}}</span>{{ dict_objects[obj_id]['id'][search_result[obj_id]['hl-end']:] }}
+                                </a>
+                            </td>
+                            <td>
+                                {% if dict_objects[obj_id]['first_seen'] %}
+                                    {{ dict_objects[obj_id]['first_seen'][0:4] }}-{{ dict_objects[obj_id]['first_seen'][4:6] }}-{{ dict_objects[obj_id]['first_seen'][6:8] }}
+                                {% endif %}
+                            </td>
+                            <td>
+                                {% if dict_objects[obj_id]['last_seen'] %}
+                                    {{ dict_objects[obj_id]['last_seen'][0:4] }}-{{ dict_objects[obj_id]['last_seen'][4:6] }}-{{ dict_objects[obj_id]['last_seen'][6:8] }}
+                                {% endif %}
+                            </td>
+                            <td>{{ dict_objects[obj_id]['nb_seen'] }}
+                            </td>
+                            <td id="sparklines_{{ obj_id }}" style="text-align:center;"></td>
+                        </tr>
+                    {% endfor %}
+                    </tbody>
+                </table>
+
+                {% if dict_page %}
+                    {% with page=dict_page['page'], nb_page_max=dict_page['nb_pages'],  nb_first_elem=dict_page['nb_first_elem'], nb_last_elem=dict_page['nb_last_elem'], nb_all_elem=dict_page['nb_all_elem'] %}
+                        {% set target_url=url_for('objects_subtypes.objects_username_search') + "?search=" + to_search + "&subtype=" + subtype + "&case_sensitive=" + case_sensitive|string %}
+                        {% include 'pagination.html' %}
+                    {% endwith %}
+                {% endif %}
+
+            </div>
+        </div>
+    </div>
+
+<script>
+    var chart = {};
+    $(document).ready(function(){
+        $("#page-Decoded").addClass("active");
+        $("#nav_username").addClass("active");
+
+        $('#table_objects').DataTable({
+            "aLengthMenu": [[5, 10, 15, -1], [5, 10, 15, "All"]],
+            "iDisplayLength": 10,
+            "order": [[ 3, "desc" ]]
+        });
+  });
+
+function toggle_sidebar(){
+	if($('#nav_menu').is(':visible')){
+		$('#nav_menu').hide();
+		$('#side_menu').removeClass('border-right')
+		$('#side_menu').removeClass('col-lg-2')
+		$('#core_content').removeClass('col-lg-10')
+	}else{
+		$('#nav_menu').show();
+		$('#side_menu').addClass('border-right')
+		$('#side_menu').addClass('col-lg-2')
+		$('#core_content').addClass('col-lg-10')
+	}
+}
+</script>
+
+<script>
+	{% for obj_id in dict_objects %}
+    sparkline("sparklines_{{ obj_id }}", {{ dict_objects[obj_id]['sparkline'] }}, {});
+	{% endfor %}
+</script>
+
+
+</body>
+</html>