mirror of https://github.com/CIRCL/AIL-framework
chg: [username] add username search
terrtia
parent
31e3da5e53
commit
4a6ab3caa1
|
|
@ -13,7 +13,7 @@ sys.path.append(os.environ['AIL_BIN'])
|
|||
# Import Project packages
|
||||
##################################
|
||||
from lib.ConfigLoader import ConfigLoader
|
||||
from lib.objects.abstract_subtype_object import AbstractSubtypeObject, get_all_id
|
||||
from lib.objects.abstract_subtype_object import AbstractSubtypeObject, AbstractSubtypeObjects, get_all_id
|
||||
|
||||
config_loader = ConfigLoader()
|
||||
baseurl = config_loader.get_config_str("Notifications", "ail_domain")
|
||||
|
|
@ -65,7 +65,7 @@ class Username(AbstractSubtypeObject):
|
|||
meta = self._get_meta(options=options)
|
||||
meta['id'] = self.id
|
||||
meta['subtype'] = self.subtype
|
||||
meta['tags'] = self.get_tags(r_list=True)
|
||||
meta['tags'] = self.get_tags(r_list=True) # TODO NB Chats
|
||||
return meta
|
||||
|
||||
def get_misp_object(self):
|
||||
|
|
@ -136,7 +136,20 @@ def search_usernames_by_name(name_to_search, subtype, r_pos=False):
|
|||
return usernames
|
||||
|
||||
|
||||
class Usernames(AbstractSubtypeObjects):
|
||||
"""
|
||||
Usernames Objects
|
||||
"""
|
||||
def __init__(self):
|
||||
super().__init__('username', Username)
|
||||
|
||||
def sanitize_id_to_search(self, subtypes, name_to_search):
|
||||
return name_to_search
|
||||
|
||||
|
||||
# if __name__ == '__main__':
|
||||
# name_to_search = 'co'
|
||||
# subtype = 'telegram'
|
||||
# print(search_usernames_by_name(name_to_search, subtype))
|
||||
# subtypes = ['telegram']
|
||||
# u = Usernames()
|
||||
# r = u.search_by_id(name_to_search, subtypes, r_pos=True)
|
||||
# print(r)
|
||||
|
|
|
|||
|
|
@ -7,8 +7,9 @@ Base Class for AIL Objects
|
|||
# Import External packages
|
||||
##################################
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from abc import ABC
|
||||
from abc import ABC, abstractmethod
|
||||
|
||||
# from flask import url_for
|
||||
|
||||
|
|
@ -192,6 +193,62 @@ class AbstractSubtypeObject(AbstractObject, ABC):
|
|||
def _delete(self):
|
||||
pass
|
||||
|
||||
|
||||
class AbstractSubtypeObjects(ABC):
|
||||
"""
|
||||
Abstract Subtype Objects
|
||||
"""
|
||||
def __init__(self, obj_type, obj_class):
|
||||
self.type = obj_type
|
||||
self.obj_class = obj_class
|
||||
|
||||
def get_ids(self): # TODO FORMAT
|
||||
ids = []
|
||||
for subtype in get_object_all_subtypes(self.type):
|
||||
ids += r_object.zrange(f'{self.type}_all:{subtype}', 0, -1)
|
||||
return ids
|
||||
|
||||
def get_id_iterators_by_subtype(self, subtype):
|
||||
return zscan_iter(r_object, f'{self.type}_all:{subtype}')
|
||||
|
||||
def get_metas(self, subtype, obj_ids, options=set()):
|
||||
dict_obj = {}
|
||||
for obj_id in obj_ids:
|
||||
obj = self.obj_class(obj_id, subtype)
|
||||
dict_obj[obj_id] = obj.get_meta(options=options)
|
||||
return dict_obj
|
||||
|
||||
@abstractmethod
|
||||
def sanitize_id_to_search(self, subtypes, id_to_search):
|
||||
return id_to_search
|
||||
|
||||
# TODO
|
||||
def search_by_id(self, name_to_search, subtypes=[], r_pos=False, case_sensitive=True):
|
||||
objs = {}
|
||||
if case_sensitive:
|
||||
flags = 0
|
||||
else:
|
||||
flags = re.IGNORECASE
|
||||
# for subtype in subtypes:
|
||||
r_name = self.sanitize_id_to_search(subtypes, name_to_search)
|
||||
if not name_to_search or isinstance(r_name, dict):
|
||||
return objs
|
||||
r_name = re.compile(r_name, flags=flags)
|
||||
for subtype in subtypes:
|
||||
for obj_id in self.get_id_iterators_by_subtype(subtype):
|
||||
obj_id = obj_id[0]
|
||||
res = re.search(r_name, obj_id)
|
||||
if res:
|
||||
objs[obj_id] = {}
|
||||
if r_pos:
|
||||
objs[obj_id]['hl-start'] = res.start()
|
||||
objs[obj_id]['hl-end'] = res.end()
|
||||
return objs
|
||||
|
||||
########################################################################
|
||||
########################################################################
|
||||
########################################################################
|
||||
|
||||
def get_all_id(obj_type, subtype):
|
||||
return r_object.zrange(f'{obj_type}_all:{subtype}', 0, -1)
|
||||
|
||||
|
|
@ -246,3 +303,5 @@ def get_subtypes_objs_range_json(obj_type, date_from, date_to):
|
|||
objs_range.append(day_dict)
|
||||
|
||||
return objs_range
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -49,7 +49,7 @@ def chats_explorer_dashboard():
|
|||
@login_read_only
|
||||
def chats_explorer_protocols():
|
||||
protocols = chats_viewer.get_chat_protocols_meta()
|
||||
return render_template('chats_protocols.html', protocols=protocols)
|
||||
return render_template('chats_protocols.html', protocols=protocols, username_subtypes=ail_core.get_object_all_subtypes('username'))
|
||||
|
||||
@chats_explorer.route("chats/explorer/networks", methods=['GET'])
|
||||
@login_required
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ from lib import ail_core
|
|||
from lib.objects import abstract_subtype_object
|
||||
from lib.objects import ail_objects
|
||||
from lib.objects import CryptoCurrencies
|
||||
from lib.objects import Usernames
|
||||
from packages import Date
|
||||
|
||||
# ============ BLUEPRINT ============
|
||||
|
|
@ -117,6 +118,43 @@ def objects_dashboard_pgp():
|
|||
def objects_dashboard_username():
|
||||
return subtypes_objects_dashboard('username', request)
|
||||
|
||||
@objects_subtypes.route("/objects/usernames/search", methods=['GET', 'POST'])
|
||||
@login_required
|
||||
@login_read_only
|
||||
def objects_username_search():
|
||||
if request.method == 'POST':
|
||||
to_search = request.form.get('to_search')
|
||||
subtype = request.form.get('search_subtype')
|
||||
page = request.form.get('page', 1)
|
||||
try:
|
||||
page = int(page)
|
||||
except (TypeError, ValueError):
|
||||
page = 1
|
||||
return redirect(url_for('objects_subtypes.objects_username_search', search=to_search, page=page, subtype=subtype))
|
||||
else:
|
||||
to_search = request.args.get('search')
|
||||
subtype = request.args.get('subtype') # TODO sanityze
|
||||
page = request.args.get('page', 1)
|
||||
try:
|
||||
page = int(page)
|
||||
except (TypeError, ValueError):
|
||||
page = 1
|
||||
|
||||
usernames = Usernames.Usernames()
|
||||
search_result = usernames.search_by_id(to_search, [subtype], page)
|
||||
|
||||
if search_result:
|
||||
ids = sorted(search_result.keys())
|
||||
dict_page = ail_core.paginate_iterator(ids, nb_obj=500, page=page)
|
||||
dict_objects = usernames.get_metas(subtype, dict_page['list_elem'], options={'icon', 'sparkline'}) # TODO OPTIONS
|
||||
else:
|
||||
dict_objects = {}
|
||||
dict_page = {}
|
||||
|
||||
return render_template("username/search_usernames_result.html", dict_objects=dict_objects, search_result=search_result,
|
||||
dict_page=dict_page, subtypes=ail_core.get_object_all_subtypes('username'),
|
||||
to_search=to_search, subtype=subtype)
|
||||
|
||||
@objects_subtypes.route("/objects/user-accounts", methods=['GET'])
|
||||
@login_required
|
||||
@login_read_only
|
||||
|
|
|
|||
|
|
@ -70,6 +70,12 @@
|
|||
</div>
|
||||
</div>
|
||||
|
||||
<br>
|
||||
<br>
|
||||
{% with subtypes=username_subtypes %}
|
||||
{% include 'username/block_usernames_search.html' %}
|
||||
{% endwith %}
|
||||
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -76,28 +76,35 @@
|
|||
<div class="mt-1" id="barchart_type">
|
||||
</div>
|
||||
|
||||
<div class="card border-secondary my-2">
|
||||
<div class="card-body text-dark">
|
||||
<h5 class="card-title">Search {{obj_type}} by name:</h5>
|
||||
{% if obj_type == 'username' %}
|
||||
{% with subtypes=subtypes %}
|
||||
{% include 'username/block_usernames_search.html' %}
|
||||
{% endwith %}
|
||||
{% else %}
|
||||
|
||||
<form action="{{ url_for('objects_subtypes.objects_subtypes_search') }}" id="search_subtype_onj" method='post'>
|
||||
<div class="card border-secondary my-2">
|
||||
<div class="card-body text-dark">
|
||||
<h5 class="card-title">Search {{obj_type}} by name:</h5>
|
||||
|
||||
<div class="input-group mb-1">
|
||||
<input type="text" class="form-control" name="type" value="{{obj_type}}" hidden>
|
||||
<select class="custom-select col-2" name="subtype" value="{{subtype}}" required>
|
||||
<option value="">{{obj_type}} Type...</option>
|
||||
{% for typ in subtypes %}
|
||||
<option value="{{typ}}">{{typ}}</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
<form action="{{ url_for('objects_subtypes.objects_subtypes_search') }}" id="search_subtype_onj" method='post'>
|
||||
|
||||
<input type="text" class="form-control col-8" name="id" value="" placeholder="{{obj_type}} ID" required>
|
||||
<button class="btn btn-primary input-group-addon search-obj col-2"><i class="fas fa-search"></i></button>
|
||||
</div>
|
||||
<div class="input-group mb-1">
|
||||
<input type="text" class="form-control" name="type" value="{{obj_type}}" hidden>
|
||||
<select class="custom-select col-2" name="subtype" value="{{subtype}}" required>
|
||||
<option value="">{{obj_type}} Type...</option>
|
||||
{% for typ in subtypes %}
|
||||
<option value="{{typ}}">{{typ}}</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
<input type="text" class="form-control col-8" name="id" value="" placeholder="{{obj_type}} ID" required>
|
||||
<button class="btn btn-primary input-group-addon search-obj col-2"><i class="fas fa-search"></i></button>
|
||||
</div>
|
||||
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
</div>
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,21 @@
|
|||
<div class="card border-secondary my-2">
|
||||
<div class="card-body text-dark">
|
||||
<h5 class="card-title">Usernames Search:</h5>
|
||||
<form action="{{ url_for('objects_subtypes.objects_username_search') }}" id="search_subtype_onj" method='post'>
|
||||
<div class="input-group mb-1">
|
||||
<input type="text" name="page" value="{% if page %}{{ page }}{% else %}1{% endif %}" hidden="">
|
||||
<select class="custom-select col-2" name="search_subtype" required>
|
||||
{% for s in subtypes %}
|
||||
<option value="{{ s }}" {% if s == subtype %}selected{% endif %}>{{ s }}</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
<input type="text" class="form-control col-8" name="to_search" value="{% if to_search %}{{ to_search }}{% endif %}" placeholder="Username to Search" required>
|
||||
<button class="btn btn-primary input-group-addon search-obj col-2"><i class="fas fa-search"></i></button>
|
||||
</div>
|
||||
<div class="custom-control custom-switch mt-1">
|
||||
<input class="custom-control-input" type="checkbox" name="case_sensitive" id="case_sensitive" {% if type_to_search %}{% if case_sensitive %}value="True" checked{% else %}value="False"{% endif %}{% else %}value="True" checked{% endif %}>
|
||||
<label class="custom-control-label" for="case_sensitive">Case Sensitive</label>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
|
|
@ -0,0 +1,130 @@
|
|||
<!DOCTYPE html>
|
||||
<html>
|
||||
|
||||
<head>
|
||||
<title>Usernames Search - AIL</title>
|
||||
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
|
||||
|
||||
<!-- Core CSS -->
|
||||
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/ail-project.css') }}" rel="stylesheet">
|
||||
|
||||
<!-- JS -->
|
||||
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/d3.min.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/d3/sparklines.js')}}"></script>
|
||||
|
||||
</head>
|
||||
<body>
|
||||
|
||||
{% include 'nav_bar.html' %}
|
||||
|
||||
<div class="container-fluid">
|
||||
<div class="row">
|
||||
|
||||
{% include 'sidebars/sidebar_objects.html' %}
|
||||
|
||||
<div class="col-12 col-lg-10" id="core_content">
|
||||
|
||||
{% with page=dict_page['page'], subtypes=subtypes, subtype=subtype %}
|
||||
{% include 'username/block_usernames_search.html' %}
|
||||
{% endwith %}
|
||||
|
||||
|
||||
<table id="table_objects" class="table table-striped table-bordered">
|
||||
<thead class="bg-dark text-white">
|
||||
<tr>
|
||||
<th>Subtype</th>
|
||||
<th>ID</th>
|
||||
<th>First Seen</th>
|
||||
<th>Last Seen</th>
|
||||
<th>NB Seen</th>
|
||||
<th>Sparkline</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody style="font-size: 15px;">
|
||||
{% for obj_id in dict_objects %}
|
||||
<tr>
|
||||
<td>
|
||||
{% with style=dict_objects[obj_id]['icon']['style'], icon=dict_objects[obj_id]['icon']['icon'] , color=dict_objects[obj_id]['icon']['color'] %}
|
||||
{% include 'objects/obj_svg_block.html' %}
|
||||
{% endwith %}
|
||||
{{ dict_objects[obj_id]['subtype'] }}
|
||||
</td>
|
||||
<td>
|
||||
<a target="_blank" href="{{ url_for('correlation.show_correlation') }}?type=username&subtype=telegram&id={{ obj_id }}">
|
||||
<span>{{ dict_objects[obj_id]['id'][:search_result[obj_id]['hl-start']] }}</span><span class="hg-text">{{dict_objects[obj_id]['id'][search_result[obj_id]['hl-start']:search_result[obj_id]['hl-end']]}}</span>{{ dict_objects[obj_id]['id'][search_result[obj_id]['hl-end']:] }}
|
||||
</a>
|
||||
</td>
|
||||
<td>
|
||||
{% if dict_objects[obj_id]['first_seen'] %}
|
||||
{{ dict_objects[obj_id]['first_seen'][0:4] }}-{{ dict_objects[obj_id]['first_seen'][4:6] }}-{{ dict_objects[obj_id]['first_seen'][6:8] }}
|
||||
{% endif %}
|
||||
</td>
|
||||
<td>
|
||||
{% if dict_objects[obj_id]['last_seen'] %}
|
||||
{{ dict_objects[obj_id]['last_seen'][0:4] }}-{{ dict_objects[obj_id]['last_seen'][4:6] }}-{{ dict_objects[obj_id]['last_seen'][6:8] }}
|
||||
{% endif %}
|
||||
</td>
|
||||
<td>{{ dict_objects[obj_id]['nb_seen'] }}
|
||||
</td>
|
||||
<td id="sparklines_{{ obj_id }}" style="text-align:center;"></td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
{% if dict_page %}
|
||||
{% with page=dict_page['page'], nb_page_max=dict_page['nb_pages'], nb_first_elem=dict_page['nb_first_elem'], nb_last_elem=dict_page['nb_last_elem'], nb_all_elem=dict_page['nb_all_elem'] %}
|
||||
{% set target_url=url_for('objects_subtypes.objects_username_search') + "?search=" + to_search + "&subtype=" + subtype + "&case_sensitive=" + case_sensitive|string %}
|
||||
{% include 'pagination.html' %}
|
||||
{% endwith %}
|
||||
{% endif %}
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
var chart = {};
|
||||
$(document).ready(function(){
|
||||
$("#page-Decoded").addClass("active");
|
||||
$("#nav_username").addClass("active");
|
||||
|
||||
$('#table_objects').DataTable({
|
||||
"aLengthMenu": [[5, 10, 15, -1], [5, 10, 15, "All"]],
|
||||
"iDisplayLength": 10,
|
||||
"order": [[ 3, "desc" ]]
|
||||
});
|
||||
});
|
||||
|
||||
function toggle_sidebar(){
|
||||
if($('#nav_menu').is(':visible')){
|
||||
$('#nav_menu').hide();
|
||||
$('#side_menu').removeClass('border-right')
|
||||
$('#side_menu').removeClass('col-lg-2')
|
||||
$('#core_content').removeClass('col-lg-10')
|
||||
}else{
|
||||
$('#nav_menu').show();
|
||||
$('#side_menu').addClass('border-right')
|
||||
$('#side_menu').addClass('col-lg-2')
|
||||
$('#core_content').addClass('col-lg-10')
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
<script>
|
||||
{% for obj_id in dict_objects %}
|
||||
sparkline("sparklines_{{ obj_id }}", {{ dict_objects[obj_id]['sparkline'] }}, {});
|
||||
{% endfor %}
|
||||
</script>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
Loading…
Reference in New Issue