mirror of https://github.com/CIRCL/AIL-framework
chg: [core + UI] search domain by tags
Terrtia
parent
b4a36d812f
commit
99897ffa9b
|
|
@ -23,6 +23,12 @@ config_loader = ConfigLoader.ConfigLoader()
|
|||
r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
|
||||
config_loader = None
|
||||
|
||||
def is_valid_object_type(object_type):
|
||||
if object_type in ['domain', 'item', 'screenshot']:
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
def get_all_objects():
|
||||
return ['domain', 'paste', 'pgp', 'cryptocurrency', 'decoded', 'screenshot']
|
||||
|
||||
|
|
@ -189,6 +195,13 @@ def get_item_url(correlation_name, value, correlation_type=None):
|
|||
url = url_for(endpoint, paste=value)
|
||||
return url
|
||||
|
||||
def get_obj_tag_table_keys(object_type):
|
||||
'''
|
||||
Warning: use only in flask (dynamic templates)
|
||||
'''
|
||||
if object_type=="domain":
|
||||
return ['id', 'first_seen', 'last_check', 'status'] # # TODO: add root screenshot
|
||||
|
||||
|
||||
def create_graph_links(links_set):
|
||||
graph_links_list = []
|
||||
|
|
@ -316,6 +329,7 @@ def get_graph_node_object_correlation(object_type, root_value, mode, correlation
|
|||
|
||||
|
||||
######## API EXPOSED ########
|
||||
|
||||
|
||||
def sanitize_object_type(object_type):
|
||||
if not is_valid_object_type(object_type):
|
||||
return ({'status': 'error', 'reason': 'Incorrect object_type'}, 400)
|
||||
######## ########
|
||||
|
|
|
|||
|
|
@ -200,6 +200,20 @@ def get_tag_metadata(tag, r_int=False):
|
|||
tag_metadata['last_seen'] = get_tag_last_seen(tag)
|
||||
return tag_metadata
|
||||
|
||||
def get_tags_min_last_seen(l_tags, r_int=False):
|
||||
'''
|
||||
Get max last seen from a list of tags (current: item only)
|
||||
'''
|
||||
min_last_seen = 99999999
|
||||
for tag in l_tags:
|
||||
last_seen = get_tag_last_seen(tag, r_int=True)
|
||||
if last_seen < min_last_seen:
|
||||
min_last_seen = last_seen
|
||||
if r_int:
|
||||
return min_last_seen
|
||||
else:
|
||||
return str(min_last_seen)
|
||||
|
||||
def is_obj_tagged(object_id, tag):
|
||||
'''
|
||||
Check if a object is tagged
|
||||
|
|
@ -431,6 +445,106 @@ def delete_obj_tags(object_id, object_type, tags=[]):
|
|||
if res:
|
||||
return res
|
||||
|
||||
def sanitise_tags_date_range(l_tags, date_from=None, date_to=None):
|
||||
if date_from and date_to is None:
|
||||
date_from = get_tags_min_last_seen(l_tags, r_int=False)
|
||||
date_to = date_from
|
||||
return Date.sanitise_date_range(date_from, date_to)
|
||||
|
||||
|
||||
# # TODO: verify tags + object_type
|
||||
# get set_keys: intersection
|
||||
def get_obj_keys_by_tags(object_type, l_tags, date_day=None):
|
||||
l_set_keys = []
|
||||
if object_type=='item':
|
||||
for tag in l_tags:
|
||||
l_set_keys.append('{}:{}'.format(tag, date_day))
|
||||
else:
|
||||
for tag in l_tags:
|
||||
l_set_keys.append('{}:{}'.format(object_type, tag))
|
||||
return l_set_keys
|
||||
|
||||
def get_obj_by_tag(key_tag):
|
||||
return r_serv_tags.smembers(key_tag)
|
||||
|
||||
def get_obj_by_tags(object_type, l_tags, date_from=None, date_to=None, nb_obj=50, page=1): # remove old object
|
||||
# with daterange
|
||||
l_tagged_obj = []
|
||||
if object_type=='item':
|
||||
#sanityze date
|
||||
date_range = sanitise_tags_date_range(l_tags, date_from=date_from, date_to=date_to)
|
||||
l_dates = Date.substract_date(date_from, date_to)
|
||||
|
||||
for date_day in l_dates:
|
||||
l_set_keys = get_obj_keys_by_tags(object_type, l_tags, date_day)
|
||||
# if len(l_set_keys) > nb_obj:
|
||||
# return l_tagged_obj
|
||||
if len(l_set_keys) < 2:
|
||||
date_day_obj = get_obj_by_tag(l_set_keys[0])
|
||||
else:
|
||||
date_day_obj = r_serv_tags.sinter(l_set_keys[0], *l_set_keys[1:])
|
||||
|
||||
# next_nb_start = len(l_tagged_obj) + len(date_day_obj) - nb_obj
|
||||
# if next_nb_start > 0:
|
||||
# get + filter nb_start
|
||||
l_tagged_obj.extend( date_day_obj )
|
||||
|
||||
# handle pagination
|
||||
nb_pages = len(l_tagged_obj) / nb_obj
|
||||
if not nb_pages.is_integer():
|
||||
nb_pages = int(nb_pages)+1
|
||||
else:
|
||||
nb_pages = int(nb_pages)
|
||||
if page > nb_pages:
|
||||
page = nb_pages
|
||||
|
||||
# select index
|
||||
start = nb_obj*(page -1)
|
||||
stop = (nb_obj*page) -1
|
||||
l_tagged_obj = l_tagged_obj[start:stop]
|
||||
|
||||
return {"tagged_obj":l_tagged_obj, "page":page, "nb_pages":nb_pages}
|
||||
|
||||
# without daterange
|
||||
else:
|
||||
l_set_keys = get_obj_keys_by_tags(object_type, l_tags)
|
||||
if len(l_set_keys) < 2:
|
||||
l_tagged_obj = get_obj_by_tag(l_set_keys[0])
|
||||
else:
|
||||
l_tagged_obj = r_serv_tags.sinter(l_set_keys[0], *l_set_keys[1:])
|
||||
|
||||
if not l_tagged_obj:
|
||||
return {"tagged_obj":l_tagged_obj, "page":0, "nb_pages":0}
|
||||
|
||||
# handle pagination
|
||||
nb_pages = len(l_tagged_obj) / nb_obj
|
||||
if not nb_pages.is_integer():
|
||||
nb_pages = int(nb_pages)+1
|
||||
else:
|
||||
nb_pages = int(nb_pages)
|
||||
if page > nb_pages:
|
||||
page = nb_pages
|
||||
|
||||
# multiple pages
|
||||
if nb_pages > 1:
|
||||
start = nb_obj*(page -1)
|
||||
stop = (nb_obj*page) -1
|
||||
current_index = 0
|
||||
l_obj = []
|
||||
for elem in l_tagged_obj:
|
||||
if current_index > stop:
|
||||
break
|
||||
if start <= current_index and stop >= current_index:
|
||||
l_obj.append(elem)
|
||||
current_index += 1
|
||||
l_tagged_obj = l_obj
|
||||
# only one page
|
||||
else:
|
||||
l_tagged_obj = list(l_tagged_obj)
|
||||
|
||||
return {"tagged_obj":l_tagged_obj, "page":page, "nb_pages":nb_pages}
|
||||
|
||||
|
||||
def get_obj_date(object_type, object_id): # # TODO: move me in another file
|
||||
if object_type == "item":
|
||||
return Item.get_item_date(object_id)
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ import Flask_config
|
|||
|
||||
# Import Role_Manager
|
||||
from Role_Manager import create_user_db, check_password_strength, check_user_role_integrity
|
||||
from Role_Manager import login_admin, login_analyst
|
||||
from Role_Manager import login_admin, login_analyst, login_read_only
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))
|
||||
import Date
|
||||
|
|
@ -77,6 +77,64 @@ def delete_tag():
|
|||
return str(res[0])
|
||||
return redirect(Correlate_object.get_item_url(object_type, object_id))
|
||||
|
||||
|
||||
@tags_ui.route('/tag/get_all_tags')
|
||||
@login_required
|
||||
@login_read_only
|
||||
def get_all_tags():
|
||||
return jsonify(Tag.get_all_tags())
|
||||
|
||||
@tags_ui.route('/tag/get_all_obj_tags')
|
||||
@login_required
|
||||
@login_read_only
|
||||
def get_all_obj_tags():
|
||||
object_type = request.args.get('object_type')
|
||||
res = Correlate_object.sanitize_object_type(object_type)
|
||||
if res:
|
||||
return jsonify(res)
|
||||
return jsonify(Tag.get_all_obj_tags(object_type))
|
||||
|
||||
@tags_ui.route('/tag/search/get_obj_by_tags')
|
||||
@login_required
|
||||
@login_read_only
|
||||
def get_obj_by_tags():
|
||||
|
||||
# # TODO: sanityze all
|
||||
object_type = request.args.get('object_type')
|
||||
ltags = request.args.get('ltags')
|
||||
page = request.args.get('ltags')
|
||||
date_from = request.args.get('ltags')
|
||||
date_to = request.args.get('ltags')
|
||||
|
||||
# unpack tags
|
||||
list_tags = ltags.split(',')
|
||||
list_tag = []
|
||||
for tag in list_tags:
|
||||
list_tag.append(tag.replace('"','\"'))
|
||||
|
||||
res = Correlate_object.sanitize_object_type(object_type)
|
||||
if res:
|
||||
return jsonify(res)
|
||||
|
||||
dict_obj = Tag.get_obj_by_tags(object_type, list_tag)
|
||||
|
||||
if dict_obj['tagged_obj']:
|
||||
dict_tagged = {"object_type":object_type, "page":dict_obj['page'] ,"nb_pages":dict_obj['nb_pages'], "tagged_obj":[]}
|
||||
for obj_id in dict_obj['tagged_obj']:
|
||||
obj_metadata = Correlate_object.get_object_metadata(object_type, obj_id)
|
||||
obj_metadata['id'] = obj_id
|
||||
dict_tagged["tagged_obj"].append(obj_metadata)
|
||||
|
||||
dict_tagged['tab_keys'] = Correlate_object.get_obj_tag_table_keys(object_type)
|
||||
|
||||
if len(list_tag) == 1:
|
||||
dict_tagged['current_tags'] = ltags.replace('"', '').replace('=', '').replace(':', '')
|
||||
else:
|
||||
dict_tagged['current_tags'] = list_tag
|
||||
|
||||
#return jsonify(dict_tagged)
|
||||
return render_template("tags/search_obj_by_tags.html", bootstrap_label=bootstrap_label, dict_tagged=dict_tagged)
|
||||
|
||||
# # add route : /crawlers/show_domain
|
||||
# @tags_ui.route('/tags/search/domain')
|
||||
# @login_required
|
||||
|
|
|
|||
|
|
@ -253,6 +253,7 @@ def dashboard():
|
|||
statDomains_regular = get_stats_last_crawled_domains('regular', date)
|
||||
|
||||
return render_template("Crawler_dashboard.html", crawler_metadata_onion = crawler_metadata_onion,
|
||||
object_type='domain',
|
||||
crawler_enabled=crawler_enabled, date=date,
|
||||
crawler_metadata_regular=crawler_metadata_regular,
|
||||
statDomains_onion=statDomains_onion, statDomains_regular=statDomains_regular)
|
||||
|
|
|
|||
|
|
@ -105,6 +105,8 @@
|
|||
</div>
|
||||
</div>
|
||||
|
||||
{% include 'tags/block_obj_tags_search.html' %}
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,101 @@
|
|||
<div class="card mb-3 mt-1">
|
||||
<div class="card-header text-white bg-dark">
|
||||
<h5 class="card-title">Search Domain by Tags :</h5>
|
||||
</div>
|
||||
<div class="card-body">
|
||||
|
||||
<!-- <div class="row mb-3"> TODO: use condition
|
||||
<div class="col-md-6">
|
||||
<div class="input-group" id="date-range-from">
|
||||
<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
|
||||
<input class="form-control" id="date-range-from-input" placeholder="yyyy-mm-dd" value="{{ date_from }}" name="date_from" autocomplete="off">
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-md-6">
|
||||
<div class="input-group" id="date-range-to">
|
||||
<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
|
||||
<input class="form-control" id="date-range-to-input" placeholder="yyyy-mm-dd" value="{{ date_to }}" name="date_to" autocomplete="off">
|
||||
</div>
|
||||
</div>
|
||||
</div> -->
|
||||
|
||||
<div class="input-group mb-3">
|
||||
<div class="input-group-prepend">
|
||||
<button class="btn btn-outline-danger" type="button" id="button-clear-tags" style="z-index: 1;" onclick="emptyTags()">
|
||||
<i class="fas fa-eraser"></i>
|
||||
</button>
|
||||
</div>
|
||||
<input id="ltags" name="ltags" type="text" class="form-control" aria-describedby="button-clear-tags" autocomplete="off">
|
||||
</div>
|
||||
|
||||
<button class="btn btn-primary" type="button" id="button-search-tags" onclick="searchTags()">
|
||||
<i class="fas fa-search"></i> Search Domains
|
||||
</button>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<link href="{{ url_for('static', filename='css/tags.css') }}" rel="stylesheet" type="text/css" />
|
||||
<script src="{{ url_for('static', filename='js/tags.js') }}"></script>
|
||||
<script>
|
||||
var ltags;
|
||||
$.getJSON("{{ url_for('tags_ui.get_all_obj_tags') }}?object_type={{dict_tagged['object_type']}}",
|
||||
function(data) {
|
||||
ltags = $('#ltags').tagSuggest({
|
||||
data: data,
|
||||
value: [{%if "current_tags" in dict_tagged%}{% for tag in dict_tagged['current_tags'] %}'{{tag|safe}}',{%endfor%}{%endif%}],
|
||||
sortOrder: 'name',
|
||||
maxDropHeight: 200,
|
||||
name: 'ltags'
|
||||
});
|
||||
});
|
||||
|
||||
function searchTags() {
|
||||
var data = ltags.getValue();
|
||||
//var date_from = $('#date-range-from-input').val();
|
||||
//var date_to =$('#date-range-to-input').val();
|
||||
//parameter = parameter + "&date_from="+date_from+"&date_to="+date_to
|
||||
window.location.replace("{{ url_for('tags_ui.get_obj_by_tags') }}?ltags=" + data + "&object_type={{dict_tagged['object_type']}}&page={{dict_tagged['page']}}");
|
||||
}
|
||||
function emptyTags() {
|
||||
ltags.clear();
|
||||
}
|
||||
|
||||
// $('#date-range-from').dateRangePicker({
|
||||
// separator : ' to ',
|
||||
// getValue: function(){
|
||||
// if ($('#date-range-from-input').val() && $('#date-range-to-input').val() )
|
||||
// return $('#date-range-from-input').val() + ' to ' + $('#date-range-to-input').val();
|
||||
// else
|
||||
// return '';
|
||||
// },
|
||||
// setValue: function(s,s1,s2){
|
||||
// $('#date-range-from-input').val(s1);
|
||||
// $('#date-range-to-input').val(s2);
|
||||
// }
|
||||
// });
|
||||
// $('#date-range-to').dateRangePicker({
|
||||
// separator : ' to ',
|
||||
// getValue: function(){
|
||||
// if ($('#date-range-from-input').val() && $('#date-range-to-input').val() )
|
||||
// return $('#date-range-from-input').val() + ' to ' + $('#date-range-to-input').val();
|
||||
// else
|
||||
// return '';
|
||||
// },
|
||||
// setValue: function(s,s1,s2){
|
||||
// $('#date-range-from-input').val(s1);
|
||||
// $('#date-range-to-input').val(s2);
|
||||
// }
|
||||
// });
|
||||
|
||||
// tagid + objtype + objid
|
||||
// $('#edit_tags_modal').on('show.bs.modal', function (event) {
|
||||
// var button = $(event.relatedTarget);
|
||||
// var tagid = button.data('tagid')
|
||||
// var objtype = button.data('objtype')
|
||||
// var objid = button.data('objid')
|
||||
// var modal = $(this)
|
||||
// modal.find('#modal_tag_edit_tag_id').text(tagid)
|
||||
// modal.find('#modal_tag_edit_delete_tag').prop("href", "{{ url_for('tags_ui.delete_tag') }}?object_type="+ objtype +"&object_id="+ objid +"&tag="+ tagid)
|
||||
// })
|
||||
</script>
|
||||
|
|
@ -0,0 +1,146 @@
|
|||
<!DOCTYPE html>
|
||||
<html>
|
||||
|
||||
<head>
|
||||
<title>Tags - AIL</title>
|
||||
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
|
||||
|
||||
<!-- Core CSS -->
|
||||
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/daterangepicker.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
|
||||
|
||||
<!-- JS -->
|
||||
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
|
||||
<script language="javascript" src="{{ url_for('static', filename='js/moment.min.js') }}"></script>
|
||||
<script language="javascript" src="{{ url_for('static', filename='js/jquery.daterangepicker.min.js') }}"></script>
|
||||
|
||||
</head>
|
||||
<body>
|
||||
|
||||
{% include 'nav_bar.html' %}
|
||||
|
||||
<!-- Modal -->
|
||||
<div id="mymodal" class="modal fade" role="dialog">
|
||||
<div class="modal-dialog modal-lg">
|
||||
|
||||
<!-- Modal content-->
|
||||
<div id="mymodalcontent" class="modal-content">
|
||||
<div id="mymodalbody" class="modal-body" max-width="850px">
|
||||
<p>Loading paste information...</p>
|
||||
<img id="loading-gif-modal" src="{{url_for('static', filename='image/loading.gif') }}" height="26" width="26" style="margin: 4px;">
|
||||
</div>
|
||||
<div class="modal-footer">
|
||||
<a id="button_show_path" target="_blank" href=""><button type="button" class="btn btn-info">Show saved paste</button></a>
|
||||
<button type="button" class="btn btn-default" data-dismiss="modal">Close</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="container-fluid">
|
||||
<div class="row">
|
||||
|
||||
{% include 'tags/menu_sidebar.html' %}
|
||||
|
||||
<div class="col-12 col-lg-10" id="core_content">
|
||||
|
||||
{% include 'tags/block_obj_tags_search.html' %}
|
||||
|
||||
<div>
|
||||
{%if dict_tagged%}
|
||||
<table class="table table-bordered table-hover" id="myTable_">
|
||||
<thead class="thead-dark">
|
||||
<tr>
|
||||
{%if dict_tagged["object_type"]=="domain"%}
|
||||
<th>first seen</th>
|
||||
<th>last check</th>
|
||||
<th style="max-width: 800px;">Domain</th>
|
||||
<th>status</th>
|
||||
{%endif%}
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
|
||||
{%if dict_tagged["object_type"]=="domain"%}
|
||||
|
||||
{% for dict_obj in dict_tagged["tagged_obj"] %}
|
||||
<tr>
|
||||
<td class="pb-0">{{ dict_obj['first_seen'] }}</td>
|
||||
<td class="pb-0">{{ dict_obj['last_check'] }}</td>
|
||||
<td class="pb-0">
|
||||
<a target="_blank" href="{{ url_for('showsavedpastes.showsavedpaste') }}?paste={{dict_obj['id']}}" class="text-secondary">
|
||||
<div style="line-height:0.9;">{{ dict_obj['id'] }}</div>
|
||||
</a>
|
||||
<div class="mb-2">
|
||||
{% for tag in dict_obj['tags'] %}
|
||||
<a href="{{ url_for('Tags.Tags_page') }}?ltags={{ tag }}">
|
||||
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }}</span>
|
||||
</a>
|
||||
{% endfor %}
|
||||
</div>
|
||||
</td>
|
||||
<td class="pb-0">
|
||||
{%if dict_obj['status'] %}
|
||||
<div style="color:Green;">
|
||||
<i class="fas fa-check-circle"></i>
|
||||
UP
|
||||
</div>
|
||||
{% else %}
|
||||
<div style="color:Red;">
|
||||
<i class="fas fa-times-circle"></i>
|
||||
DOWN
|
||||
</div>
|
||||
{% endif %}
|
||||
</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
|
||||
{% endif %}
|
||||
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
{%endif%}
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</body>
|
||||
|
||||
<script>
|
||||
var ltags;
|
||||
var search_table;
|
||||
|
||||
$(document).ready(function(){
|
||||
//search_table = $('#myTable_').DataTable({ "order": [[ 0, "asc" ]] });
|
||||
$('#myTable_').DataTable({ "order": [[ 0, "asc" ]] });
|
||||
|
||||
});
|
||||
</script>
|
||||
|
||||
<script>
|
||||
function toggle_sidebar(){
|
||||
if($('#nav_menu').is(':visible')){
|
||||
$('#nav_menu').hide();
|
||||
$('#side_menu').removeClass('border-right')
|
||||
$('#side_menu').removeClass('col-lg-2')
|
||||
$('#core_content').removeClass('col-lg-10')
|
||||
}else{
|
||||
$('#nav_menu').show();
|
||||
$('#side_menu').addClass('border-right')
|
||||
$('#side_menu').addClass('col-lg-2')
|
||||
$('#core_content').addClass('col-lg-10')
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
|
||||
</html>
|
||||
Loading…
Reference in New Issue