CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

chg: [UI tags] search domains by tags

pull/449/head
Terrtia 2020-01-13 11:10:03 +01:00
parent 99897ffa9b
commit fb320ca5d5
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
10 changed files with 147 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
bin/lib/Correlate_object.py
View File

@ -24,7 +24,7 @@ r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
config_loader = None
def is_valid_object_type(object_type):
if object_type in ['domain', 'item', 'screenshot']:
if object_type in ['domain', 'item', 'image']:
return True
else:
return False

15
bin/packages/Tag.py
View File

@ -490,7 +490,8 @@ def get_obj_by_tags(object_type, l_tags, date_from=None, date_to=None, nb_obj=50
l_tagged_obj.extend( date_day_obj )
# handle pagination
nb_pages = len(l_tagged_obj) / nb_obj
nb_all_elem = len(l_tagged_obj)
nb_pages = nb_all_elem / nb_obj
if not nb_pages.is_integer():
nb_pages = int(nb_pages)+1
else:
@ -503,7 +504,7 @@ def get_obj_by_tags(object_type, l_tags, date_from=None, date_to=None, nb_obj=50
stop = (nb_obj*page) -1
l_tagged_obj = l_tagged_obj[start:stop]
return {"tagged_obj":l_tagged_obj, "page":page, "nb_pages":nb_pages}
return {"tagged_obj":l_tagged_obj, "page":page, "nb_pages":nb_pages, "nb_first_elem":start+1, "nb_last_elem":stop+1, "nb_all_elem":nb_all_elem}
# without daterange
else:
@ -517,7 +518,8 @@ def get_obj_by_tags(object_type, l_tags, date_from=None, date_to=None, nb_obj=50
return {"tagged_obj":l_tagged_obj, "page":0, "nb_pages":0}
# handle pagination
nb_pages = len(l_tagged_obj) / nb_obj
nb_all_elem = len(l_tagged_obj)
nb_pages = nb_all_elem / nb_obj
if not nb_pages.is_integer():
nb_pages = int(nb_pages)+1
else:
@ -538,11 +540,16 @@ def get_obj_by_tags(object_type, l_tags, date_from=None, date_to=None, nb_obj=50
l_obj.append(elem)
current_index += 1
l_tagged_obj = l_obj
stop += 1
if stop > nb_all_elem:
stop = nb_all_elem
# only one page
else:
start = 0
stop = nb_all_elem
l_tagged_obj = list(l_tagged_obj)
return {"tagged_obj":l_tagged_obj, "page":page, "nb_pages":nb_pages}
return {"tagged_obj":l_tagged_obj, "page":page, "nb_pages":nb_pages, "nb_first_elem":start+1, "nb_last_elem":stop, "nb_all_elem":nb_all_elem}
def get_obj_date(object_type, object_id): # # TODO: move me in another file

39
var/www/blueprints/tags_ui.py
View File

@ -94,6 +94,22 @@ def get_all_obj_tags():
return jsonify(res)
return jsonify(Tag.get_all_obj_tags(object_type))
@tags_ui.route('/tag/search/domain')
@login_required
@login_read_only
def tags_search_domains():
object_type = 'domain'
dict_tagged = {"object_type":object_type, "object_name":object_type.title() + "s"}
return render_template("tags/search_obj_by_tags.html", bootstrap_label=bootstrap_label, dict_tagged=dict_tagged)
@tags_ui.route('/tag/search/image')
@login_required
@login_read_only
def tags_search_images():
object_type = 'image'
dict_tagged = {"object_type":object_type, "object_name":object_type.title() + "s"}
return render_template("tags/search_obj_by_tags.html", bootstrap_label=bootstrap_label, dict_tagged=dict_tagged)
@tags_ui.route('/tag/search/get_obj_by_tags')
@login_required
@login_read_only
@ -102,7 +118,7 @@ def get_obj_by_tags():
# # TODO: sanityze all
object_type = request.args.get('object_type')
ltags = request.args.get('ltags')
page = request.args.get('ltags')
page = request.args.get('page')
date_from = request.args.get('ltags')
date_to = request.args.get('ltags')
@ -112,14 +128,23 @@ def get_obj_by_tags():
for tag in list_tags:
list_tag.append(tag.replace('"','\"'))
# object_type
res = Correlate_object.sanitize_object_type(object_type)
if res:
return jsonify(res)
dict_obj = Tag.get_obj_by_tags(object_type, list_tag)
# page
try:
page = int(page)
except:
page = 1
dict_obj = Tag.get_obj_by_tags(object_type, list_tag, page=page)
if dict_obj['tagged_obj']:
dict_tagged = {"object_type":object_type, "page":dict_obj['page'] ,"nb_pages":dict_obj['nb_pages'], "tagged_obj":[]}
dict_tagged = {"object_type":object_type, "object_name":object_type.title() + "s",
"tagged_obj":[], "page":dict_obj['page'] ,"nb_pages":dict_obj['nb_pages'],
"nb_first_elem":dict_obj['nb_first_elem'], "nb_last_elem":dict_obj['nb_last_elem'], "nb_all_elem":dict_obj['nb_all_elem']}
for obj_id in dict_obj['tagged_obj']:
obj_metadata = Correlate_object.get_object_metadata(object_type, obj_id)
obj_metadata['id'] = obj_id
@ -128,12 +153,16 @@ def get_obj_by_tags():
dict_tagged['tab_keys'] = Correlate_object.get_obj_tag_table_keys(object_type)
if len(list_tag) == 1:
dict_tagged['current_tags'] = ltags.replace('"', '').replace('=', '').replace(':', '')
dict_tagged['current_tags'] = [ltags.replace('"', '\"')]
else:
dict_tagged['current_tags'] = list_tag
dict_tagged['current_tags_str'] = ltags
#return jsonify(dict_tagged)
return render_template("tags/search_obj_by_tags.html", bootstrap_label=bootstrap_label, dict_tagged=dict_tagged)
else:
dict_tagged = {"object_type":object_type, "object_name":object_type.title() + "s"}
return render_template("tags/search_obj_by_tags.html", bootstrap_label=bootstrap_label, dict_tagged=dict_tagged)
# # add route : /crawlers/show_domain
# @tags_ui.route('/tags/search/domain')

4
var/www/modules/Tags/templates/Tags.html
View File

@ -217,8 +217,8 @@ var last_clicked_paste;
var can_change_modal_content = true;
$(document).ready(function(){
$("#nav_quick_search").removeClass("text-muted");
$("#nav_tag_{{tag_nav}}").addClass("active");
$("#nav_tags_search").removeClass("text-muted");
$("#nav_tags_search_item").addClass("active");
search_table = $('#myTable_').DataTable({ "order": [[ 0, "asc" ]] });
// Use to bind the button with the new displayed data

1
var/www/modules/hiddenServices/Flask_hiddenServices.py
View File

@ -253,7 +253,6 @@ def dashboard():
statDomains_regular = get_stats_last_crawled_domains('regular', date)
return render_template("Crawler_dashboard.html", crawler_metadata_onion = crawler_metadata_onion,
object_type='domain',
crawler_enabled=crawler_enabled, date=date,
crawler_metadata_regular=crawler_metadata_regular,
statDomains_onion=statDomains_onion, statDomains_regular=statDomains_regular)

4
var/www/modules/hiddenServices/templates/Crawler_dashboard.html
View File

@ -105,7 +105,9 @@
</div>
</div>
{% include 'tags/block_obj_tags_search.html' %}
{% with object_type='domain' %}
{% include 'tags/block_obj_tags_search.html' %}
{% endwith %}
</div>
</div>

50
var/www/templates/pagination.html Normal file
View File

@ -0,0 +1,50 @@
<div class="my-2">
<div class="d-flex justify-content-center border-top">
<nav class="mt-4" aria-label="...">
<ul class="pagination">
<li class="page-item {%if page==1%}disabled{%endif%}">
<a class="page-link" href="{{ target_url }}&page={{page-1}}">Previous</a>
</li>
{%if page>3%}
<li class="page-item"><a class="page-link" href="{{ target_url }}&page=1">1</a></li>
<li class="page-item disabled"><a class="page-link" aria-disabled="true" href="#">...</a></li>
<li class="page-item"><a class="page-link" href="{{ target_url }}&page={{page-1}}">{{page-1}}</a></li>
<li class="page-item active"><a class="page-link" href="{{ target_url }}&page={{page}}">{{page}}</a></li>
{%else%}
{%if page>2%}<li class="page-item"><a class="page-link" href="{{ target_url }}&page={{page-2}}">{{page-2}}</a></li>{%endif%}
{%if page>1%}<li class="page-item"><a class="page-link" href="{{ target_url }}&page={{page-1}}">{{page-1}}</a></li>{%endif%}
<li class="page-item active"><a class="page-link" href="{{ target_url }}&page={{page}}">{{page}}</a></li>
{%endif%}
{%if nb_page_max-page>3%}
<li class="page-item"><a class="page-link" href="{{ target_url }}&page={{page+1}}">{{page+1}}</a></li>
<li class="page-item disabled"><a class="page-link" aria-disabled="true" href="#">...</a></li>
<li class="page-item"><a class="page-link" href="{{ target_url }}&page={{nb_page_max}}">{{nb_page_max}}</a></li>
{%else%}
{%if nb_page_max-page>2%}<li class="page-item"><a class="page-link" href="{{ target_url }}&page={{nb_page_max-2}}">{{nb_page_max-2}}</a></li>{%endif%}
{%if nb_page_max-page>1%}<li class="page-item"><a class="page-link" href="{{ target_url }}&page={{nb_page_max-1}}">{{nb_page_max-1}}</a></li>{%endif%}
{%if nb_page_max-page>0%}<li class="page-item"><a class="page-link" href="{{ target_url }}&page={{nb_page_max}}">{{nb_page_max}}</a></li>{%endif%}
{%endif%}
<li class="page-item {%if page==nb_page_max%}disabled{%endif%}">
<a class="page-link" href="{{ target_url }}&page={{page+1}}" aria-disabled="true">Next</a>
</li>
</ul>
</nav>
</div>
{%if nb_all_elem%}
<div class="d-flex justify-content-center">
<span class="badge badge-info text-white">
{{object_name}}:&nbsp;
<span class="badge badge-light">{{nb_first_elem}}-{{nb_last_elem}}</span>
<span> / </span>
<span class="badge badge-light">{{nb_all_elem}}</span>
</span>
</div>
<br>
<br>
<br>
{%endif%}
</div>

12
var/www/templates/tags/block_obj_tags_search.html
View File

@ -1,6 +1,6 @@
<div class="card mb-3 mt-1">
<div class="card-header text-white bg-dark">
<h5 class="card-title">Search Domain by Tags :</h5>
<h5 class="card-title">Search {{object_name}} by Tags :</h5>
</div>
<div class="card-body">
@ -29,21 +29,22 @@
</div>
<button class="btn btn-primary" type="button" id="button-search-tags" onclick="searchTags()">
<i class="fas fa-search"></i> Search Domains
<i class="fas fa-search"></i> Search {{object_name}}
</button>
</div>
</div>
<link href="{{ url_for('static', filename='css/tags.css') }}" rel="stylesheet" type="text/css" />
<script src="{{ url_for('static', filename='js/tags.js') }}"></script>
<script>
var ltags;
$.getJSON("{{ url_for('tags_ui.get_all_obj_tags') }}?object_type={{dict_tagged['object_type']}}",
$.getJSON("{{ url_for('tags_ui.get_all_obj_tags') }}?object_type={{ object_type }}",
function(data) {
ltags = $('#ltags').tagSuggest({
data: data,
value: [{%if "current_tags" in dict_tagged%}{% for tag in dict_tagged['current_tags'] %}'{{tag|safe}}',{%endfor%}{%endif%}],
value: [{%if "current_tags"%}{% for tag in current_tags %}'{{tag|safe}}',{%endfor%}{%endif%}],
sortOrder: 'name',
maxDropHeight: 200,
name: 'ltags'
@ -55,7 +56,8 @@
//var date_from = $('#date-range-from-input').val();
//var date_to =$('#date-range-to-input').val();
//parameter = parameter + "&date_from="+date_from+"&date_to="+date_to
window.location.replace("{{ url_for('tags_ui.get_obj_by_tags') }}?ltags=" + data + "&object_type={{dict_tagged['object_type']}}&page={{dict_tagged['page']}}");
var parameter = "?ltags=" + data + "&object_type={{ object_type }}{%if page%}&page={{ page }}{%endif%}";
window.location.replace("{{ url_for('tags_ui.get_obj_by_tags') }}" + parameter);
}
function emptyTags() {
ltags.clear();

23
var/www/templates/tags/menu_sidebar.html
View File

@ -6,6 +6,29 @@
</button>
<nav class="navbar navbar-expand navbar-light bg-light flex-md-column flex-row align-items-start py-2" id="nav_menu">
<h5 class="d-flex text-muted w-100" id="nav_tags_search">
<span>Tags Search</span>
</h5>
<ul class="nav flex-md-column flex-row navbar-nav justify-content-between w-100">
<li class="nav-item">
<a class="nav-link" href="{{ url_for('Tags.Tags_page') }}" id="nav_tags_search_item">
<i class="far fa-file"></i>
Search Items by Tags
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="{{ url_for('tags_ui.tags_search_domains') }}" id="nav_tags_search_domain">
<i class="fab fa-html5"></i>
Search Domains by Tags
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="{{ url_for('tags_ui.tags_search_images') }}" id="nav_tags_search_image">
<i class="fas fa-image"></i>
Search Images by Tags
</a>
</li>
</ul>
<h5 class="d-flex text-muted w-100">
<span>Tags Management </span>
</h5>

40
var/www/templates/tags/search_obj_by_tags.html
View File

@ -25,24 +25,6 @@
{% include 'nav_bar.html' %}
<!-- Modal -->
<div id="mymodal" class="modal fade" role="dialog">
<div class="modal-dialog modal-lg">
<!-- Modal content-->
<div id="mymodalcontent" class="modal-content">
<div id="mymodalbody" class="modal-body" max-width="850px">
<p>Loading paste information...</p>
<img id="loading-gif-modal" src="{{url_for('static', filename='image/loading.gif') }}" height="26" width="26" style="margin: 4px;">
</div>
<div class="modal-footer">
<a id="button_show_path" target="_blank" href=""><button type="button" class="btn btn-info">Show saved paste</button></a>
<button type="button" class="btn btn-default" data-dismiss="modal">Close</button>
</div>
</div>
</div>
</div>
<div class="container-fluid">
<div class="row">
@ -50,10 +32,12 @@
<div class="col-12 col-lg-10" id="core_content">
{% include 'tags/block_obj_tags_search.html' %}
{% with object_type=dict_tagged['object_type'], current_tags=dict_tagged['current_tags'], object_name=dict_tagged['object_name'] %}
{% include 'tags/block_obj_tags_search.html' %}
{% endwith %}
<div>
{%if dict_tagged%}
{%if 'tagged_obj' in dict_tagged%}
<table class="table table-bordered table-hover" id="myTable_">
<thead class="thead-dark">
<tr>
@ -80,7 +64,7 @@
<div class="mb-2">
{% for tag in dict_obj['tags'] %}
<a href="{{ url_for('Tags.Tags_page') }}?ltags={{ tag }}">
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }}</span>
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span>
</a>
{% endfor %}
</div>
@ -99,15 +83,22 @@
{% endif %}
</td>
</tr>
{% endfor %}
{% endfor %}
{% endif %}
{% endif %}
</tbody>
</table>
{%endif%}
</div>
{%if 'tagged_obj' in dict_tagged%}
{% with page=dict_tagged['page'], nb_page_max=dict_tagged['nb_pages'], nb_first_elem=dict_tagged['nb_first_elem'], nb_last_elem=dict_tagged['nb_last_elem'], nb_all_elem=dict_tagged['nb_all_elem'] %}
{% set object_name= dict_tagged['object_name'] %}
{% set target_url=url_for('tags_ui.get_obj_by_tags') + "?object_type=" + dict_tagged['object_type'] + "&ltags=" + dict_tagged['current_tags_str'] %}
{% include 'pagination.html' %}
{% endwith %}
{%endif%}
</div>
</div>
@ -120,7 +111,8 @@ var ltags;
var search_table;
$(document).ready(function(){
//search_table = $('#myTable_').DataTable({ "order": [[ 0, "asc" ]] });
$("#nav_tags_search").removeClass("text-muted");
$("#nav_tags_search_{{dict_tagged['object_type']}}").addClass("active");
$('#myTable_').DataTable({ "order": [[ 0, "asc" ]] });
});