mirror of https://github.com/CIRCL/AIL-framework
chg: [UI Domain] UI: tag domain
Terrtia
parent
a3e5e44c9b
commit
3c6e424ac3
|
|
@ -131,7 +131,28 @@ def get_domain_items_crawled(domain, domain_type, port, epoch=None, items_link=F
|
|||
def get_link_tree():
|
||||
pass
|
||||
|
||||
def get_domain_last_check(domain, domain_type=None, r_format="str"):
|
||||
'''
|
||||
Get domain last check date
|
||||
|
||||
:param domain: crawled domain
|
||||
:type domain: str
|
||||
:param domain_type: domain type
|
||||
:type domain_type: str
|
||||
|
||||
:return: domain last check date
|
||||
:rtype: str
|
||||
'''
|
||||
if not domain_type:
|
||||
domain_type = get_domain_type(domain)
|
||||
last_check = r_serv_onion.hget('{}_metadata:{}'.format(domain_type, domain), 'last_check')
|
||||
if last_check is not None:
|
||||
if r_format=="int":
|
||||
last_check = int(last_check)
|
||||
# str
|
||||
else:
|
||||
last_check = '{}/{}/{}'.format(last_check[0:4], last_check[4:6], last_check[6:8])
|
||||
return last_check
|
||||
|
||||
def get_domain_tags(domain):
|
||||
'''
|
||||
|
|
@ -257,10 +278,7 @@ class Domain(object):
|
|||
:return: domain last check date
|
||||
:rtype: str
|
||||
'''
|
||||
last_check = r_serv_onion.hget('{}_metadata:{}'.format(self.type, self.domain), 'last_check')
|
||||
if last_check is not None:
|
||||
last_check = '{}/{}/{}'.format(last_check[0:4], last_check[4:6], last_check[6:8])
|
||||
return last_check
|
||||
return get_domain_last_check(self.domain, domain_type=self.type)
|
||||
|
||||
def is_domain_up(self): # # TODO: handle multiple ports
|
||||
'''
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ import Item
|
|||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||
import ConfigLoader
|
||||
import Domain
|
||||
|
||||
from pytaxonomies import Taxonomies
|
||||
from pymispgalaxies import Galaxies, Clusters
|
||||
|
|
@ -110,7 +111,7 @@ def get_item_tags_minimal(item_id):
|
|||
return [ {"tag": tag, "min_tag": get_min_tag(tag)} for tag in get_item_tags(item_id) ]
|
||||
|
||||
# TEMPLATE + API QUERY
|
||||
def add_items_tag(tags=[], galaxy_tags=[], item_id=None):
|
||||
def add_items_tag(tags=[], galaxy_tags=[], item_id=None): ## TODO: remove me
|
||||
res_dict = {}
|
||||
if item_id == None:
|
||||
return ({'status': 'error', 'reason': 'Item id not found'}, 404)
|
||||
|
|
@ -138,18 +139,58 @@ def add_items_tag(tags=[], galaxy_tags=[], item_id=None):
|
|||
return (res_dict, 200)
|
||||
|
||||
|
||||
def add_item_tag(tag, item_path):
|
||||
# TEMPLATE + API QUERY
|
||||
def add_items_tags(tags=[], galaxy_tags=[], item_id=None, item_type="paste"):
|
||||
res_dict = {}
|
||||
if item_id == None:
|
||||
return ({'status': 'error', 'reason': 'Item id not found'}, 404)
|
||||
if not tags and not galaxy_tags:
|
||||
return ({'status': 'error', 'reason': 'Tags or Galaxy not specified'}, 400)
|
||||
if item_type not in ('paste', 'domain'):
|
||||
return ({'status': 'error', 'reason': 'Incorrect item_type'}, 400)
|
||||
|
||||
item_date = int(Item.get_item_date(item_path))
|
||||
res_dict['tags'] = []
|
||||
for tag in tags:
|
||||
if tag:
|
||||
taxonomie = get_taxonomie_from_tag(tag)
|
||||
if is_taxonomie_tag_enabled(taxonomie, tag):
|
||||
add_item_tag(tag, item_id, item_type=item_type)
|
||||
res_dict['tags'].append(tag)
|
||||
else:
|
||||
return ({'status': 'error', 'reason': 'Tags or Galaxy not enabled'}, 400)
|
||||
|
||||
#add tag
|
||||
r_serv_metadata.sadd('tag:{}'.format(item_path), tag)
|
||||
r_serv_tags.sadd('{}:{}'.format(tag, item_date), item_path)
|
||||
for tag in galaxy_tags:
|
||||
if tag:
|
||||
galaxy = get_galaxy_from_tag(tag)
|
||||
if is_galaxy_tag_enabled(galaxy, tag):
|
||||
add_item_tag(tag, item_id, item_type=item_type)
|
||||
res_dict['tags'].append(tag)
|
||||
else:
|
||||
return ({'status': 'error', 'reason': 'Tags or Galaxy not enabled'}, 400)
|
||||
|
||||
if Item.is_crawled(item_path):
|
||||
domain = Item.get_item_domain(item_path)
|
||||
r_serv_metadata.sadd('tag:{}'.format(domain), tag)
|
||||
r_serv_tags.sadd('domain:{}:{}'.format(tag, item_date), domain)
|
||||
res_dict['id'] = item_id
|
||||
res_dict['type'] = item_type
|
||||
return (res_dict, 200)
|
||||
|
||||
|
||||
def add_item_tag(tag, item_path, item_type="paste"):
|
||||
|
||||
if item_type=="paste":
|
||||
item_date = int(Item.get_item_date(item_path))
|
||||
|
||||
#add tag
|
||||
r_serv_metadata.sadd('tag:{}'.format(item_path), tag)
|
||||
r_serv_tags.sadd('{}:{}'.format(tag, item_date), item_path)
|
||||
|
||||
if Item.is_crawled(item_path):
|
||||
domain = Item.get_item_domain(item_path)
|
||||
r_serv_metadata.sadd('tag:{}'.format(domain), tag)
|
||||
r_serv_tags.sadd('domain:{}:{}'.format(tag, item_date), domain)
|
||||
# domain item
|
||||
else:
|
||||
item_date = int(Domain.get_domain_last_check(item_path, r_format="int"))
|
||||
r_serv_metadata.sadd('tag:{}'.format(item_path), tag)
|
||||
r_serv_tags.sadd('domain:{}:{}'.format(tag, item_date), item_path)
|
||||
|
||||
r_serv_tags.hincrby('daily_tags:{}'.format(item_date), tag, 1)
|
||||
|
||||
|
|
@ -250,3 +291,12 @@ def update_tag_last_seen(tag, tag_first_seen, tag_last_seen):
|
|||
else:
|
||||
tag_last_seen = Date.date_substract_day(tag_last_seen)
|
||||
update_tag_last_seen(tag, tag_first_seen, tag_last_seen)
|
||||
|
||||
|
||||
# used by modal
|
||||
def get_modal_add_tags(item_id, tag_type='paste'):
|
||||
'''
|
||||
Modal: add tags to domain or Paste
|
||||
'''
|
||||
return {"active_taxonomies": get_active_taxonomies(), "active_galaxies": get_active_galaxies(),
|
||||
"item_id": item_id, "type": tag_type}
|
||||
|
|
|
|||
|
|
@ -20,6 +20,9 @@ import Flask_config
|
|||
from Role_Manager import create_user_db, check_password_strength, check_user_role_integrity
|
||||
from Role_Manager import login_admin, login_analyst
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))
|
||||
from Tag import get_modal_add_tags
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
|
||||
import Domain
|
||||
|
||||
|
|
@ -40,8 +43,8 @@ def api_validator(api_response):
|
|||
if api_response:
|
||||
return Response(json.dumps(api_response[0], indent=2, sort_keys=True), mimetype='application/json'), api_response[1]
|
||||
|
||||
|
||||
# ============= ROUTES ==============
|
||||
# add route : /crawlers/show_domain
|
||||
@crawler_splash.route('/crawlers/showDomain')
|
||||
#@login_required
|
||||
#@login_analyst
|
||||
|
|
@ -65,4 +68,5 @@ def showDomain():
|
|||
dict_domain['crawler_history'] = domain.get_domain_items_crawled(items_link=True, epoch=epoch, item_screenshot=True, item_tag=True) # # TODO: handle multiple port
|
||||
dict_domain['crawler_history']['random_item'] = random.choice(dict_domain['crawler_history']['items'])
|
||||
|
||||
return render_template("showDomain.html", dict_domain=dict_domain, bootstrap_label=bootstrap_label)
|
||||
return render_template("showDomain.html", dict_domain=dict_domain, bootstrap_label=bootstrap_label,
|
||||
modal_add_tags=get_modal_add_tags(dict_domain['domain'], tag_type="domain"))
|
||||
|
|
|
|||
|
|
@ -442,6 +442,28 @@ def addTags():
|
|||
# success
|
||||
return redirect(url_for('showsavedpastes.showsavedpaste', paste=path))
|
||||
|
||||
@Tags.route("/Tags/add_item_tags")
|
||||
@login_required
|
||||
@login_analyst
|
||||
def add_item_tags():
|
||||
|
||||
tags = request.args.get('tags')
|
||||
tagsgalaxies = request.args.get('tagsgalaxies')
|
||||
item_id = request.args.get('item_id')
|
||||
item_type = request.args.get('type')
|
||||
|
||||
list_tag = tags.split(',')
|
||||
list_tag_galaxies = tagsgalaxies.split(',')
|
||||
|
||||
res = Tag.add_items_tags(tags=list_tag, galaxy_tags=list_tag_galaxies, item_id=item_id, item_type=item_type)
|
||||
# error
|
||||
if res[1] != 200:
|
||||
return str(res[0])
|
||||
# success
|
||||
if item_type=='domain':
|
||||
return redirect(url_for('crawler_splash.showDomain', domain=item_id))
|
||||
else:
|
||||
return redirect(url_for('showsavedpastes.showsavedpaste', paste=item_id))
|
||||
|
||||
@Tags.route("/Tags/taxonomies")
|
||||
@login_required
|
||||
|
|
|
|||
|
|
@ -7,11 +7,24 @@
|
|||
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/tags.css') }}" rel="stylesheet" type="text/css" />
|
||||
<!-- JS -->
|
||||
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/bootstrap4.min.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/tags.js') }}"></script>
|
||||
|
||||
<style>
|
||||
.icon_img:hover{
|
||||
cursor: pointer;
|
||||
color: #17a2b8;
|
||||
}
|
||||
.icon_selected{
|
||||
color: #007bff;
|
||||
}
|
||||
</style>
|
||||
|
||||
</head>
|
||||
<body>
|
||||
|
|
@ -77,6 +90,10 @@
|
|||
</a>
|
||||
{% endfor %}
|
||||
<br>
|
||||
{% include 'modals/add_tags.html' %}
|
||||
<button type="button" class="btn btn-light" data-toggle="modal" data-target="#add_tags_modal">
|
||||
<i class="far fa-plus-square"></i>
|
||||
</button>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
|
@ -243,7 +260,7 @@
|
|||
</td>
|
||||
<td class="text-center">
|
||||
{%if item["screenshot"]%}
|
||||
<button class="btn" onclick="reload_image('{{ item["screenshot"] }}', '{{ item["link"] }}', '{{ item["id"] }}');">
|
||||
<span class="icon_img" onclick="reload_image('{{ item["screenshot"] }}', '{{ item["link"] }}', '{{ item["id"] }}');" id="{{ item["screenshot"].replace('/', '') }}">
|
||||
<i class="far fa-image"></i>
|
||||
</button>
|
||||
{%endif%}
|
||||
|
|
@ -358,8 +375,8 @@ function toggle_sidebar(){
|
|||
|
||||
<script>
|
||||
var ctx = canvas.getContext('2d'), img = new Image();
|
||||
var base_url = "{{ url_for('showsavedpastes.screenshot', filename="") }}"
|
||||
var screenshot_href = "{{ url_for('showsavedpastes.showsavedpaste') }}?paste="
|
||||
var base_url = "{{ url_for('showsavedpastes.screenshot', filename="") }}";
|
||||
var screenshot_href = "{{ url_for('showsavedpastes.showsavedpaste') }}?paste=";
|
||||
|
||||
/// turn off image smoothing
|
||||
ctx.webkitImageSmoothingEnabled = false;
|
||||
|
|
@ -369,12 +386,17 @@ img.onload = pixelate;
|
|||
img.addEventListener("error", img_error);
|
||||
var draw_img = false;
|
||||
|
||||
{%if 'crawler_history' in dict_domain%}
|
||||
var screenshot = "{{dict_domain['crawler_history']['random_item']['screenshot']}}"
|
||||
$("#screenshot_link").attr("href", "screenshot_href + {{dict_domain['crawler_history']['random_item']['id']}}")
|
||||
$("#screenshot_link").text("{{dict_domain['crawler_history']['random_item']['link']}}")
|
||||
{%if dict_domain['crawler_history']['random_item']['screenshot']%}
|
||||
var screenshot = "{{dict_domain['crawler_history']['random_item']['screenshot']}}";
|
||||
var selected_icon = $("#"+screenshot.replace(/\//g, ""));
|
||||
selected_icon.addClass("icon_selected");
|
||||
selected_icon.removeClass("icon_img");
|
||||
|
||||
|
||||
$("#screenshot_link").attr("href", "screenshot_href + {{dict_domain['crawler_history']['random_item']['id']}}");
|
||||
$("#screenshot_link").text("{{dict_domain['crawler_history']['random_item']['link']}}");
|
||||
{%else%}
|
||||
var screenshot = ""
|
||||
var screenshot = "";
|
||||
{%endif%}
|
||||
|
||||
img.src = base_url + screenshot;
|
||||
|
|
@ -411,14 +433,22 @@ function img_error() {
|
|||
pixelate;
|
||||
}
|
||||
|
||||
function reload_image(screenshot, link, item_id) {
|
||||
function reload_image(new_screenshot, link, item_id) {
|
||||
$("#"+screenshot.replace(/\//g, "")).removeClass("icon_selected").addClass("icon_img");
|
||||
screenshot = new_screenshot;
|
||||
|
||||
img.src=base_url + screenshot;
|
||||
$("#screenshot_link").attr("href", screenshot_href + item_id)
|
||||
$("#screenshot_link").text(link)
|
||||
selected_icon = $("#"+screenshot.replace(/\//g, ""));
|
||||
selected_icon.addClass("icon_selected");
|
||||
selected_icon.removeClass("icon_img")
|
||||
|
||||
$("#screenshot_link").attr("href", screenshot_href + item_id);
|
||||
$("#screenshot_link").text(link);
|
||||
pixelate;
|
||||
}
|
||||
|
||||
|
||||
</script>
|
||||
|
||||
|
||||
</html>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,131 @@
|
|||
<div id="add_tags_modal" class="modal fade" role="dialog">
|
||||
<div class="modal-dialog modal-lg">
|
||||
|
||||
<div id="add_tags_modal_content" class="modal-content">
|
||||
<div class="modal-header" style="border-bottom: 4px solid #cccccc; background-color: #cccccc; color: #ffffff;">
|
||||
<h4>Add New Tags</h4>
|
||||
</div>
|
||||
|
||||
<div class="modal-body">
|
||||
|
||||
<div class="input-group" >
|
||||
<input id="ltags" type="text" class="form-control" autocomplete="off" style="width: 760px">
|
||||
</div>
|
||||
|
||||
<div class="dropdown">
|
||||
<button type="button" class="btn btn-info dropdown-toggle mt-1 mb-3" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false" id="dropdown-taxonomie">
|
||||
Taxonomie Selected
|
||||
</button>
|
||||
<div class="dropdown-menu dropdown-menu-right" aria-labelledby="dropdown-taxonomie"> <!-- TODO: make dropdown-scrollable -->
|
||||
<h6 class="dropdown-header">Taxonomie Tags</h6>
|
||||
<button class="dropdown-item" type="button" id="all-tags-taxonomies">All Tags <i class="fas fa-tags"></i></button>
|
||||
<div class="dropdown-divider"></div>
|
||||
{% for taxo in modal_add_tags['active_taxonomies'] %}
|
||||
<button class="dropdown-item" type="button" id="{{ taxo }}-id{{ loop.index0 }}">{{ taxo }}</button>
|
||||
{% endfor %}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="input-group">
|
||||
<input id="ltagsgalaxies" type="text" class="form-control" autocomplete="off" style="width: 760px">
|
||||
</div>
|
||||
|
||||
<div class="dropdown">
|
||||
<button type="button" class="btn btn-info dropdown-toggle mt-1" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false" id="dropdown-galaxy">
|
||||
Galaxy Selected
|
||||
</button>
|
||||
<div class="dropdown-menu dropdown-menu-right" aria-labelledby="dropdown-galaxy"> <!-- TODO: make dropdown-scrollable -->
|
||||
<h6 class="dropdown-header">Galaxy Tags</h6>
|
||||
<button class="dropdown-item" type="button" id="all-tags-galaxies">All Tags <i class="fas fa-tags"></i></button>
|
||||
<div class="dropdown-divider"></div>
|
||||
{% for galaxy in modal_add_tags['active_galaxies'] %}
|
||||
<button class="dropdown-item" type="button" id="{{ galaxy }}-idgalax{{ loop.index0 }}">{{ galaxy }}</button>
|
||||
{% endfor %}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
<div class="modal-footer">
|
||||
<a class="btn btn-link" href="{{ url_for('Tags.taxonomies') }}" target="_blank">
|
||||
<span class="label-icon">Edit Taxonomies List </span>
|
||||
<i class="fas fa-wrench fa-2x"></i>
|
||||
</a>
|
||||
<a class="btn btn-link" href="{{ url_for('Tags.galaxies') }}" target="_blank">
|
||||
<span class="label-icon">Edit Galaxies List</span>
|
||||
<i class="fas fa-rocket fa-2x"></i>
|
||||
</a>
|
||||
<button class="btn btn-primary" onclick="addTags()">
|
||||
<i class="fas fa-plus"></i>
|
||||
Add Tags
|
||||
</button>
|
||||
<button type="button" class="btn btn-default" data-dismiss="modal" >Close</button>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script> // TODO: add tags to PASTE or DOMAIN
|
||||
var ltags;
|
||||
var ltagsgalaxies;
|
||||
|
||||
$.getJSON("{{ url_for('Tags.get_all_tags_taxonomies') }}",
|
||||
function(data) {
|
||||
|
||||
ltags = $('#ltags').tagSuggest({
|
||||
data: data,
|
||||
maxDropHeight: 200,
|
||||
name: 'ltags'
|
||||
});
|
||||
});
|
||||
|
||||
$.getJSON("{{ url_for('Tags.get_all_tags_galaxy') }}",
|
||||
function(data) {
|
||||
|
||||
ltagsgalaxies = $('#ltagsgalaxies').tagSuggest({
|
||||
data: data,
|
||||
maxDropHeight: 200,
|
||||
name: 'ltagsgalaxies'
|
||||
});
|
||||
});
|
||||
|
||||
jQuery("#all-tags-taxonomies").click(function(e){
|
||||
//change input tags list
|
||||
$.getJSON("{{ url_for('Tags.get_all_tags_taxonomies') }}",
|
||||
function(data) {
|
||||
ltags.setData(data)
|
||||
});
|
||||
});
|
||||
|
||||
jQuery("#all-tags-galaxies").click(function(e){
|
||||
$.getJSON("{{ url_for('Tags.get_all_tags_galaxy') }}",
|
||||
function(data) {
|
||||
ltagsgalaxies.setData(data)
|
||||
});
|
||||
});
|
||||
|
||||
{% for taxo in modal_add_tags['active_taxonomies'] %}
|
||||
jQuery("#{{ taxo }}-id{{ loop.index0 }}").click(function(e){
|
||||
$.getJSON("{{ url_for('Tags.get_tags_taxonomie') }}?taxonomie={{ taxo }}",
|
||||
function(data) {
|
||||
ltags.setData(data)
|
||||
});
|
||||
});
|
||||
{% endfor %}
|
||||
|
||||
{% for galaxy in modal_add_tags['active_galaxies'] %}
|
||||
jQuery("#{{ galaxy }}-idgalax{{ loop.index0 }}").click(function(e){
|
||||
$.getJSON("{{ url_for('Tags.get_tags_galaxy') }}?galaxy={{ galaxy }}",
|
||||
function(data) {
|
||||
ltagsgalaxies.setData(data)
|
||||
});
|
||||
});
|
||||
{% endfor %}
|
||||
|
||||
function addTags() {
|
||||
var tags = ltags.getValue()
|
||||
var tagsgalaxy = ltagsgalaxies.getValue()
|
||||
window.location.replace("{{ url_for('Tags.add_item_tags') }}?tags=" + tags + "&tagsgalaxies=" + tagsgalaxy + "&item_id={{ modal_add_tags['item_id'] }}&type={{ modal_add_tags['type'] }}");
|
||||
}
|
||||
</script>
|
||||
Loading…
Reference in New Issue