From 3c6e424ac388ccb4c8df869e82599c4753bbac22 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 5 Nov 2019 09:49:51 +0100 Subject: [PATCH] chg: [UI Domain] UI: tag domain --- bin/lib/Domain.py | 26 +++- bin/packages/Tag.py | 70 ++++++++-- var/www/blueprints/crawler_splash.py | 8 +- var/www/modules/Tags/Flask_Tags.py | 22 +++ .../crawler/crawler_splash/showDomain.html | 52 +++++-- var/www/templates/modals/add_tags.html | 131 ++++++++++++++++++ 6 files changed, 282 insertions(+), 27 deletions(-) create mode 100644 var/www/templates/modals/add_tags.html diff --git a/bin/lib/Domain.py b/bin/lib/Domain.py index 89d39870..f305afa6 100755 --- a/bin/lib/Domain.py +++ b/bin/lib/Domain.py @@ -131,7 +131,28 @@ def get_domain_items_crawled(domain, domain_type, port, epoch=None, items_link=F def get_link_tree(): pass +def get_domain_last_check(domain, domain_type=None, r_format="str"): + ''' + Get domain last check date + :param domain: crawled domain + :type domain: str + :param domain_type: domain type + :type domain_type: str + + :return: domain last check date + :rtype: str + ''' + if not domain_type: + domain_type = get_domain_type(domain) + last_check = r_serv_onion.hget('{}_metadata:{}'.format(domain_type, domain), 'last_check') + if last_check is not None: + if r_format=="int": + last_check = int(last_check) + # str + else: + last_check = '{}/{}/{}'.format(last_check[0:4], last_check[4:6], last_check[6:8]) + return last_check def get_domain_tags(domain): ''' @@ -257,10 +278,7 @@ class Domain(object): :return: domain last check date :rtype: str ''' - last_check = r_serv_onion.hget('{}_metadata:{}'.format(self.type, self.domain), 'last_check') - if last_check is not None: - last_check = '{}/{}/{}'.format(last_check[0:4], last_check[4:6], last_check[6:8]) - return last_check + return get_domain_last_check(self.domain, domain_type=self.type) def is_domain_up(self): # # TODO: handle multiple ports ''' diff --git a/bin/packages/Tag.py b/bin/packages/Tag.py index 44ac43b2..ac5143d1 100755 --- a/bin/packages/Tag.py +++ b/bin/packages/Tag.py @@ -10,6 +10,7 @@ import Item sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) import ConfigLoader +import Domain from pytaxonomies import Taxonomies from pymispgalaxies import Galaxies, Clusters @@ -110,7 +111,7 @@ def get_item_tags_minimal(item_id): return [ {"tag": tag, "min_tag": get_min_tag(tag)} for tag in get_item_tags(item_id) ] # TEMPLATE + API QUERY -def add_items_tag(tags=[], galaxy_tags=[], item_id=None): +def add_items_tag(tags=[], galaxy_tags=[], item_id=None): ## TODO: remove me res_dict = {} if item_id == None: return ({'status': 'error', 'reason': 'Item id not found'}, 404) @@ -138,18 +139,58 @@ def add_items_tag(tags=[], galaxy_tags=[], item_id=None): return (res_dict, 200) -def add_item_tag(tag, item_path): +# TEMPLATE + API QUERY +def add_items_tags(tags=[], galaxy_tags=[], item_id=None, item_type="paste"): + res_dict = {} + if item_id == None: + return ({'status': 'error', 'reason': 'Item id not found'}, 404) + if not tags and not galaxy_tags: + return ({'status': 'error', 'reason': 'Tags or Galaxy not specified'}, 400) + if item_type not in ('paste', 'domain'): + return ({'status': 'error', 'reason': 'Incorrect item_type'}, 400) - item_date = int(Item.get_item_date(item_path)) + res_dict['tags'] = [] + for tag in tags: + if tag: + taxonomie = get_taxonomie_from_tag(tag) + if is_taxonomie_tag_enabled(taxonomie, tag): + add_item_tag(tag, item_id, item_type=item_type) + res_dict['tags'].append(tag) + else: + return ({'status': 'error', 'reason': 'Tags or Galaxy not enabled'}, 400) - #add tag - r_serv_metadata.sadd('tag:{}'.format(item_path), tag) - r_serv_tags.sadd('{}:{}'.format(tag, item_date), item_path) + for tag in galaxy_tags: + if tag: + galaxy = get_galaxy_from_tag(tag) + if is_galaxy_tag_enabled(galaxy, tag): + add_item_tag(tag, item_id, item_type=item_type) + res_dict['tags'].append(tag) + else: + return ({'status': 'error', 'reason': 'Tags or Galaxy not enabled'}, 400) - if Item.is_crawled(item_path): - domain = Item.get_item_domain(item_path) - r_serv_metadata.sadd('tag:{}'.format(domain), tag) - r_serv_tags.sadd('domain:{}:{}'.format(tag, item_date), domain) + res_dict['id'] = item_id + res_dict['type'] = item_type + return (res_dict, 200) + + +def add_item_tag(tag, item_path, item_type="paste"): + + if item_type=="paste": + item_date = int(Item.get_item_date(item_path)) + + #add tag + r_serv_metadata.sadd('tag:{}'.format(item_path), tag) + r_serv_tags.sadd('{}:{}'.format(tag, item_date), item_path) + + if Item.is_crawled(item_path): + domain = Item.get_item_domain(item_path) + r_serv_metadata.sadd('tag:{}'.format(domain), tag) + r_serv_tags.sadd('domain:{}:{}'.format(tag, item_date), domain) + # domain item + else: + item_date = int(Domain.get_domain_last_check(item_path, r_format="int")) + r_serv_metadata.sadd('tag:{}'.format(item_path), tag) + r_serv_tags.sadd('domain:{}:{}'.format(tag, item_date), item_path) r_serv_tags.hincrby('daily_tags:{}'.format(item_date), tag, 1) @@ -250,3 +291,12 @@ def update_tag_last_seen(tag, tag_first_seen, tag_last_seen): else: tag_last_seen = Date.date_substract_day(tag_last_seen) update_tag_last_seen(tag, tag_first_seen, tag_last_seen) + + +# used by modal +def get_modal_add_tags(item_id, tag_type='paste'): + ''' + Modal: add tags to domain or Paste + ''' + return {"active_taxonomies": get_active_taxonomies(), "active_galaxies": get_active_galaxies(), + "item_id": item_id, "type": tag_type} diff --git a/var/www/blueprints/crawler_splash.py b/var/www/blueprints/crawler_splash.py index c996a49a..6977aa4b 100644 --- a/var/www/blueprints/crawler_splash.py +++ b/var/www/blueprints/crawler_splash.py @@ -20,6 +20,9 @@ import Flask_config from Role_Manager import create_user_db, check_password_strength, check_user_role_integrity from Role_Manager import login_admin, login_analyst +sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages')) +from Tag import get_modal_add_tags + sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib')) import Domain @@ -40,8 +43,8 @@ def api_validator(api_response): if api_response: return Response(json.dumps(api_response[0], indent=2, sort_keys=True), mimetype='application/json'), api_response[1] - # ============= ROUTES ============== +# add route : /crawlers/show_domain @crawler_splash.route('/crawlers/showDomain') #@login_required #@login_analyst @@ -65,4 +68,5 @@ def showDomain(): dict_domain['crawler_history'] = domain.get_domain_items_crawled(items_link=True, epoch=epoch, item_screenshot=True, item_tag=True) # # TODO: handle multiple port dict_domain['crawler_history']['random_item'] = random.choice(dict_domain['crawler_history']['items']) - return render_template("showDomain.html", dict_domain=dict_domain, bootstrap_label=bootstrap_label) + return render_template("showDomain.html", dict_domain=dict_domain, bootstrap_label=bootstrap_label, + modal_add_tags=get_modal_add_tags(dict_domain['domain'], tag_type="domain")) diff --git a/var/www/modules/Tags/Flask_Tags.py b/var/www/modules/Tags/Flask_Tags.py index 4772e82e..f2ebaab1 100644 --- a/var/www/modules/Tags/Flask_Tags.py +++ b/var/www/modules/Tags/Flask_Tags.py @@ -442,6 +442,28 @@ def addTags(): # success return redirect(url_for('showsavedpastes.showsavedpaste', paste=path)) +@Tags.route("/Tags/add_item_tags") +@login_required +@login_analyst +def add_item_tags(): + + tags = request.args.get('tags') + tagsgalaxies = request.args.get('tagsgalaxies') + item_id = request.args.get('item_id') + item_type = request.args.get('type') + + list_tag = tags.split(',') + list_tag_galaxies = tagsgalaxies.split(',') + + res = Tag.add_items_tags(tags=list_tag, galaxy_tags=list_tag_galaxies, item_id=item_id, item_type=item_type) + # error + if res[1] != 200: + return str(res[0]) + # success + if item_type=='domain': + return redirect(url_for('crawler_splash.showDomain', domain=item_id)) + else: + return redirect(url_for('showsavedpastes.showsavedpaste', paste=item_id)) @Tags.route("/Tags/taxonomies") @login_required diff --git a/var/www/templates/crawler/crawler_splash/showDomain.html b/var/www/templates/crawler/crawler_splash/showDomain.html index 7b71f8da..5ee2bf4d 100644 --- a/var/www/templates/crawler/crawler_splash/showDomain.html +++ b/var/www/templates/crawler/crawler_splash/showDomain.html @@ -7,11 +7,24 @@ + + + + + @@ -77,6 +90,10 @@ {% endfor %}
+ {% include 'modals/add_tags.html' %} + @@ -243,7 +260,7 @@ {%if item["screenshot"]%} - {%endif%} @@ -358,8 +375,8 @@ function toggle_sidebar(){ + diff --git a/var/www/templates/modals/add_tags.html b/var/www/templates/modals/add_tags.html new file mode 100644 index 00000000..98cb5479 --- /dev/null +++ b/var/www/templates/modals/add_tags.html @@ -0,0 +1,131 @@ + + +