diff --git a/bin/lib/Correlate_object.py b/bin/lib/Correlate_object.py index 1854902c..68fbed9e 100755 --- a/bin/lib/Correlate_object.py +++ b/bin/lib/Correlate_object.py @@ -23,6 +23,12 @@ config_loader = ConfigLoader.ConfigLoader() r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata") config_loader = None +def is_valid_object_type(object_type): + if object_type in ['domain', 'item', 'screenshot']: + return True + else: + return False + def get_all_objects(): return ['domain', 'paste', 'pgp', 'cryptocurrency', 'decoded', 'screenshot'] @@ -189,6 +195,13 @@ def get_item_url(correlation_name, value, correlation_type=None): url = url_for(endpoint, paste=value) return url +def get_obj_tag_table_keys(object_type): + ''' + Warning: use only in flask (dynamic templates) + ''' + if object_type=="domain": + return ['id', 'first_seen', 'last_check', 'status'] # # TODO: add root screenshot + def create_graph_links(links_set): graph_links_list = [] @@ -316,6 +329,7 @@ def get_graph_node_object_correlation(object_type, root_value, mode, correlation ######## API EXPOSED ######## - - +def sanitize_object_type(object_type): + if not is_valid_object_type(object_type): + return ({'status': 'error', 'reason': 'Incorrect object_type'}, 400) ######## ######## diff --git a/bin/packages/Tag.py b/bin/packages/Tag.py index fde6545b..45c657da 100755 --- a/bin/packages/Tag.py +++ b/bin/packages/Tag.py @@ -200,6 +200,20 @@ def get_tag_metadata(tag, r_int=False): tag_metadata['last_seen'] = get_tag_last_seen(tag) return tag_metadata +def get_tags_min_last_seen(l_tags, r_int=False): + ''' + Get max last seen from a list of tags (current: item only) + ''' + min_last_seen = 99999999 + for tag in l_tags: + last_seen = get_tag_last_seen(tag, r_int=True) + if last_seen < min_last_seen: + min_last_seen = last_seen + if r_int: + return min_last_seen + else: + return str(min_last_seen) + def is_obj_tagged(object_id, tag): ''' Check if a object is tagged @@ -431,6 +445,106 @@ def delete_obj_tags(object_id, object_type, tags=[]): if res: return res +def sanitise_tags_date_range(l_tags, date_from=None, date_to=None): + if date_from and date_to is None: + date_from = get_tags_min_last_seen(l_tags, r_int=False) + date_to = date_from + return Date.sanitise_date_range(date_from, date_to) + + +# # TODO: verify tags + object_type +# get set_keys: intersection +def get_obj_keys_by_tags(object_type, l_tags, date_day=None): + l_set_keys = [] + if object_type=='item': + for tag in l_tags: + l_set_keys.append('{}:{}'.format(tag, date_day)) + else: + for tag in l_tags: + l_set_keys.append('{}:{}'.format(object_type, tag)) + return l_set_keys + +def get_obj_by_tag(key_tag): + return r_serv_tags.smembers(key_tag) + +def get_obj_by_tags(object_type, l_tags, date_from=None, date_to=None, nb_obj=50, page=1): # remove old object + # with daterange + l_tagged_obj = [] + if object_type=='item': + #sanityze date + date_range = sanitise_tags_date_range(l_tags, date_from=date_from, date_to=date_to) + l_dates = Date.substract_date(date_from, date_to) + + for date_day in l_dates: + l_set_keys = get_obj_keys_by_tags(object_type, l_tags, date_day) + # if len(l_set_keys) > nb_obj: + # return l_tagged_obj + if len(l_set_keys) < 2: + date_day_obj = get_obj_by_tag(l_set_keys[0]) + else: + date_day_obj = r_serv_tags.sinter(l_set_keys[0], *l_set_keys[1:]) + + # next_nb_start = len(l_tagged_obj) + len(date_day_obj) - nb_obj + # if next_nb_start > 0: + # get + filter nb_start + l_tagged_obj.extend( date_day_obj ) + + # handle pagination + nb_pages = len(l_tagged_obj) / nb_obj + if not nb_pages.is_integer(): + nb_pages = int(nb_pages)+1 + else: + nb_pages = int(nb_pages) + if page > nb_pages: + page = nb_pages + + # select index + start = nb_obj*(page -1) + stop = (nb_obj*page) -1 + l_tagged_obj = l_tagged_obj[start:stop] + + return {"tagged_obj":l_tagged_obj, "page":page, "nb_pages":nb_pages} + + # without daterange + else: + l_set_keys = get_obj_keys_by_tags(object_type, l_tags) + if len(l_set_keys) < 2: + l_tagged_obj = get_obj_by_tag(l_set_keys[0]) + else: + l_tagged_obj = r_serv_tags.sinter(l_set_keys[0], *l_set_keys[1:]) + + if not l_tagged_obj: + return {"tagged_obj":l_tagged_obj, "page":0, "nb_pages":0} + + # handle pagination + nb_pages = len(l_tagged_obj) / nb_obj + if not nb_pages.is_integer(): + nb_pages = int(nb_pages)+1 + else: + nb_pages = int(nb_pages) + if page > nb_pages: + page = nb_pages + + # multiple pages + if nb_pages > 1: + start = nb_obj*(page -1) + stop = (nb_obj*page) -1 + current_index = 0 + l_obj = [] + for elem in l_tagged_obj: + if current_index > stop: + break + if start <= current_index and stop >= current_index: + l_obj.append(elem) + current_index += 1 + l_tagged_obj = l_obj + # only one page + else: + l_tagged_obj = list(l_tagged_obj) + + return {"tagged_obj":l_tagged_obj, "page":page, "nb_pages":nb_pages} + + def get_obj_date(object_type, object_id): # # TODO: move me in another file if object_type == "item": return Item.get_item_date(object_id) diff --git a/var/www/blueprints/tags_ui.py b/var/www/blueprints/tags_ui.py index 06b27a62..d39b14b1 100644 --- a/var/www/blueprints/tags_ui.py +++ b/var/www/blueprints/tags_ui.py @@ -18,7 +18,7 @@ import Flask_config # Import Role_Manager from Role_Manager import create_user_db, check_password_strength, check_user_role_integrity -from Role_Manager import login_admin, login_analyst +from Role_Manager import login_admin, login_analyst, login_read_only sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages')) import Date @@ -77,6 +77,64 @@ def delete_tag(): return str(res[0]) return redirect(Correlate_object.get_item_url(object_type, object_id)) + +@tags_ui.route('/tag/get_all_tags') +@login_required +@login_read_only +def get_all_tags(): + return jsonify(Tag.get_all_tags()) + +@tags_ui.route('/tag/get_all_obj_tags') +@login_required +@login_read_only +def get_all_obj_tags(): + object_type = request.args.get('object_type') + res = Correlate_object.sanitize_object_type(object_type) + if res: + return jsonify(res) + return jsonify(Tag.get_all_obj_tags(object_type)) + +@tags_ui.route('/tag/search/get_obj_by_tags') +@login_required +@login_read_only +def get_obj_by_tags(): + + # # TODO: sanityze all + object_type = request.args.get('object_type') + ltags = request.args.get('ltags') + page = request.args.get('ltags') + date_from = request.args.get('ltags') + date_to = request.args.get('ltags') + + # unpack tags + list_tags = ltags.split(',') + list_tag = [] + for tag in list_tags: + list_tag.append(tag.replace('"','\"')) + + res = Correlate_object.sanitize_object_type(object_type) + if res: + return jsonify(res) + + dict_obj = Tag.get_obj_by_tags(object_type, list_tag) + + if dict_obj['tagged_obj']: + dict_tagged = {"object_type":object_type, "page":dict_obj['page'] ,"nb_pages":dict_obj['nb_pages'], "tagged_obj":[]} + for obj_id in dict_obj['tagged_obj']: + obj_metadata = Correlate_object.get_object_metadata(object_type, obj_id) + obj_metadata['id'] = obj_id + dict_tagged["tagged_obj"].append(obj_metadata) + + dict_tagged['tab_keys'] = Correlate_object.get_obj_tag_table_keys(object_type) + + if len(list_tag) == 1: + dict_tagged['current_tags'] = ltags.replace('"', '').replace('=', '').replace(':', '') + else: + dict_tagged['current_tags'] = list_tag + + #return jsonify(dict_tagged) + return render_template("tags/search_obj_by_tags.html", bootstrap_label=bootstrap_label, dict_tagged=dict_tagged) + # # add route : /crawlers/show_domain # @tags_ui.route('/tags/search/domain') # @login_required diff --git a/var/www/modules/hiddenServices/Flask_hiddenServices.py b/var/www/modules/hiddenServices/Flask_hiddenServices.py index 0db6bbe6..b92374ef 100644 --- a/var/www/modules/hiddenServices/Flask_hiddenServices.py +++ b/var/www/modules/hiddenServices/Flask_hiddenServices.py @@ -253,6 +253,7 @@ def dashboard(): statDomains_regular = get_stats_last_crawled_domains('regular', date) return render_template("Crawler_dashboard.html", crawler_metadata_onion = crawler_metadata_onion, + object_type='domain', crawler_enabled=crawler_enabled, date=date, crawler_metadata_regular=crawler_metadata_regular, statDomains_onion=statDomains_onion, statDomains_regular=statDomains_regular) diff --git a/var/www/modules/hiddenServices/templates/Crawler_dashboard.html b/var/www/modules/hiddenServices/templates/Crawler_dashboard.html index 769beb7f..ee67be8b 100644 --- a/var/www/modules/hiddenServices/templates/Crawler_dashboard.html +++ b/var/www/modules/hiddenServices/templates/Crawler_dashboard.html @@ -105,6 +105,8 @@ + {% include 'tags/block_obj_tags_search.html' %} + diff --git a/var/www/templates/tags/block_obj_tags_search.html b/var/www/templates/tags/block_obj_tags_search.html new file mode 100644 index 00000000..e4e34e07 --- /dev/null +++ b/var/www/templates/tags/block_obj_tags_search.html @@ -0,0 +1,101 @@ +
+
+
Search Domain by Tags :
+
+
+ + + +
+
+ +
+ +
+ + + +
+
+ + + + diff --git a/var/www/templates/tags/search_obj_by_tags.html b/var/www/templates/tags/search_obj_by_tags.html new file mode 100644 index 00000000..f0bfa8f3 --- /dev/null +++ b/var/www/templates/tags/search_obj_by_tags.html @@ -0,0 +1,146 @@ + + + + + Tags - AIL + + + + + + + + + + + + + + + + + + + + + {% include 'nav_bar.html' %} + + + + +
+
+ + {% include 'tags/menu_sidebar.html' %} + +
+ + {% include 'tags/block_obj_tags_search.html' %} + +
+ {%if dict_tagged%} + + + + {%if dict_tagged["object_type"]=="domain"%} + + + + + {%endif%} + + + + + {%if dict_tagged["object_type"]=="domain"%} + + {% for dict_obj in dict_tagged["tagged_obj"] %} + + + + + + + {% endfor %} + + {% endif %} + + +
first seenlast checkDomainstatus
{{ dict_obj['first_seen'] }}{{ dict_obj['last_check'] }} + +
{{ dict_obj['id'] }}
+ +
+ {% for tag in dict_obj['tags'] %} + + {{ tag }} + + {% endfor %} +
+ 		+ {%if dict_obj['status'] %} +
+ + UP +
+ {% else %} +
+ + DOWN +
+ {% endif %} +
+ + {%endif%} +
+ +
+
+
+ + + + + + + + +