chg: [UI term tracker] refractor term management: trackers list + show trackers + add new trackers

pull/389/head
Terrtia 2019-08-14 09:44:49 +02:00
parent 80f9535074
commit 7ed09bc923
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
8 changed files with 865 additions and 356 deletions

View File

@ -17,6 +17,8 @@ from textblob import TextBlob
sys.path.append(os.path.join(os.environ['AIL_FLASK'], 'modules'))
import Flask_config
import Date
r_serv_term = Flask_config.r_serv_term
email_regex = Flask_config.email_regex
@ -235,7 +237,7 @@ def add_tracked_term(term , term_type, user_id, level, tags, mails, dashboard=0)
if level == 0: # user only
r_serv_term.sadd('user:tracked_term:{}'.format(user_id), term_uuid)
elif level == 1: # global
r_serv_term.sadd('gobal:tracked_term', term_uuid)
r_serv_term.sadd('global:tracked_term', term_uuid)
# create term tags list
for tag in tags:
@ -274,7 +276,7 @@ def delete_term(term_uuid):
user_id = term_type = r_serv_term.hget('tracked_term:{}'.format(term_uuid), 'user_id')
r_serv_term.srem('user:tracked_term:{}'.format(user_id), term_uuid)
elif level == 1: # global
r_serv_term.srem('gobal:tracked_term', term_uuid)
r_serv_term.srem('global:tracked_term', term_uuid)
# delete metatadata
r_serv_term.delete('tracked_term:{}'.format(term_uuid))
@ -291,6 +293,20 @@ def delete_term(term_uuid):
r_serv_term.delete('tracked_term:item:{}:{}'.format(term_uuid, date))
r_serv_term.delete('tracked_term:stat:{}'.format(term_uuid))
def replace_tracked_term_tags(term_uuid, tags):
r_serv_term.delete('tracked_term:tags:{}'.format(term_uuid))
for tag in tags:
r_serv_term.sadd('tracked_term:tags:{}'.format(term_uuid), tag)
def replace_tracked_term_mails(term_uuid, mails):
res = verify_mail_list(mails)
if res:
return res
else:
r_serv_term.delete('tracked_term:mail:{}'.format(term_uuid))
for mail in mails:
r_serv_term.sadd('tracked_term:mail:{}'.format(term_uuid), mail)
def get_term_uuid_list(term, term_type):
return list(r_serv_term.smembers('all:tracked_term_uuid:{}:{}'.format(term_type, term)))
@ -336,7 +352,7 @@ def parse_get_tracker_term_item(dict_input, user_id):
date_to = dict_input.get('date_to', None)
if date_from is None:
date_from = r_serv_term.zrevrange('tracked_term:stat:{}'.format(term_uuid), 0, 0)
date_from = get_tracked_term_first_seen(term_uuid)
if date_from:
date_from = date_from[0]
@ -355,45 +371,83 @@ def parse_get_tracker_term_item(dict_input, user_id):
res_dict['items'] = all_item_id
return (res_dict, 200)
def get_tracked_term_first_seen(term_uuid):
res = r_serv_term.zrange('tracked_term:stat:{}'.format(term_uuid), 0, 0)
if res:
return res[0]
else:
return None
def get_tracked_term_last_seen(term_uuid):
res = r_serv_term.zrevrange('tracked_term:stat:{}'.format(term_uuid), 0, 0)
if res:
return res[0]
else:
return None
def get_term_metedata(term_uuid, user_id=False, level=False, tags=False, mails=False, sparkline=False):
dict_uuid = {}
dict_uuid['term'] = r_serv_term.hget('tracked_term:{}'.format(term_uuid), 'tracked')
dict_uuid['type'] = r_serv_term.hget('tracked_term:{}'.format(term_uuid), 'type')
dict_uuid['date'] = r_serv_term.hget('tracked_term:{}'.format(term_uuid), 'date')
dict_uuid['first_seen'] = get_tracked_term_first_seen(term_uuid)
dict_uuid['last_seen'] = get_tracked_term_last_seen(term_uuid)
if user_id:
dict_uuid['user_id'] = r_serv_term.hget('tracked_term:{}'.format(term_uuid), 'user_id')
if level:
dict_uuid['level'] = r_serv_term.hget('tracked_term:{}'.format(term_uuid), 'level')
if mails:
dict_uuid['mails'] = get_list_trackeed_term_mails(term_uuid)
if tags:
dict_uuid['tags'] = get_list_trackeed_term_tags(term_uuid)
if sparkline:
dict_uuid['sparkline'] = get_tracked_term_sparkline(term_uuid)
dict_uuid['uuid'] = term_uuid
return dict_uuid
def get_tracked_term_sparkline(term_uuid, num_day=6):
date_range_sparkline = Date.get_date_range(num_day)
sparklines_value = []
for date_day in date_range_sparkline:
nb_seen_this_day = r_serv_term.zscore('tracked_term:stat:{}'.format(term_uuid), date_day)
if nb_seen_this_day is None:
nb_seen_this_day = 0
sparklines_value.append(int(nb_seen_this_day))
return sparklines_value
def get_list_trackeed_term_tags(term_uuid):
res = r_serv_term.smembers('tracked_term:tags:{}'.format(term_uuid))
if res:
return list(res)
else:
return []
def get_list_trackeed_term_mails(term_uuid):
res = r_serv_term.smembers('tracked_term:mail:{}'.format(term_uuid))
if res:
return list(res)
else:
return []
def get_user_tracked_term_uuid(user_id):
return list(r_serv_term.smembers('user:tracked_term:{}'.format(user_id)))
def get_global_tracked_term_uuid():
return list(r_serv_term.smembers('global:tracked_term'))
def get_all_user_tracked_terms(user_id):
all_user_term = []
all_user_term_uuid = get_user_tracked_term_uuid(user_id)
for term_uuid in all_user_term_uuid:
all_user_term.append(get_term_metedata(term_uuid, tags=True, mails=True))
return all_user_term
def get_all_global_tracked_terms():
all_user_term = []
all_user_term_uuid = get_global_tracked_term_uuid()
def get_global_tracked_term():
dict_tracked = {}
tracked_set = list(r_serv_term.smembers('global:TrackedSetSet'))
tracked_regex = list(r_serv_term.smembers('global:TrackedRegexSet'))
tracked_terms = list(r_serv_term.smembers('global:TrackedSetTermSet'))
return {'term': tracked_terms, 'set': tracked_terms, 'regex': tracked_regex}
def get_user_tracked_term(user_id):
dict_tracked = {}
tracked_set = list(r_serv_term.smembers('user:{}:TrackedSetSet'.format(user_id)))
tracked_regex = list(r_serv_term.smembers('user:{}:TrackedRegexSet').format(user_id))
tracked_terms = list(r_serv_term.smembers('user:{}:TrackedSetTermSet').format(user_id))
return {'term': tracked_terms, 'set': tracked_terms, 'regex': tracked_regex}
for term_uuid in all_user_term_uuid:
all_user_term.append(get_term_metedata(term_uuid, user_id=True, tags=True, mails=True))
return all_user_term

View File

@ -708,7 +708,7 @@ curl https://127.0.0.1:7000/api/v1/add/tracker/term --header "Authorization: iHc
### Delete term tracker: `api/v1/add/tracker/term/item`<a name="delete_term_tracker"></a>
### Delete term tracker: `api/v1/delete/tracker/term/item`<a name="delete_term_tracker"></a>
#### Description
Delete term tracker

View File

@ -6,20 +6,25 @@
note: The matching of credential against supplied credential is done using Levenshtein distance
'''
import json
import redis
import datetime
import calendar
import flask
from flask import Flask, render_template, jsonify, request, Blueprint, url_for, redirect
from flask import Flask, render_template, jsonify, request, Blueprint, url_for, redirect, Response
from Role_Manager import login_admin, login_analyst
from flask_login import login_required
from flask_login import login_required, current_user
import re
import Paste
from pprint import pprint
import Levenshtein
# ---------------------------------------------------------------
import Paste
import Term
# ============ VARIABLES ============
import Flask_config
@ -146,337 +151,110 @@ def save_tag_to_auto_push(list_tag):
# ============ ROUTES ============
@terms.route("/terms_management/")
@terms.route("/tracker_term")
def tracked_term_menu():
user_id = current_user.get_id()
user_term = Term.get_all_user_tracked_terms(user_id)
global_term = Term.get_all_global_tracked_terms()
return render_template("tracker_term_management.html", user_term=user_term, global_term=global_term, bootstrap_label=bootstrap_label)
@terms.route("/tracker/add", methods=['GET', 'POST'])
@login_required
@login_analyst
def terms_management():
per_paste = request.args.get('per_paste')
if per_paste == "1" or per_paste is None:
per_paste_text = "per_paste_"
per_paste = 1
else:
per_paste_text = ""
per_paste = 0
def add_tracked_term_menu():
if request.method == 'POST':
term = request.form.get("term")
term_type = request.form.get("tracker_type")
nb_words = request.form.get("nb_word", 1)
level = request.form.get("level", 1)
tags = request.form.get("tags", [])
mails = request.form.get("mails", [])
today = datetime.datetime.now()
today = today.replace(hour=0, minute=0, second=0, microsecond=0)
today_timestamp = calendar.timegm(today.timetuple())
# Map tracking if notifications are enabled for a specific term
notificationEnabledDict = {}
# Maps a specific term to the associated email addresses
notificationEMailTermMapping = {}
notificationTagsTermMapping = {}
#Regex
trackReg_list = []
trackReg_list_values = []
trackReg_list_num_of_paste = []
for tracked_regex in r_serv_term.smembers(TrackedRegexSet_Name):
notificationEMailTermMapping[tracked_regex] = r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_regex)
notificationTagsTermMapping[tracked_regex] = r_serv_term.smembers(TrackedTermsNotificationTagsPrefix_Name + tracked_regex)
if tracked_regex not in notificationEnabledDict:
notificationEnabledDict[tracked_regex] = False
trackReg_list.append(tracked_regex)
value_range = Term_getValueOverRange(tracked_regex, today_timestamp, [1, 7, 31], per_paste=per_paste_text)
term_date = r_serv_term.hget(TrackedRegexDate_Name, tracked_regex)
set_paste_name = "regex_" + tracked_regex
trackReg_list_num_of_paste.append(r_serv_term.scard(set_paste_name))
term_date = datetime.datetime.utcfromtimestamp(int(term_date)) if term_date is not None else "No date recorded"
value_range.append(term_date)
trackReg_list_values.append(value_range)
if tracked_regex in r_serv_term.smembers(TrackedTermsNotificationEnabled_Name):
notificationEnabledDict[tracked_regex] = True
#Set
trackSet_list = []
trackSet_list_values = []
trackSet_list_num_of_paste = []
for tracked_set in r_serv_term.smembers(TrackedSetSet_Name):
tracked_set = tracked_set
notificationEMailTermMapping[tracked_set] = r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_set)
notificationTagsTermMapping[tracked_set] = r_serv_term.smembers(TrackedTermsNotificationTagsPrefix_Name + tracked_set)
if tracked_set not in notificationEnabledDict:
notificationEnabledDict[tracked_set] = False
trackSet_list.append(tracked_set)
value_range = Term_getValueOverRange(tracked_set, today_timestamp, [1, 7, 31], per_paste=per_paste_text)
term_date = r_serv_term.hget(TrackedSetDate_Name, tracked_set)
set_paste_name = "set_" + tracked_set
trackSet_list_num_of_paste.append(r_serv_term.scard(set_paste_name))
term_date = datetime.datetime.utcfromtimestamp(int(term_date)) if term_date is not None else "No date recorded"
value_range.append(term_date)
trackSet_list_values.append(value_range)
if tracked_set in r_serv_term.smembers(TrackedTermsNotificationEnabled_Name):
notificationEnabledDict[tracked_set] = True
#Tracked terms
track_list = []
track_list_values = []
track_list_num_of_paste = []
for tracked_term in r_serv_term.smembers(TrackedTermsSet_Name):
notificationEMailTermMapping[tracked_term] = r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_term)
notificationTagsTermMapping[tracked_term] = r_serv_term.smembers(TrackedTermsNotificationTagsPrefix_Name + tracked_term)
if tracked_term not in notificationEnabledDict:
notificationEnabledDict[tracked_term] = False
track_list.append(tracked_term)
value_range = Term_getValueOverRange(tracked_term, today_timestamp, [1, 7, 31], per_paste=per_paste_text)
term_date = r_serv_term.hget(TrackedTermsDate_Name, tracked_term)
set_paste_name = "tracked_" + tracked_term
track_list_num_of_paste.append( r_serv_term.scard(set_paste_name) )
term_date = datetime.datetime.utcfromtimestamp(int(term_date)) if term_date is not None else "No date recorded"
value_range.append(term_date)
track_list_values.append(value_range)
if tracked_term in r_serv_term.smembers(TrackedTermsNotificationEnabled_Name):
notificationEnabledDict[tracked_term] = True
#blacklist terms
black_list = []
for blacked_term in r_serv_term.smembers(BlackListTermsSet_Name):
term_date = r_serv_term.hget(BlackListTermsDate_Name, blacked_term)
term_date = datetime.datetime.utcfromtimestamp(int(term_date)) if term_date is not None else "No date recorded"
black_list.append([blacked_term, term_date])
return render_template("terms_management.html",
black_list=black_list, track_list=track_list, trackReg_list=trackReg_list, trackSet_list=trackSet_list,
track_list_values=track_list_values, track_list_num_of_paste=track_list_num_of_paste,
trackReg_list_values=trackReg_list_values, trackReg_list_num_of_paste=trackReg_list_num_of_paste,
trackSet_list_values=trackSet_list_values, trackSet_list_num_of_paste=trackSet_list_num_of_paste,
per_paste=per_paste, notificationEnabledDict=notificationEnabledDict, bootstrap_label=bootstrap_label,
notificationEMailTermMapping=notificationEMailTermMapping, notificationTagsTermMapping=notificationTagsTermMapping)
@terms.route("/terms_management_query_paste/")
@login_required
@login_analyst
def terms_management_query_paste():
term = request.args.get('term')
paste_info = []
# check if regex or not
if term.startswith('/') and term.endswith('/'):
set_paste_name = "regex_" + term
track_list_path = r_serv_term.smembers(set_paste_name)
elif term.startswith('\\') and term.endswith('\\'):
set_paste_name = "set_" + term
track_list_path = r_serv_term.smembers(set_paste_name)
else:
set_paste_name = "tracked_" + term
track_list_path = r_serv_term.smembers(set_paste_name)
for path in track_list_path:
paste = Paste.Paste(path)
p_date = str(paste._get_p_date())
p_date = p_date[0:4]+'/'+p_date[4:6]+'/'+p_date[6:8]
p_source = paste.p_source
p_size = paste.p_size
p_mime = paste.p_mime
p_lineinfo = paste.get_lines_info()
p_content = paste.get_p_content()
if p_content != 0:
p_content = p_content[0:400]
paste_info.append({"path": path, "date": p_date, "source": p_source, "size": p_size, "mime": p_mime, "lineinfo": p_lineinfo, "content": p_content})
return jsonify(paste_info)
@terms.route("/terms_management_query/")
@login_required
@login_analyst
def terms_management_query():
TrackedTermsDate_Name = "TrackedTermDate"
BlackListTermsDate_Name = "BlackListTermDate"
term = request.args.get('term')
section = request.args.get('section')
today = datetime.datetime.now()
today = today.replace(hour=0, minute=0, second=0, microsecond=0)
today_timestamp = calendar.timegm(today.timetuple())
value_range = Term_getValueOverRange(term, today_timestamp, [1, 7, 31])
if section == "followTerm":
term_date = r_serv_term.hget(TrackedTermsDate_Name, term)
elif section == "blacklistTerm":
term_date = r_serv_term.hget(BlackListTermsDate_Name, term)
term_date = datetime.datetime.utcfromtimestamp(int(term_date)) if term_date is not None else "No date recorded"
value_range.append(str(term_date))
return jsonify(value_range)
@terms.route("/terms_management_action/", methods=['GET'])
@login_required
@login_analyst
def terms_management_action():
today = datetime.datetime.now()
today = today.replace(microsecond=0)
today_timestamp = calendar.timegm(today.timetuple())
section = request.args.get('section')
action = request.args.get('action')
term = request.args.get('term')
notificationEmailsParam = request.args.get('emailAddresses')
input_tags = request.args.get('tags')
if action is None or term is None or notificationEmailsParam is None:
return "None"
else:
if section == "followTerm":
if action == "add":
# Make a list of all passed email addresses
notificationEmails = notificationEmailsParam.split()
validNotificationEmails = []
# check for valid email addresses
for email in notificationEmails:
# Really basic validation:
# has exactly one @ sign, and at least one . in the part after the @
if re.match(r"[^@]+@[^@]+\.[^@]+", email):
validNotificationEmails.append(email)
# create tags list
list_tags = input_tags.split()
# check if regex/set or simple term
#regex
if term.startswith('/') and term.endswith('/'):
r_serv_term.sadd(TrackedRegexSet_Name, term)
r_serv_term.hset(TrackedRegexDate_Name, term, today_timestamp)
# add all valid emails to the set
for email in validNotificationEmails:
r_serv_term.sadd(TrackedTermsNotificationEmailsPrefix_Name + term, email)
# enable notifications by default
r_serv_term.sadd(TrackedTermsNotificationEnabled_Name, term)
# add tags list
for tag in list_tags:
r_serv_term.sadd(TrackedTermsNotificationTagsPrefix_Name + term, tag)
save_tag_to_auto_push(list_tags)
#set
elif term.startswith('\\') and term.endswith('\\'):
tab_term = term[1:-1]
perc_finder = re.compile("\[[0-9]{1,3}\]").search(tab_term)
if perc_finder is not None:
match_percent = perc_finder.group(0)[1:-1]
set_to_add = term
else:
match_percent = DEFAULT_MATCH_PERCENT
set_to_add = "\\" + tab_term[:-1] + ", [{}]]\\".format(match_percent)
r_serv_term.sadd(TrackedSetSet_Name, set_to_add)
r_serv_term.hset(TrackedSetDate_Name, set_to_add, today_timestamp)
# add all valid emails to the set
for email in validNotificationEmails:
r_serv_term.sadd(TrackedTermsNotificationEmailsPrefix_Name + set_to_add, email)
# enable notifications by default
r_serv_term.sadd(TrackedTermsNotificationEnabled_Name, set_to_add)
# add tags list
for tag in list_tags:
r_serv_term.sadd(TrackedTermsNotificationTagsPrefix_Name + set_to_add, tag)
save_tag_to_auto_push(list_tags)
#simple term
else:
r_serv_term.sadd(TrackedTermsSet_Name, term.lower())
r_serv_term.hset(TrackedTermsDate_Name, term.lower(), today_timestamp)
# add all valid emails to the set
for email in validNotificationEmails:
r_serv_term.sadd(TrackedTermsNotificationEmailsPrefix_Name + term.lower(), email)
# enable notifications by default
r_serv_term.sadd(TrackedTermsNotificationEnabled_Name, term.lower())
# add tags list
for tag in list_tags:
r_serv_term.sadd(TrackedTermsNotificationTagsPrefix_Name + term.lower(), tag)
save_tag_to_auto_push(list_tags)
elif action == "toggleEMailNotification":
# get the current state
if term in r_serv_term.smembers(TrackedTermsNotificationEnabled_Name):
# remove it
r_serv_term.srem(TrackedTermsNotificationEnabled_Name, term.lower())
else:
# add it
r_serv_term.sadd(TrackedTermsNotificationEnabled_Name, term.lower())
#del action
else:
if term.startswith('/') and term.endswith('/'):
r_serv_term.srem(TrackedRegexSet_Name, term)
r_serv_term.hdel(TrackedRegexDate_Name, term)
elif term.startswith('\\') and term.endswith('\\'):
r_serv_term.srem(TrackedSetSet_Name, term)
r_serv_term.hdel(TrackedSetDate_Name, term)
else:
r_serv_term.srem(TrackedTermsSet_Name, term.lower())
r_serv_term.hdel(TrackedTermsDate_Name, term.lower())
# delete the associated notification emails too
r_serv_term.delete(TrackedTermsNotificationEmailsPrefix_Name + term)
# delete the associated tags set
r_serv_term.delete(TrackedTermsNotificationTagsPrefix_Name + term)
elif section == "blacklistTerm":
if action == "add":
r_serv_term.sadd(BlackListTermsSet_Name, term.lower())
r_serv_term.hset(BlackListTermsDate_Name, term, today_timestamp)
else:
r_serv_term.srem(BlackListTermsSet_Name, term.lower())
if mails:
mails = mails.split()
if tags:
tags = tags.split()
input_dict = {"term": term, "type": term_type, "nb_words": nb_words, "tags": tags, "mails": mails}
user_id = current_user.get_id()
res = Term.parse_json_term_to_add(input_dict, user_id)
if res[1] == 200:
return redirect(url_for('terms.tracked_term_menu'))
else:
return "None"
## TODO: use modal
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
else:
return render_template("Add_tracker.html")
to_return = {}
to_return["section"] = section
to_return["action"] = action
to_return["term"] = term
return jsonify(to_return)
@terms.route("/terms_management/delete_terms_tags", methods=['POST'])
@terms.route("/tracker/show_term_tracker")
@login_required
@login_analyst
def delete_terms_tags():
term = request.form.get('term')
tags_to_delete = request.form.getlist('tags_to_delete')
def show_term_tracker():
user_id = current_user.get_id()
term_uuid = request.args.get('uuid', None)
res = Term.check_term_uuid_valid_access(term_uuid, user_id)
if res: # invalid access
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
if term is not None and tags_to_delete is not None:
for tag in tags_to_delete:
r_serv_term.srem(TrackedTermsNotificationTagsPrefix_Name + term, tag)
return redirect(url_for('terms.terms_management'))
date_from = request.args.get('date_from')
date_to = request.args.get('date_to')
if date_from:
date_from = date_from.replace('-', '')
if date_to:
date_to = date_to.replace('-', '')
term_metadata = Term.get_term_metedata(term_uuid, user_id=True, level=True, tags=True, mails=True, sparkline=True)
if date_from:
res = Term.parse_get_tracker_term_item({'uuid': term_uuid, 'date_from': date_from, 'date_to': date_to}, user_id)
if res[1] !=200:
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
term_metadata['items'] = res[0]['items']
term_metadata['date_from'] = res[0]['date_from']
term_metadata['date_to'] = res[0]['date_to']
else:
return 'None args', 400
term_metadata['items'] = []
term_metadata['date_from'] = ''
term_metadata['date_to'] = ''
@terms.route("/terms_management/delete_terms_email", methods=['GET'])
return render_template("showTrackerTerm.html", term_metadata=term_metadata, bootstrap_label=bootstrap_label)
@terms.route("/tracker/update_tracker_tags", methods=['POST'])
@login_required
@login_analyst
def delete_terms_email():
term = request.args.get('term')
email = request.args.get('email')
if term is not None and email is not None:
r_serv_term.srem(TrackedTermsNotificationEmailsPrefix_Name + term, email)
return redirect(url_for('terms.terms_management'))
def update_tracker_tags():
user_id = current_user.get_id()
term_uuid = request.form.get('uuid')
res = Term.check_term_uuid_valid_access(term_uuid, user_id)
if res: # invalid access
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
tags = request.form.get('tags')
if tags:
tags = tags.split()
else:
return 'None args', 400
tags = []
Term.replace_tracked_term_tags(term_uuid, tags)
return redirect(url_for('terms.show_term_tracker', uuid=term_uuid))
@terms.route("/tracker/update_tracker_mails", methods=['POST'])
@login_required
@login_analyst
def update_tracker_mails():
user_id = current_user.get_id()
term_uuid = request.form.get('uuid')
res = Term.check_term_uuid_valid_access(term_uuid, user_id)
if res: # invalid access
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
mails = request.form.get('mails')
if mails:
mails = mails.split()
else:
mails = []
res = Term.replace_tracked_term_mails(term_uuid, mails)
if res: # invalid mail
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
return redirect(url_for('terms.show_term_tracker', uuid=term_uuid))
@terms.route("/terms_plot_tool/")

View File

@ -0,0 +1,153 @@
<!DOCTYPE html>
<html>
<head>
<title>AIL-Framework</title>
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png')}}">
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/daterangepicker.min.css') }}" rel="stylesheet">
<!-- JS -->
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
</head>
<body>
{% include 'nav_bar.html' %}
<div class="container-fluid">
<div class="row">
{% include 'crawler/menu_sidebar.html' %}
<div class="col-12 col-lg-10" id="core_content">
<div class="card mb-3 mt-1">
<div class="card-header">
<h5 class="card-title">Create a new tracker</h5>
</div>
<div class="card-body">
<p class="card-text">Enter a domain and choose what kind of data you want.</p>
<form action="{{ url_for('terms.add_tracked_term_menu') }}" method='post'>
<div class="row">
<div class="col-12 col-xl-9">
<div class="input-group mb-2 mr-sm-2">
<div class="input-group-prepend">
<div class="input-group-text"><i class="fas fa-tag"></i></div>
</div>
<input id="tags" name="tags" class="form-control" placeholder="Tags (optional, space separated)" type="text">
</div>
<div class="input-group mb-2 mr-sm-2">
<div class="input-group-prepend">
<div class="input-group-text"><i class="fas fa-at"></i></div>
</div>
<input id="mails" name="mails" class="form-control" placeholder="E-Mails Notification (optional, space separated)" type="text">
</div>
</div>
<div class="col-12 col-xl-3">
<div class="custom-control custom-switch mt-1">
<input class="custom-control-input" type="checkbox" name="level" id="id_level" checked>
<label class="custom-control-label" for="id_level">
<i class="fas fa-users"></i>&nbsp;Show tracker to all Users
</label>
</div>
</div>
</div>
<hr>
<select id="tracker_type" name="tracker_type" class="custom-select w-25 mb-3">
<option disabled selected value> -- Select a tracker type -- </option>
<option value="word">Word</option>
<option value="set">Set</option>
<option value="regex">Regex</option>
</select>
<p id="tracker_desc">Terms to track (space separated)</p>
<div class="row">
<div class="col-12 col-lg-10">
<input id="term" name="term" class="form-control" placeholder="Terms to track (space separated)" type="text">
</div>
<div class="col-12 col-lg-2">
<input type="number" id="nb_word" name="nb_word" name="quantity" min="1" placeholder="Nb of keywords">
</div>
</div>
<br>
<button class="btn btn-success mt-2">
<i class="fas fa-plus"></i> Add Tracker
</button>
</form>
</div>
</div>
</div>
</div>
</div>
</body>
<script>
var chart = {};
$(document).ready(function(){
$("#page-Crawler").addClass("active");
$("#nav_manual_crawler").addClass("active");
$("#tracker_desc").hide();
$("#term").hide();
$("#nb_word").hide();
$('#tracker_type').on('change', function() {
var tracker_type = this.value;
if (tracker_type=="word") {
$("#tracker_desc").text("Term to track:");
$("#tracker_desc").show();
$("#term").show();
$("#nb_word").hide();
} else if (tracker_type=="set") {
$("#tracker_desc").text("Terms to track (space separated), explain nb");
$("#tracker_desc").show();
$("#term").show();
$("#nb_word").show();
} else {
$("#tracker_desc").text("valid python regex");
$("#tracker_desc").show();
$("#term").show();
$("#nb_word").hide();
}
});
});
function toggle_sidebar(){
if($('#nav_menu').is(':visible')){
$('#nav_menu').hide();
$('#side_menu').removeClass('border-right')
$('#side_menu').removeClass('col-lg-2')
$('#core_content').removeClass('col-lg-10')
}else{
$('#nav_menu').show();
$('#side_menu').addClass('border-right')
$('#side_menu').addClass('col-lg-2')
$('#core_content').addClass('col-lg-10')
}
}
</script>

View File

@ -0,0 +1,319 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<title>AIL Framework - AIL</title>
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/daterangepicker.min.css') }}" rel="stylesheet">
<!-- JS -->
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
<script language="javascript" src="{{ url_for('static', filename='js/d3.min.js') }}"></script>
<script language="javascript" src="{{ url_for('static', filename='js/moment.min.js') }}"></script>
<script language="javascript" src="{{ url_for('static', filename='js/jquery.daterangepicker.min.js') }}"></script>
<style>
.line_sparkline {
fill: none;
stroke: #000;
stroke-width: 2.0px;
}
.btn-link {
color: #000000
}
.mouse_pointer{
cursor: pointer;
}
</style>
</head>
<body>
{% include 'nav_bar.html' %}
<div class="container-fluid">
<div class="row">
{% include 'tracker/menu_sidebar.html' %}
<div class="col-12 col-lg-10" id="core_content">
<div class="card my-3">
<div class="card-header" style="background-color:#d9edf7;font-size: 15px">
<h4 class="text-secondary">{{ term_metadata['uuid'] }} </h4>
<ul class="list-group mb-2">
<li class="list-group-item py-0">
<div class="row">
<div class="col-md-10">
<table class="table">
<thead>
<tr>
<th>Type</th>
<th>Tracked</th>
<th>Date added</th>
<th>Level</th>
<th>Created by</th>
<th>First seen</th>
<th>Last seen</th>
<th>Tags <span class="btn-link btn-interaction mouse_pointer" title="Edit Tags List" onclick="edit_tags();"><i class="fas fa-pencil-alt" style="color:Red;"></i></span></th>
<th>Email <span class="btn-link btn-interaction mouse_pointer" title="Edit Email List" onclick="edit_mails();"><i class="fas fa-pencil-alt" style="color:Red;"></i></span></th>
</tr>
</thead>
<tbody>
<tr>
<td>{{ term_metadata['type'] }}</td>
<td>{{ term_metadata['term'] }}</td>
<td>{{ term_metadata['date'][0:4] }}/{{ term_metadata['date'][4:6] }}/{{ term_metadata['date'][6:8] }}</td>
<td>{{ term_metadata['level'] }}</td>
<td>{{ term_metadata['user_id'] }}</td>
<td>
{% if term_metadata['first_seen'] %}
{{ term_metadata['first_seen'][0:4] }}/{{ term_metadata['first_seen'][4:6] }}/{{ term_metadata['first_seen'][6:8] }}
{% endif %}
</td>
<td>
{% if term_metadata['last_seen'] %}
{{ term_metadata['last_seen'][0:4] }}/{{ term_metadata['last_seen'][4:6] }}/{{ term_metadata['last_seen'][6:8] }}
{% endif %}
</td>
<td>
{% for tag in term_metadata['tags'] %}
<a href="{{ url_for('Tags.Tags_page') }}?ltags={{ tag }}">
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span>
</a>
{% endfor %}
</td>
<td>
{% for mail in term_metadata['mails'] %}
{{ mail }}<br>
{% endfor %}
</td>
</tr>
</tbody>
</table>
</div>
<div class="col-md-1">
<div id="sparkline"></div>
</div>
</div>
</li>
</ul>
<div id="div_edit_tags">
<form action="{{ url_for('terms.update_tracker_tags') }}" method='post'>
<input name="uuid" type="text" value="{{term_metadata['uuid']}}" hidden>
<div>All Tags added for this tracker, space separated: </div>
<div class="input-group mb-2 mr-sm-2">
<div class="input-group-prepend">
<div class="input-group-text"><i class="fas fa-tag"></i></div>
</div>
<input id="tags" name="tags" class="form-control" placeholder="Tags (optional, space separated)" type="text"
value="{% for tag in term_metadata['tags'] %}{{tag}} {% endfor %}">
</div>
<button class="btn btn-info">
<i class="fas fa-pencil-alt"></i> Edit Tags
</button>
</form>
</div>
<div id="div_edit_mails">
<form action="{{ url_for('terms.update_tracker_mails') }}" method='post'>
<input name="uuid" type="text" value="{{term_metadata['uuid']}}" hidden>
<div>All E-Mails to Notify for this tracker, space separated: </div>
<div class="input-group mb-2 mr-sm-2">
<div class="input-group-prepend">
<div class="input-group-text"><i class="fas fa-at"></i></div>
</div>
<input id="mails" name="mails" class="form-control" placeholder="E-Mails Notification (optional, space separated)" type="text"
value="{% for mail in term_metadata['mails'] %}{{mail}} {% endfor %}">
</div>
<button class="btn btn-info">
<i class="fas fa-pencil-alt"></i> Edit Email Notification
</button>
</form>
</div>
<a href="{{ url_for('hashDecoded.downloadHash') }}?hash={{hash}}" target="blank" class="float-right" style="font-size: 15px">
<button class='btn btn-danger'><i class="fas fa-trash-alt"></i>
</button>
</a>
</div>
</div>
<div class="card mb-3 mt-1">
<div class="card-body">
<div class="row mb-3">
<div class="col-md-6">
<div class="input-group" id="date-range-from">
<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
<input class="form-control" id="date-range-from-input" placeholder="yyyy-mm-dd" value="{{ term_metadata['date_from'] }}" name="date_from" autocomplete="off">
</div>
</div>
<div class="col-md-6">
<div class="input-group" id="date-range-to">
<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
<input class="form-control" id="date-range-to-input" placeholder="yyyy-mm-dd" value="{{ term_metadata['date_to'] }}" name="date_to" autocomplete="off">
</div>
</div>
</div>
<button class="btn btn-info" type="button" id="button-search-tags" onclick="getItems();">
<i class="fas fa-search"></i> Search Tracked Items
</button>
</div>
</div>
{%if term_metadata['items']%}
<table class="table table-bordered table-hover" id="myTable_">
<thead class="thead-dark">
<tr>
<th>Item</th>
</tr>
</thead>
<tbody>
{% for item in term_metadata['items'] %}
<tr>
<td>
<a target="_blank" href="{{ url_for('showsavedpastes.showsavedpaste') }}?paste={{item}}">
<div style="line-height:0.9;">{{ item }}</div>
</a>
</td>
</tr>
{% endfor %}
</tbody>
</table>
{% endif %}
</div>
</div>
</div>
<script>
$(document).ready(function(){
$('#div_edit_mails').hide();
$('#div_edit_tags').hide();
$("#page-Decoded").addClass("active");
$('#date-range-from').dateRangePicker({
separator : ' to ',
getValue: function(){
if ($('#date-range-from-input').val() && $('#date-range-to-input').val() )
return $('#date-range-from-input').val() + ' to ' + $('#date-range-to-input').val();
else
return '';
},
setValue: function(s,s1,s2){
$('#date-range-from-input').val(s1);
$('#date-range-to-input').val(s2);
}
});
$('#date-range-to').dateRangePicker({
separator : ' to ',
getValue: function(){
if ($('#date-range-from-input').val() && $('#date-range-to-input').val() )
return $('#date-range-from-input').val() + ' to ' + $('#date-range-to-input').val();
else
return '';
},
setValue: function(s,s1,s2){
$('#date-range-from-input').val(s1);
$('#date-range-to-input').val(s2);
}
});
sparklines("sparkline", {{ term_metadata['sparkline'] }});
});
function toggle_sidebar(){
if($('#nav_menu').is(':visible')){
$('#nav_menu').hide();
$('#side_menu').removeClass('border-right')
$('#side_menu').removeClass('col-lg-2')
$('#core_content').removeClass('col-lg-10')
}else{
$('#nav_menu').show();
$('#side_menu').addClass('border-right')
$('#side_menu').addClass('col-lg-2')
$('#core_content').addClass('col-lg-10')
}
}
function edit_tags(){
$('#div_edit_mails').hide();
$('#div_edit_tags').show();
}
function edit_mails(){
$('#div_edit_tags').hide();
$('#div_edit_mails').show();
}
function getItems() {
var date_from = $('#date-range-from-input').val();
var date_to =$('#date-range-to-input').val();
window.location.replace("{{ url_for('terms.show_term_tracker') }}?uuid={{ term_metadata['uuid'] }}&date_from="+date_from+"&date_to="+date_to);
}
</script>
<script>
//var data = [6,3,3,2,5,3,9];
// a sparklines plot
function sparklines(id, points) {
var width_spark = 100, height_spark = 60;
var data = []
for (i = 0; i < points.length; i++) {
data[i] = {
'x': i,
'y': +points[i]
}
}
var x = d3.scaleLinear()
.range([0, width_spark - 10])
.domain([0,5]);
var y = d3.scaleLinear()
.range([height_spark, 0])
.domain([0,10]);
var line = d3.line()
.x(function(d) {return x(d.x)})
.y(function(d) {return y(d.y)});
d3.select("#"+id).append('svg')
.attr('width', width_spark)
.attr('height', height_spark)
.append('path')
.attr('class','line_sparkline')
.datum(data)
.attr('d', line);
}
</script>
</body>
</html>

View File

@ -0,0 +1,181 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Terms Management</title>
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
<style>
.btn-link {
color: #000000
}
.mouse_pointer{
cursor: pointer;
}
.lb-md {
font-size: 16px;
}
</style>
</head>
<body>
{% include 'nav_bar.html' %}
<div class="container-fluid">
<div class="row">
{% include 'tracker/menu_sidebar.html' %}
<div class="col-12 col-lg-10" id="core_content">
<div class="card my-3">
<div class="card-header">
<h5 class="card-title">Your Tracked Terms</h5>
</div>
<div class="card-body">
<table id="table_user_terms" class="table table-striped table-bordered">
<thead class="bg-dark text-white">
<tr>
<th>Type</th>
<th>Tracked Term</th>
<th>First seen</th>
<th>Last seen</th>
<th>Email notification</th>
</tr>
</thead>
<tbody style="font-size: 15px;">
{% for dict_uuid in user_term %}
<tr>
<td>{{dict_uuid['type']}}</td>
<td>
<span><a target="_blank" href="{{ url_for('terms.show_term_tracker') }}?uuid={{ dict_uuid['uuid'] }}">{{dict_uuid['term']}}</a></span>
<div>
{% for tag in dict_uuid['tags'] %}
<a href="{{ url_for('Tags.Tags_page') }}?ltags={{ tag }}">
<span class="label label-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }}</span>
</a>
{% endfor %}
</div>
</td>
<td>
{% if dict_uuid['first_seen'] %}
{{dict_uuid['first_seen'][0:4]}}/{{dict_uuid['first_seen'][4:6]}}/{{dict_uuid['first_seen'][6:8]}}
{% endif %}
</td>
<td>
{% if dict_uuid['last_seen'] %}
{{dict_uuid['last_seen'][0:4]}}/{{dict_uuid['last_seen'][4:6]}}/{{dict_uuid['last_seen'][6:8]}}
{% endif %}
</td>
<td>
{% for mail in dict_uuid['mails'] %}
{{ mail }}<br>
{% endfor %}
</td>
</tr>
{% endfor %}
</tbody>
</table>
</div>
</div>
<div class="card my-3">
<div class="card-header">
<h5 class="card-title">Global Tracked Terms</h5>
</div>
<div class="card-body">
<table id="table_global_terms" class="table table-striped table-bordered">
<thead class="bg-dark text-white">
<tr>
<th>Type</th>
<th>Tracked Term</th>
<th>First seen</th>
<th>Last seen</th>
<th>Email notification</th>
</tr>
</thead>
<tbody style="font-size: 15px;">
{% for dict_uuid in global_term %}
<tr>
<td>{{dict_uuid['type']}}</td>
<td>
<span><a target="_blank" href="{{ url_for('terms.show_term_tracker') }}?uuid={{ dict_uuid['uuid'] }}">{{dict_uuid['term']}}</a></span>
<div>
{% for tag in dict_uuid['tags'] %}
<a href="{{ url_for('Tags.Tags_page') }}?ltags={{ tag }}">
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span>
</a>
{% endfor %}
</div>
</td>
<td>
{% if dict_uuid['first_seen'] %}
{{dict_uuid['first_seen'][0:4]}}/{{dict_uuid['first_seen'][4:6]}}/{{dict_uuid['first_seen'][6:8]}}
{% endif %}
</td>
<td>
{% if dict_uuid['last_seen'] %}
{{dict_uuid['last_seen'][0:4]}}/{{dict_uuid['last_seen'][4:6]}}/{{dict_uuid['last_seen'][6:8]}}
{% endif %}
</td>
<td>
{% for mail in dict_uuid['mails'] %}
{{ mail }}<br>
{% endfor %}
</td>
</tr>
{% endfor %}
</tbody>
</table>
</div>
</div>
</div>
</div>
</div>
<script>
$(document).ready(function(){
$('#table_user_terms').DataTable({
"aLengthMenu": [[5, 10, 15, -1], [5, 10, 15, "All"]],
"iDisplayLength": 10,
"order": [[ 0, "desc" ]]
});
$('#table_global_terms').DataTable({
"aLengthMenu": [[5, 10, 15, -1], [5, 10, 15, "All"]],
"iDisplayLength": 10,
"order": [[ 0, "desc" ]]
});
});
function toggle_sidebar(){
if($('#nav_menu').is(':visible')){
$('#nav_menu').hide();
$('#side_menu').removeClass('border-right')
$('#side_menu').removeClass('col-lg-2')
$('#core_content').removeClass('col-lg-10')
}else{
$('#nav_menu').show();
$('#side_menu').addClass('border-right')
$('#side_menu').addClass('col-lg-2')
$('#core_content').addClass('col-lg-10')
}
}
</script>
</body>
</html>

View File

@ -19,7 +19,7 @@
<a class="nav-link" id="page-Browse-Items" href="{{ url_for('Tags.Tags_page') }}" aria-disabled="true"><i class="fas fa-tag"></i> Browse Items</a>
</li>
<li class="nav-item mr-3">
<a class="nav-link" href="{{ url_for('terms.terms_management') }}" aria-disabled="true"><i class="fas fa-crosshairs"></i> Leaks Hunter</a>
<a class="nav-link" id="page-Tracker" href="{{ url_for('terms.tracked_term_menu') }}" aria-disabled="true"><i class="fas fa-crosshairs"></i> Leaks Hunter</a>
</li>
<li class="nav-item mr-3">
<a class="nav-link" id="page-Crawler" href="{{ url_for('hiddenServices.dashboard') }}" tabindex="-1" aria-disabled="true"><i class="fas fa-spider"></i> Crawlers</a>

View File

@ -0,0 +1,24 @@
<div class="col-12 col-lg-2 p-0 bg-light border-right" id="side_menu">
<button type="button" class="btn btn-outline-secondary mt-1 ml-3" onclick="toggle_sidebar()">
<i class="fas fa-align-left"></i>
<span>Toggle Sidebar</span>
</button>
<nav class="navbar navbar-expand navbar-light bg-light flex-md-column flex-row align-items-start py-2" id="nav_menu">
<h5 class="d-flex text-muted w-100">
<span>Terms Tracker </span>
<a class="ml-auto" href="{{url_for('terms.add_tracked_term_menu')}}">
<i class="fas fa-plus-circle ml-auto"></i>
</a>
</h5>
<ul class="nav flex-md-column flex-row navbar-nav justify-content-between w-100"> <!--nav-pills-->
<li class="nav-item">
<a class="nav-link" href="{{url_for('terms.tracked_term_menu')}}" id="nav_dashboard">
<i class="fas fa-search"></i>
<span>Tracked Terms</span>
</a>
</li>
</ul>
</nav>
</div>