mirror of https://github.com/CIRCL/AIL-framework
chg: [Terms] tag tracked terms
Terrtia
parent
7ef5cc3205
commit
979472df15
18
OVERVIEW.md
18
OVERVIEW.md
|
|
@ -26,6 +26,24 @@ ARDB overview
|
|||
ARDB_DB
|
||||
* DB 1 - Curve
|
||||
* DB 2 - TermFreq
|
||||
----------------------------------------- TERM ----------------------------------------
|
||||
|
||||
SET - 'TrackedRegexSet' term
|
||||
|
||||
HSET - 'TrackedRegexDate' tracked_regex today_timestamp
|
||||
|
||||
SET - 'TrackedSetSet' set_to_add
|
||||
|
||||
HSET - 'TrackedSetDate' set_to_add today_timestamp
|
||||
|
||||
SET - 'TrackedSetTermSet' term
|
||||
|
||||
HSET - 'TrackedTermDate' tracked_regex today_timestamp
|
||||
|
||||
SET - 'TrackedNotificationEmails_'+term/set email
|
||||
|
||||
SET - 'TrackedNotifications' term/set
|
||||
|
||||
* DB 3 - Trending
|
||||
* DB 4 - Sentiment
|
||||
* DB 5 - TermCred
|
||||
|
|
|
|||
|
|
@ -48,6 +48,8 @@ top_termFreq_setName_week = ["TopTermFreq_set_week", 7]
|
|||
top_termFreq_setName_month = ["TopTermFreq_set_month", 31]
|
||||
top_termFreq_set_array = [top_termFreq_setName_day,top_termFreq_setName_week, top_termFreq_setName_month]
|
||||
|
||||
TrackedTermsNotificationTagsPrefix_Name = "TrackedNotificationTags_"
|
||||
|
||||
# create direct link in mail
|
||||
full_paste_url = "/showsavedpaste/?paste="
|
||||
|
||||
|
|
@ -71,6 +73,11 @@ def check_if_tracked_term(term, path):
|
|||
for email in server_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + term):
|
||||
sendEmailNotification(email, 'Term', mail_body)
|
||||
|
||||
# tag paste
|
||||
for tag in server_term.smembers(TrackedTermsNotificationTagsPrefix_Name + term):
|
||||
msg = '{};{}'.format(tag, path)
|
||||
p.populate_set_out(msg, 'Tags')
|
||||
|
||||
|
||||
def getValueOverRange(word, startDate, num_day):
|
||||
to_return = 0
|
||||
|
|
|
|||
|
|
@ -42,6 +42,8 @@ top_termFreq_setName_week = ["TopTermFreq_set_week", 7]
|
|||
top_termFreq_setName_month = ["TopTermFreq_set_month", 31]
|
||||
top_termFreq_set_array = [top_termFreq_setName_day, top_termFreq_setName_week, top_termFreq_setName_month]
|
||||
|
||||
TrackedTermsNotificationTagsPrefix_Name = "TrackedNotificationTags_"
|
||||
|
||||
# create direct link in mail
|
||||
full_paste_url = "/showsavedpaste/?paste="
|
||||
|
||||
|
|
@ -129,6 +131,11 @@ if __name__ == "__main__":
|
|||
for email in server_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + regex_str_complete):
|
||||
sendEmailNotification(email, 'Term', mail_body)
|
||||
|
||||
# tag paste
|
||||
for tag in server_term.smembers(TrackedTermsNotificationTagsPrefix_Name + regex_str_complete):
|
||||
msg = '{};{}'.format(tag, filename)
|
||||
p.populate_set_out(msg, 'Tags')
|
||||
|
||||
set_name = 'regex_' + dico_regexname_to_redis[regex_str]
|
||||
new_to_the_set = server_term.sadd(set_name, filename)
|
||||
new_to_the_set = True if new_to_the_set == 1 else False
|
||||
|
|
|
|||
|
|
@ -34,6 +34,8 @@ top_termFreq_setName_week = ["TopTermFreq_set_week", 7]
|
|||
top_termFreq_setName_month = ["TopTermFreq_set_month", 31]
|
||||
top_termFreq_set_array = [top_termFreq_setName_day,top_termFreq_setName_week, top_termFreq_setName_month]
|
||||
|
||||
TrackedTermsNotificationTagsPrefix_Name = "TrackedNotificationTags_"
|
||||
|
||||
# create direct link in mail
|
||||
full_paste_url = "/showsavedpaste/?paste="
|
||||
|
||||
|
|
@ -121,6 +123,11 @@ if __name__ == "__main__":
|
|||
for email in server_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + dico_setname_to_redis[str(the_set)]):
|
||||
sendEmailNotification(email, 'Term', mail_body)
|
||||
|
||||
# tag paste
|
||||
for tag in server_term.smembers(TrackedTermsNotificationTagsPrefix_Name + dico_setname_to_redis[str(the_set)]):
|
||||
msg = '{};{}'.format(tag, filename)
|
||||
p.populate_set_out(msg, 'Tags')
|
||||
|
||||
print(the_set, "matched in", filename)
|
||||
set_name = 'set_' + dico_setname_to_redis[the_set]
|
||||
new_to_the_set = server_term.sadd(set_name, filename)
|
||||
|
|
|
|||
|
|
@ -32,13 +32,15 @@ publish = Redis_Words
|
|||
|
||||
[Curve]
|
||||
subscribe = Redis_Words
|
||||
publish = Redis_CurveManageTopSets
|
||||
publish = Redis_CurveManageTopSets,Redis_Tags
|
||||
|
||||
[RegexForTermsFrequency]
|
||||
subscribe = Redis_Global
|
||||
publish = Redis_Tags
|
||||
|
||||
[SetForTermsFrequency]
|
||||
subscribe = Redis_Global
|
||||
publish = Redis_Tags
|
||||
|
||||
[CurveManageTopSets]
|
||||
subscribe = Redis_CurveManageTopSets
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ cfg = Flask_config.cfg
|
|||
baseUrl = Flask_config.baseUrl
|
||||
r_serv_term = Flask_config.r_serv_term
|
||||
r_serv_cred = Flask_config.r_serv_cred
|
||||
bootstrap_label = Flask_config.bootstrap_label
|
||||
|
||||
terms = Blueprint('terms', __name__, template_folder='templates')
|
||||
|
||||
|
|
@ -51,6 +52,7 @@ TrackedTermsNotificationEnabled_Name = "TrackedNotifications"
|
|||
# same value as in `bin/NotificationHelper.py`
|
||||
# Keys will be e.g. TrackedNotificationEmails_<TERMNAME>
|
||||
TrackedTermsNotificationEmailsPrefix_Name = "TrackedNotificationEmails_"
|
||||
TrackedTermsNotificationTagsPrefix_Name = "TrackedNotificationTags_"
|
||||
|
||||
'''CRED'''
|
||||
REGEX_CRED = '[a-z]+|[A-Z]{3,}|[A-Z]{1,2}[a-z]+|[0-9]+'
|
||||
|
|
@ -152,6 +154,7 @@ def terms_management():
|
|||
|
||||
# Maps a specific term to the associated email addresses
|
||||
notificationEMailTermMapping = {}
|
||||
notificationTagsTermMapping = {}
|
||||
|
||||
#Regex
|
||||
trackReg_list = []
|
||||
|
|
@ -160,6 +163,7 @@ def terms_management():
|
|||
for tracked_regex in r_serv_term.smembers(TrackedRegexSet_Name):
|
||||
|
||||
notificationEMailTermMapping[tracked_regex] = "\n".join( (r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_regex)) )
|
||||
notificationTagsTermMapping[tracked_regex] = r_serv_term.smembers(TrackedTermsNotificationTagsPrefix_Name + tracked_regex)
|
||||
|
||||
if tracked_regex not in notificationEnabledDict:
|
||||
notificationEnabledDict[tracked_regex] = False
|
||||
|
|
@ -186,7 +190,7 @@ def terms_management():
|
|||
tracked_set = tracked_set
|
||||
|
||||
notificationEMailTermMapping[tracked_set] = "\n".join( (r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_set)) )
|
||||
|
||||
notificationTagsTermMapping[tracked_set] = r_serv_term.smembers(TrackedTermsNotificationTagsPrefix_Name + tracked_set)
|
||||
|
||||
if tracked_set not in notificationEnabledDict:
|
||||
notificationEnabledDict[tracked_set] = False
|
||||
|
|
@ -212,6 +216,7 @@ def terms_management():
|
|||
for tracked_term in r_serv_term.smembers(TrackedTermsSet_Name):
|
||||
|
||||
notificationEMailTermMapping[tracked_term] = "\n".join( r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_term))
|
||||
notificationTagsTermMapping[tracked_term] = r_serv_term.smembers(TrackedTermsNotificationTagsPrefix_Name + tracked_term)
|
||||
|
||||
if tracked_term not in notificationEnabledDict:
|
||||
notificationEnabledDict[tracked_term] = False
|
||||
|
|
@ -239,12 +244,14 @@ def terms_management():
|
|||
term_date = datetime.datetime.utcfromtimestamp(int(term_date)) if term_date is not None else "No date recorded"
|
||||
black_list.append([blacked_term, term_date])
|
||||
|
||||
print(notificationTagsTermMapping)
|
||||
return render_template("terms_management.html",
|
||||
black_list=black_list, track_list=track_list, trackReg_list=trackReg_list, trackSet_list=trackSet_list,
|
||||
track_list_values=track_list_values, track_list_num_of_paste=track_list_num_of_paste,
|
||||
trackReg_list_values=trackReg_list_values, trackReg_list_num_of_paste=trackReg_list_num_of_paste,
|
||||
trackSet_list_values=trackSet_list_values, trackSet_list_num_of_paste=trackSet_list_num_of_paste,
|
||||
per_paste=per_paste, notificationEnabledDict=notificationEnabledDict, notificationEMailTermMapping=notificationEMailTermMapping)
|
||||
per_paste=per_paste, notificationEnabledDict=notificationEnabledDict, bootstrap_label=bootstrap_label,
|
||||
notificationEMailTermMapping=notificationEMailTermMapping, notificationTagsTermMapping=notificationTagsTermMapping)
|
||||
|
||||
|
||||
@terms.route("/terms_management_query_paste/")
|
||||
|
|
@ -313,6 +320,7 @@ def terms_management_action():
|
|||
action = request.args.get('action')
|
||||
term = request.args.get('term')
|
||||
notificationEmailsParam = request.args.get('emailAddresses')
|
||||
input_tags = request.args.get('tags')
|
||||
|
||||
if action is None or term is None or notificationEmailsParam is None:
|
||||
return "None"
|
||||
|
|
@ -334,6 +342,8 @@ def terms_management_action():
|
|||
if re.match(r"[^@]+@[^@]+\.[^@]+", email):
|
||||
validNotificationEmails.append(email)
|
||||
|
||||
# create tags list
|
||||
list_tags = input_tags.split()
|
||||
|
||||
# check if regex/set or simple term
|
||||
#regex
|
||||
|
|
@ -345,6 +355,9 @@ def terms_management_action():
|
|||
r_serv_term.sadd(TrackedTermsNotificationEmailsPrefix_Name + term, email)
|
||||
# enable notifications by default
|
||||
r_serv_term.sadd(TrackedTermsNotificationEnabled_Name, term)
|
||||
# add tags list
|
||||
for tag in list_tags:
|
||||
r_serv_term.sadd(TrackedTermsNotificationTagsPrefix_Name + term, tag)
|
||||
|
||||
#set
|
||||
elif term.startswith('\\') and term.endswith('\\'):
|
||||
|
|
@ -363,6 +376,9 @@ def terms_management_action():
|
|||
r_serv_term.sadd(TrackedTermsNotificationEmailsPrefix_Name + set_to_add, email)
|
||||
# enable notifications by default
|
||||
r_serv_term.sadd(TrackedTermsNotificationEnabled_Name, set_to_add)
|
||||
# add tags list
|
||||
for tag in list_tags:
|
||||
r_serv_term.sadd(TrackedTermsNotificationTagsPrefix_Name + set_to_add, tag)
|
||||
|
||||
#simple term
|
||||
else:
|
||||
|
|
@ -373,6 +389,9 @@ def terms_management_action():
|
|||
r_serv_term.sadd(TrackedTermsNotificationEmailsPrefix_Name + term.lower(), email)
|
||||
# enable notifications by default
|
||||
r_serv_term.sadd(TrackedTermsNotificationEnabled_Name, term.lower())
|
||||
# add tags list
|
||||
for tag in list_tags:
|
||||
r_serv_term.sadd(TrackedTermsNotificationTagsPrefix_Name + term.lower(), tag)
|
||||
|
||||
elif action == "toggleEMailNotification":
|
||||
# get the current state
|
||||
|
|
@ -397,6 +416,8 @@ def terms_management_action():
|
|||
|
||||
# delete the associated notification emails too
|
||||
r_serv_term.delete(TrackedTermsNotificationEmailsPrefix_Name + term)
|
||||
# delete the associated tags set
|
||||
r_serv_term.delete(TrackedTermsNotificationTagsPrefix_Name + term)
|
||||
|
||||
elif section == "blacklistTerm":
|
||||
if action == "add":
|
||||
|
|
|
|||
|
|
@ -98,7 +98,8 @@
|
|||
<div class="form-group input-group" style="margin-bottom: 30px;">
|
||||
<span class="input-group-addon"><span class="fa fa-eye"></span></span>
|
||||
<input id="followTermInput" class="form-control" placeholder="Term to track." type="text" style="max-width: 400px;">
|
||||
<input id="followTermEMailNotificationReceiversInput" class="form-control" placeholder="Notification E-Mails (comma separated)" type="text" style="max-width: 400px;">
|
||||
<input id="followTermEMailNotificationReceiversInput" class="form-control" placeholder="Notification E-Mails (optional, comma separated)" type="text" style="max-width: 400px;">
|
||||
<input id="followTermTag" class="form-control" placeholder="Tags (optional, space separated)" type="text" style="max-width: 400px;">
|
||||
<button id="followTermBtn" class="btn btn-success btn-interaction" style="margin-left: 10px;" data-section="followTerm" data-action="add"> Add term</button>
|
||||
</div>
|
||||
|
||||
|
|
@ -119,7 +120,16 @@
|
|||
<!-- SET -->
|
||||
{% for set in trackSet_list %}
|
||||
<tr style="background-color: #cdffca;">
|
||||
<td>{{ set }}</td>
|
||||
<td>
|
||||
<span class="term_name">{{ set }}</span>
|
||||
<div>
|
||||
{% for tag in notificationTagsTermMapping[set] %}
|
||||
<a href="{{ url_for('Tags.get_tagged_paste') }}?ltags={{ tag }}">
|
||||
<span class="label label-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }}</span>
|
||||
</a>
|
||||
{% endfor %}
|
||||
</div>
|
||||
</td>
|
||||
<td>{{ trackSet_list_values[loop.index0][3] }}</td>
|
||||
<td>{{ trackSet_list_values[loop.index0][0] }}</td>
|
||||
<td>{{ trackSet_list_values[loop.index0][1] }}</td>
|
||||
|
|
@ -136,7 +146,16 @@
|
|||
<!-- REGEX -->
|
||||
{% for regex in trackReg_list %}
|
||||
<tr style="background-color: #fffdca;">
|
||||
<td>{{ regex }}</td>
|
||||
<td>
|
||||
<span class="term_name">{{ regex }}</span>
|
||||
<div>
|
||||
{% for tag in notificationTagsTermMapping[regex] %}
|
||||
<a href="{{ url_for('Tags.get_tagged_paste') }}?ltags={{ tag }}">
|
||||
<span class="label label-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }}</span>
|
||||
</a>
|
||||
{% endfor %}
|
||||
</div>
|
||||
</td>
|
||||
<td>{{ trackReg_list_values[loop.index0][3] }}</td>
|
||||
<td>{{ trackReg_list_values[loop.index0][0] }}</td>
|
||||
<td>{{ trackReg_list_values[loop.index0][1] }}</td>
|
||||
|
|
@ -153,7 +172,16 @@
|
|||
<!-- Normal term -->
|
||||
{% for term in track_list %}
|
||||
<tr>
|
||||
<td>{{ term }}</td>
|
||||
<td>
|
||||
<span class="term_name">{{ term }}</span>
|
||||
<div>
|
||||
{% for tag in notificationTagsTermMapping[term] %}
|
||||
<a href="{{ url_for('Tags.get_tagged_paste') }}?ltags={{ tag }}">
|
||||
<span class="label label-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }}</span>
|
||||
</a>
|
||||
{% endfor %}
|
||||
</div>
|
||||
</td>
|
||||
<td>{{ track_list_values[loop.index0][3] }}</td>
|
||||
<td>{{ track_list_values[loop.index0][0] }}</td>
|
||||
<td>{{ track_list_values[loop.index0][1] }}</td>
|
||||
|
|
@ -351,17 +379,19 @@ function perform_binding() {
|
|||
function perform_operation(){
|
||||
var curr_section = $(this).attr('data-section');
|
||||
var curr_action = $(this).attr('data-action');
|
||||
var row_tr = $(this).closest("tr");
|
||||
if (curr_action == "add") {
|
||||
var curr_term = $('#'+curr_section+'Input').val();
|
||||
var email_addresses = $('#followTermEMailNotificationReceiversInput').val();
|
||||
var tags = $('#followTermTag').val();
|
||||
} else {
|
||||
var curr_term = $(this).attr('data-content');
|
||||
var email_addresses = "";
|
||||
}
|
||||
var data_to_send = { section: curr_section, action: curr_action, term: curr_term, emailAddresses: email_addresses};
|
||||
var data_to_send = { section: curr_section, action: curr_action, term: curr_term, emailAddresses: email_addresses, tags: tags};
|
||||
|
||||
if (curr_term != "") {
|
||||
console.log(data_to_send);
|
||||
//console.log(data_to_send);
|
||||
$.get("{{ url_for('terms.terms_management_action') }}", data_to_send, function(data, status){
|
||||
if(status == "success") {
|
||||
var json = data;
|
||||
|
|
@ -372,13 +402,8 @@ function perform_operation(){
|
|||
$.get("{{ url_for('terms.terms_management_query') }}", { term: json.term, section: json.section }, function(data2, status){
|
||||
reload_per_paste();
|
||||
});
|
||||
} else if (json.action == "delete") {
|
||||
// Find indexes of row which have the term in the first column
|
||||
var index = table_track.rows().eq( 0 ).filter( function (rowIdx) {
|
||||
console.log(table_track.cell( rowIdx, 0 ).data())
|
||||
return table_track.cell( rowIdx, 0 ).data() === json.term;
|
||||
} );
|
||||
table_track.rows(index).remove().draw( false );
|
||||
} else if (json.action == "delete") {
|
||||
row_tr.remove()
|
||||
}
|
||||
} else if(json.section == "blacklistTerm"){
|
||||
if(json.action == "add") {
|
||||
|
|
|
|||
Loading…
Reference in New Issue