mirror of https://github.com/CIRCL/AIL-framework
111 lines
4.1 KiB
Python
Executable File
111 lines
4.1 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
# -*-coding:UTF-8 -*
|
|
|
|
import os
|
|
import re
|
|
import sys
|
|
import time
|
|
import redis
|
|
import datetime
|
|
|
|
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))
|
|
import Item
|
|
import Term
|
|
|
|
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
|
import ConfigLoader
|
|
|
|
|
|
def rreplace(s, old, new, occurrence):
|
|
li = s.rsplit(old, occurrence)
|
|
return new.join(li)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
|
|
start_deb = time.time()
|
|
|
|
config_loader = ConfigLoader.ConfigLoader()
|
|
|
|
r_serv_term_stats = config_loader.get_redis_conn("ARDB_Trending")
|
|
r_serv_termfreq = config_loader.get_redis_conn("ARDB_TermFreq")
|
|
config_loader = None
|
|
|
|
r_serv_term_stats.flushdb()
|
|
|
|
#convert all regex:
|
|
all_regex = r_serv_termfreq.smembers('TrackedRegexSet')
|
|
for regex in all_regex:
|
|
tags = list( r_serv_termfreq.smembers('TrackedNotificationTags_{}'.format(regex)) )
|
|
mails = list( r_serv_termfreq.smembers('TrackedNotificationEmails_{}'.format(regex)) )
|
|
|
|
new_term = regex[1:-1]
|
|
res = Term.parse_json_term_to_add({"term": new_term, "type": 'regex', "tags": tags, "mails": mails, "level": 1}, 'admin@admin.test')
|
|
if res[1] == 200:
|
|
term_uuid = res[0]['uuid']
|
|
list_items = r_serv_termfreq.smembers('regex_{}'.format(regex))
|
|
for paste_item in list_items:
|
|
item_id = Item.get_item_id(paste_item)
|
|
item_date = Item.get_item_date(item_id)
|
|
Term.add_tracked_item(term_uuid, item_id, item_date)
|
|
|
|
# Invalid Tracker => remove it
|
|
else:
|
|
print('Invalid Regex Removed: {}'.format(regex))
|
|
print(res[0])
|
|
# allow reprocess
|
|
r_serv_termfreq.srem('TrackedRegexSet', regex)
|
|
|
|
all_tokens = r_serv_termfreq.smembers('TrackedSetTermSet')
|
|
for token in all_tokens:
|
|
tags = list( r_serv_termfreq.smembers('TrackedNotificationTags_{}'.format(token)) )
|
|
mails = list( r_serv_termfreq.smembers('TrackedNotificationEmails_{}'.format(token)) )
|
|
|
|
res = Term.parse_json_term_to_add({"term": token, "type": 'word', "tags": tags, "mails": mails, "level": 1}, 'admin@admin.test')
|
|
if res[1] == 200:
|
|
term_uuid = res[0]['uuid']
|
|
list_items = r_serv_termfreq.smembers('tracked_{}'.format(token))
|
|
for paste_item in list_items:
|
|
item_id = Item.get_item_id(paste_item)
|
|
item_date = Item.get_item_date(item_id)
|
|
Term.add_tracked_item(term_uuid, item_id, item_date)
|
|
# Invalid Tracker => remove it
|
|
else:
|
|
print('Invalid Token Removed: {}'.format(token))
|
|
print(res[0])
|
|
# allow reprocess
|
|
r_serv_termfreq.srem('TrackedSetTermSet', token)
|
|
|
|
all_set = r_serv_termfreq.smembers('TrackedSetSet')
|
|
for curr_set in all_set:
|
|
tags = list( r_serv_termfreq.smembers('TrackedNotificationTags_{}'.format(curr_set)) )
|
|
mails = list( r_serv_termfreq.smembers('TrackedNotificationEmails_{}'.format(curr_set)) )
|
|
|
|
to_remove = ',{}'.format(curr_set.split(',')[-1])
|
|
new_set = rreplace(curr_set, to_remove, '', 1)
|
|
new_set = new_set[2:]
|
|
new_set = new_set.replace(',', '')
|
|
|
|
res = Term.parse_json_term_to_add({"term": new_set, "type": 'set', "nb_words": 1, "tags": tags, "mails": mails, "level": 1}, 'admin@admin.test')
|
|
if res[1] == 200:
|
|
term_uuid = res[0]['uuid']
|
|
list_items = r_serv_termfreq.smembers('tracked_{}'.format(curr_set))
|
|
for paste_item in list_items:
|
|
item_id = Item.get_item_id(paste_item)
|
|
item_date = Item.get_item_date(item_id)
|
|
Term.add_tracked_item(term_uuid, item_id, item_date)
|
|
# Invalid Tracker => remove it
|
|
else:
|
|
print('Invalid Set Removed: {}'.format(curr_set))
|
|
print(res[0])
|
|
# allow reprocess
|
|
r_serv_termfreq.srem('TrackedSetSet', curr_set)
|
|
|
|
r_serv_termfreq.flushdb()
|
|
|
|
#Set current ail version
|
|
r_serv.set('ail:version', 'v2.2')
|
|
|
|
#Set current ail version
|
|
r_serv.hset('ail:update_date', 'v2.2', datetime.datetime.now().strftime("%Y%m%d"))
|