CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

Added save of paste_baseName in set for searches. This is done in ModuleStats to not surcharge the Global module.

pull/68/head
Mokaddem 2016-08-23 08:52:54 +02:00
parent 6be557890c
commit 047331df3e
2 changed files with 44 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
bin/ModuleStats.py
View File

@ -58,11 +58,12 @@ def compute_most_posted(server, message):
print redis_progression_name_set print redis_progression_name_set
def compute_provider_info(server, path): def compute_provider_info(server_trend, server_pasteName, path):
redis_all_provider = 'all_provider_set' redis_all_provider = 'all_provider_set'
paste = Paste.Paste(path) paste = Paste.Paste(path)
paste_baseName = paste.p_name.split('.')[0]
paste_size = paste._get_p_size() paste_size = paste._get_p_size()
paste_provider = paste.p_source paste_provider = paste.p_source
paste_date = str(paste._get_p_date()) paste_date = str(paste._get_p_date())
@ -71,12 +72,13 @@ def compute_provider_info(server, path):
redis_providers_name_set = 'providers_set_' + paste_date redis_providers_name_set = 'providers_set_' + paste_date
# Add/Update in Redis # Add/Update in Redis
server.sadd(redis_all_provider, paste_provider) server_pasteName.sadd(paste_baseName, path)
server_trend.sadd(redis_all_provider, paste_provider)
num_paste = int(server.hincrby(paste_provider+'_num', paste_date, 1)) num_paste = int(server_trend.hincrby(paste_provider+'_num', paste_date, 1))
sum_size = float(server.hincrbyfloat(paste_provider+'_size', paste_date, paste_size)) sum_size = float(server_trend.hincrbyfloat(paste_provider+'_size', paste_date, paste_size))
new_avg = float(sum_size) / float(num_paste) new_avg = float(sum_size) / float(num_paste)
server.hset(paste_provider +'_avg', paste_date, new_avg) server_trend.hset(paste_provider +'_avg', paste_date, new_avg)
# #
@ -84,33 +86,33 @@ def compute_provider_info(server, path):
# #
# Size # Size
if server.zcard(redis_sum_size_set) < max_set_cardinality or server.zscore(redis_sum_size_set, paste_provider) != "nil": if server_trend.zcard(redis_sum_size_set) < max_set_cardinality or server_trend.zscore(redis_sum_size_set, paste_provider) != "nil":
server.zadd(redis_sum_size_set, float(num_paste), paste_provider) server_trend.zadd(redis_sum_size_set, float(num_paste), paste_provider)
server.zadd(redis_avg_size_name_set, float(new_avg), paste_provider) server_trend.zadd(redis_avg_size_name_set, float(new_avg), paste_provider)
else: #set full capacity else: #set full capacity
member_set = server.zrangebyscore(redis_sum_size_set, '-inf', '+inf', withscores=True, start=0, num=1) member_set = server_trend.zrangebyscore(redis_sum_size_set, '-inf', '+inf', withscores=True, start=0, num=1)
# Member set is a list of (value, score) pairs # Member set is a list of (value, score) pairs
if float(member_set[0][1]) < new_avg: if float(member_set[0][1]) < new_avg:
#remove min from set and add the new one #remove min from set and add the new one
print 'Size - adding ' +paste_provider+ '(' +str(new_avg)+') in set and removing '+member_set[0][0]+'('+str(member_set[0][1])+')' print 'Size - adding ' +paste_provider+ '(' +str(new_avg)+') in set and removing '+member_set[0][0]+'('+str(member_set[0][1])+')'
server.zrem(redis_sum_size_set, member_set[0][0]) server_trend.zrem(redis_sum_size_set, member_set[0][0])
server.zadd(redis_sum_size_set, float(sum_size), paste_provider) server_trend.zadd(redis_sum_size_set, float(sum_size), paste_provider)
server.zrem(redis_avg_size_name_set, member_set[0][0]) server_trend.zrem(redis_avg_size_name_set, member_set[0][0])
server.zadd(redis_avg_size_name_set, float(new_avg), paste_provider) server_trend.zadd(redis_avg_size_name_set, float(new_avg), paste_provider)
# Num # Num
# if set not full or provider already present # if set not full or provider already present
if server.zcard(redis_providers_name_set) < max_set_cardinality or server.zscore(redis_providers_name_set, paste_provider) != "nil": if server_trend.zcard(redis_providers_name_set) < max_set_cardinality or server_trend.zscore(redis_providers_name_set, paste_provider) != "nil":
server.zadd(redis_providers_name_set, float(num_paste), paste_provider) server_trend.zadd(redis_providers_name_set, float(num_paste), paste_provider)
else: #set at full capacity else: #set at full capacity
member_set = server.zrangebyscore(redis_providers_name_set, '-inf', '+inf', withscores=True, start=0, num=1) member_set = server_trend.zrangebyscore(redis_providers_name_set, '-inf', '+inf', withscores=True, start=0, num=1)
# Member set is a list of (value, score) pairs # Member set is a list of (value, score) pairs
if int(member_set[0][1]) < num_paste: if int(member_set[0][1]) < num_paste:
#remove min from set and add the new one #remove min from set and add the new one
print 'Num - adding ' +paste_provider+ '(' +str(num_paste)+') in set and removing '+member_set[0][0]+'('+str(member_set[0][1])+')' print 'Num - adding ' +paste_provider+ '(' +str(num_paste)+') in set and removing '+member_set[0][0]+'('+str(member_set[0][1])+')'
server.zrem(member_set[0][0]) server_trend.zrem(member_set[0][0])
server.zadd(redis_providers_name_set, float(num_paste), paste_provider) server_trend.zadd(redis_providers_name_set, float(num_paste), paste_provider)
if __name__ == '__main__': if __name__ == '__main__':
@ -135,6 +137,11 @@ if __name__ == '__main__':
port=p.config.get("Redis_Level_DB_Trending", "port"), port=p.config.get("Redis_Level_DB_Trending", "port"),
db=p.config.get("Redis_Level_DB_Trending", "db")) db=p.config.get("Redis_Level_DB_Trending", "db"))
r_serv_pasteName = redis.StrictRedis(
host=p.config.get("Redis_Paste_Name", "host"),
port=p.config.get("Redis_Paste_Name", "port"),
db=p.config.get("Redis_Paste_Name", "db"))
# Endless loop getting messages from the input queue # Endless loop getting messages from the input queue
while True: while True:
# Get one message from the input queue # Get one message from the input queue
@ -151,4 +158,4 @@ if __name__ == '__main__':
if len(message.split(';')) > 1: if len(message.split(';')) > 1:
compute_most_posted(r_serv_trend, message) compute_most_posted(r_serv_trend, message)
else: else:
compute_provider_info(r_serv_trend, message) compute_provider_info(r_serv_trend, r_serv_pasteName, message)

30
var/www/Flask_server.py
View File

@ -61,6 +61,11 @@ r_serv_term = redis.StrictRedis(
port=cfg.getint("Redis_Level_DB_TermFreq", "port"), port=cfg.getint("Redis_Level_DB_TermFreq", "port"),
db=cfg.getint("Redis_Level_DB_TermFreq", "db")) db=cfg.getint("Redis_Level_DB_TermFreq", "db"))
r_serv_pasteName = redis.StrictRedis(
host=cfg.get("Redis_Paste_Name", "host"),
port=cfg.getint("Redis_Paste_Name", "port"),
db=cfg.getint("Redis_Paste_Name", "db"))
app = Flask(__name__, static_url_path='/static/') app = Flask(__name__, static_url_path='/static/')
@ -348,19 +353,20 @@ def search():
c = [] #preview of the paste content c = [] #preview of the paste content
paste_date = [] paste_date = []
paste_size = [] paste_size = []
# Search filename # Search filename
from os import walk print r_serv_pasteName.smembers(q[0])
for (dirpath, dirnames, filenames) in walk(os.path.join(os.environ['AIL_HOME'], 'PASTES/')): for path in r_serv_pasteName.smembers(q[0]):
if q[0] in filenames: print path
r.append(dirpath+'/'+q[0]) r.append(path)
paste = Paste.Paste(dirpath+'/'+q[0]) paste = Paste.Paste(path)
content = paste.get_p_content().decode('utf8', 'ignore') content = paste.get_p_content().decode('utf8', 'ignore')
content_range = max_preview_char if len(content)>max_preview_char else len(content)-1 content_range = max_preview_char if len(content)>max_preview_char else len(content)-1
c.append(content[0:content_range]) c.append(content[0:content_range])
curr_date = str(paste._get_p_date()) curr_date = str(paste._get_p_date())
curr_date = curr_date[0:4]+'/'+curr_date[4:6]+'/'+curr_date[6:] curr_date = curr_date[0:4]+'/'+curr_date[4:6]+'/'+curr_date[6:]
paste_date.append(curr_date) paste_date.append(curr_date)
paste_size.append(paste._get_p_size()) paste_size.append(paste._get_p_size())
# Search full line # Search full line
from whoosh import index from whoosh import index