CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

chg: [tags] remove/add tags by daterange + improve item loading time

pull/342/head
Terrtia 2019-03-20 15:57:55 +01:00
parent 8b5266706e
commit ce6475e96e
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
2 changed files with 103 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

130
var/www/modules/Tags/Flask_Tags.py
View File

@ -9,7 +9,6 @@ from flask import Flask, render_template, jsonify, request, Blueprint, redirect,
import json
import datetime
import ssdeep
import Paste
@ -57,6 +56,16 @@ for name, tags in clusters.items(): #galaxie name + tags
def one():
return 1
def date_substract_day(date, num_day=1):
new_date = datetime.date(int(date[0:4]), int(date[4:6]), int(date[6:8])) - datetime.timedelta(num_day)
new_date = str(new_date).replace('-', '')
return new_date
def date_add_day(date, num_day=1):
new_date = datetime.date(int(date[0:4]), int(date[4:6]), int(date[6:8])) + datetime.timedelta(num_day)
new_date = str(new_date).replace('-', '')
return new_date
def get_tags_with_synonyms(tag):
str_synonyms = ' - synonyms: '
synonyms = r_serv_tags.smembers('synonym_tag_' + tag)
@ -109,6 +118,92 @@ def get_all_dates_range(date_from, date_to):
all_dates['date_range'] = date_range
return all_dates
def add_item_tag(tag, item_path):
item_date = int(get_item_date(item_path))
#add tag
r_serv_metadata.sadd('tag:{}'.format(item_path), tag)
r_serv_tags.sadd('{}:{}'.format(tag, item_date), item_path)
r_serv_tags.hincrby('daily_tags:{}'.format(item_date), tag, 1)
tag_first_seen = r_serv_tags.hget('tag_metadata:{}'.format(tag), 'last_seen')
if tag_first_seen is None:
tag_first_seen = 99999999
else:
tag_first_seen = int(tag_first_seen)
tag_last_seen = r_serv_tags.hget('tag_metadata:{}'.format(tag), 'last_seen')
if tag_last_seen is None:
tag_last_seen = 0
else:
tag_last_seen = int(tag_last_seen)
#add new tag in list of all used tags
r_serv_tags.sadd('list_tags', tag)
# update fisrt_seen/last_seen
if item_date < tag_first_seen:
r_serv_tags.hset('tag_metadata:{}'.format(tag), 'first_seen', item_date)
# update metadata last_seen
if item_date > tag_last_seen:
r_serv_tags.hset('tag_metadata:{}'.format(tag), 'last_seen', item_date)
def remove_item_tag(tag, item_path):
item_date = int(get_item_date(item_path))
#remove tag
r_serv_metadata.srem('tag:{}'.format(item_path), tag)
res = r_serv_tags.srem('{}:{}'.format(tag, item_date), item_path)
if res ==1:
# no tag for this day
if int(r_serv_tags.hget('daily_tags:{}'.format(item_date), tag)) == 1:
r_serv_tags.hdel('daily_tags:{}'.format(item_date), tag)
else:
r_serv_tags.hincrby('daily_tags:{}'.format(item_date), tag, -1)
tag_first_seen = int(r_serv_tags.hget('tag_metadata:{}'.format(tag), 'last_seen'))
tag_last_seen = int(r_serv_tags.hget('tag_metadata:{}'.format(tag), 'last_seen'))
# update fisrt_seen/last_seen
if item_date == tag_first_seen:
update_tag_first_seen(tag, tag_first_seen, tag_last_seen)
if item_date == tag_last_seen:
update_tag_last_seen(tag, tag_first_seen, tag_last_seen)
else:
return 'Error incorrect tag'
def update_tag_first_seen(tag, tag_first_seen, tag_last_seen):
if tag_first_seen == tag_last_seen:
if r_serv_tags.scard('{}:{}'.format(tag, tag_first_seen)) > 0:
r_serv_tags.hset('tag_metadata:{}'.format(tag), 'first_seen', tag_first_seen)
# no tag in db
else:
r_serv_tags.srem('list_tags', tag)
r_serv_tags.hdel('tag_metadata:{}'.format(tag), 'first_seen')
r_serv_tags.hdel('tag_metadata:{}'.format(tag), 'last_seen')
else:
if r_serv_tags.scard('{}:{}'.format(tag, tag_first_seen)) > 0:
r_serv_tags.hset('tag_metadata:{}'.format(tag), 'first_seen', tag_first_seen)
else:
tag_first_seen = date_add_day(tag_first_seen)
update_tag_first_seen(tag, tag_first_seen, tag_last_seen)
def update_tag_last_seen(tag, tag_first_seen, tag_last_seen):
if tag_first_seen == tag_last_seen:
if r_serv_tags.scard('{}:{}'.format(tag, tag_last_seen)) > 0:
r_serv_tags.hset('tag_metadata:{}'.format(tag), 'last_seen', tag_last_seen)
# no tag in db
else:
r_serv_tags.srem('list_tags', tag)
r_serv_tags.hdel('tag_metadata:{}'.format(tag), 'first_seen')
r_serv_tags.hdel('tag_metadata:{}'.format(tag), 'last_seen')
else:
if r_serv_tags.scard('{}:{}'.format(tag, tag_last_seen)) > 0:
r_serv_tags.hset('tag_metadata:{}'.format(tag), 'last_seen', tag_last_seen)
else:
tag_last_seen = date_substract_day(tag_last_seen)
update_tag_last_seen(tag, tag_first_seen, tag_last_seen)
# ============= ROUTES ==============
@ -121,7 +216,7 @@ def Tags_page():
tags = request.args.get('ltags')
if tags is None:
return render_template("Tags.html", date_from=current_date, date_to=current_date)
return render_template("Tags.html", date_from=dates['date_from'], date_to=dates['date_to'])
else:
tags = request.args.get('ltags')
@ -340,12 +435,7 @@ def remove_tag():
path = request.args.get('paste')
tag = request.args.get('tag')
#remove tag
r_serv_metadata.srem('tag:'+path, tag)
r_serv_tags.srem(tag, path)
if r_serv_tags.scard(tag) == 0:
r_serv_tags.srem('list_tags', tag)
remove_item_tag(tag, path)
return redirect(url_for('showsavedpastes.showsavedpaste', paste=path))
@ -357,17 +447,11 @@ def confirm_tag():
tag = request.args.get('tag')
if(tag[9:28] == 'automatic-detection'):
#remove automatic tag
r_serv_metadata.srem('tag:'+path, tag)
r_serv_tags.srem(tag, path)
remove_item_tag(tag, path)
tag = tag.replace('automatic-detection','analyst-detection', 1)
#add analyst tag
r_serv_metadata.sadd('tag:'+path, tag)
r_serv_tags.sadd(tag, path)
#add new tag in list of all used tags
r_serv_tags.sadd('list_tags', tag)
add_item_tag(tag, path)
return redirect(url_for('showsavedpastes.showsavedpaste', paste=path))
@ -417,12 +501,7 @@ def addTags():
tax = tag.split(':')[0]
if tax in active_taxonomies:
if tag in r_serv_tags.smembers('active_tag_' + tax):
#add tag
r_serv_metadata.sadd('tag:'+path, tag)
r_serv_tags.sadd(tag, path)
#add new tag in list of all used tags
r_serv_tags.sadd('list_tags', tag)
add_item_tag(tag, path)
else:
return 'INCORRECT INPUT1'
@ -437,12 +516,7 @@ def addTags():
if gal in active_galaxies:
if tag in r_serv_tags.smembers('active_tag_galaxies_' + gal):
#add tag
r_serv_metadata.sadd('tag:'+path, tag)
r_serv_tags.sadd(tag, path)
#add new tag in list of all used tags
r_serv_tags.sadd('list_tags', tag)
add_item_tag(tag, path)
else:
return 'INCORRECT INPUT3'

2
var/www/modules/showpaste/Flask_showpaste.py
View File

@ -58,7 +58,7 @@ def showpaste(content_range, requested_path):
p_date = p_date[6:]+'/'+p_date[4:6]+'/'+p_date[0:4]
p_source = paste.p_source
p_encoding = paste._get_p_encoding()
p_language = paste._get_p_language()
p_language = 'None'
p_size = paste.p_size
p_mime = paste.p_mime
p_lineinfo = paste.get_lines_info()