CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

chg: [Tags] add backend, daterange selector

pull/342/head
Terrtia 2019-03-20 13:25:02 +01:00
parent 65e6eae00d
commit a9e86231ad
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
5 changed files with 128 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
bin/Tags.py
View File

@ -17,8 +17,8 @@ from pubsublogger import publisher
from Helper import Process
from packages import Paste
def get_paste_date(paste_filename):
l_directory = paste_filename.split('/')
def get_item_date(item_filename):
l_directory = item_filename.split('/')
return '{}{}{}'.format(l_directory[-4], l_directory[-3], l_directory[-2])
def set_tag_metadata(tag, date):
@ -75,14 +75,15 @@ if __name__ == '__main__':
if res == 1:
print("new tags added : {}".format(tag))
# add the path to the tag set
curr_date = datetime.date.today().strftime("%Y%m%d")
res = server.sadd('{}:{}'.format(tag, curr_date), path)
#curr_date = datetime.date.today().strftime("%Y%m%d")
item_date = get_item_date(path)
res = server.sadd('{}:{}'.format(tag, item_date), path)
if res == 1:
print("new paste: {}".format(path))
print(" tagged: {}".format(tag))
set_tag_metadata(tag, curr_date)
set_tag_metadata(tag, item_date)
server_metadata.sadd('tag:{}'.format(path), tag)
curr_date = datetime.date.today().strftime("%Y%m%d")
server.hincrby('daily_tags:{}'.format(curr_date), tag, 1)
server.hincrby('daily_tags:{}'.format(item_date), tag, 1)
p.populate_set_out(message, 'MISP_The_Hive_feeder')

2
var/www/modules/Flask_config.py
View File

@ -154,6 +154,8 @@ if baseUrl != '':
max_preview_char = int(cfg.get("Flask", "max_preview_char")) # Maximum number of character to display in the tooltip
max_preview_modal = int(cfg.get("Flask", "max_preview_modal")) # Maximum number of character to display in the modal
max_tags_result = 50
DiffMaxLineLength = int(cfg.get("Flask", "DiffMaxLineLength"))#Use to display the estimated percentage instead of a raw value
bootstrap_label = ['primary', 'success', 'danger', 'warning', 'info']

85
var/www/modules/Tags/Flask_Tags.py
View File

@ -28,6 +28,7 @@ r_serv_statistics = Flask_config.r_serv_statistics
max_preview_char = Flask_config.max_preview_char
max_preview_modal = Flask_config.max_preview_modal
bootstrap_label = Flask_config.bootstrap_label
max_tags_result = Flask_config.max_tags_result
PASTES_FOLDER = Flask_config.PASTES_FOLDER
Tags = Blueprint('Tags', __name__, template_folder='templates')
@ -68,20 +69,61 @@ def get_tags_with_synonyms(tag):
else:
return {'name':tag,'id':tag}
def get_item_date(item_filename):
l_directory = item_filename.split('/')
return '{}{}{}'.format(l_directory[-4], l_directory[-3], l_directory[-2])
def substract_date(date_from, date_to):
date_from = datetime.date(int(date_from[0:4]), int(date_from[4:6]), int(date_from[6:8]))
date_to = datetime.date(int(date_to[0:4]), int(date_to[4:6]), int(date_to[6:8]))
delta = date_to - date_from # timedelta
l_date = []
for i in range(delta.days + 1):
date = date_from + datetime.timedelta(i)
l_date.append( date.strftime('%Y%m%d') )
return l_date
def get_all_dates_range(date_from, date_to):
all_dates = {}
date_range = []
if date_from is not None and date_to is not None:
#change format
try:
if len(date_from) != 8:
date_from = date_from[0:4] + date_from[5:7] + date_from[8:10]
date_to = date_to[0:4] + date_to[5:7] + date_to[8:10]
date_range = substract_date(date_from, date_to)
except:
pass
if not date_range:
date_range.append(datetime.date.today().strftime("%Y%m%d"))
date_from = date_range[0][0:4] + '-' + date_range[0][4:6] + '-' + date_range[0][6:8]
date_to = date_from
else:
date_from = date_from[0:4] + '-' + date_from[4:6] + '-' + date_from[6:8]
date_to = date_to[0:4] + '-' + date_to[4:6] + '-' + date_to[6:8]
all_dates['date_from'] = date_from
all_dates['date_to'] = date_to
all_dates['date_range'] = date_range
return all_dates
# ============= ROUTES ==============
@Tags.route("/tags/", methods=['GET'])
def Tags_page():
current_date = datetime.date.today().strftime("%Y-%m-%d")
date_from = request.args.get('date_from')
date_to = request.args.get('date_to')
dates = get_all_dates_range(date_from, date_to)
tags = request.args.get('ltags')
if tags is None:
return render_template("Tags.html", date_from=current_date, date_to=current_date)
else:
tags = request.args.get('ltags')
list_tags = tags.split(',')
@ -97,11 +139,23 @@ def Tags_page():
print('empty')
# 1 tag
elif len(list_tags) < 2:
tagged_pastes = r_serv_tags.smembers(list_tags[0])
tagged_pastes = []
for date in dates['date_range']:
tagged_pastes.extend(r_serv_tags.smembers('{}:{}'.format(list_tags[0], date)))
# 2 tags or more
else:
tagged_pastes = r_serv_tags.sinter(list_tags[0], *list_tags[1:])
tagged_pastes = []
for date in dates['date_range']:
tag_keys = []
for tag in list_tags:
tag_keys.append('{}:{}'.format(tag, date))
if len(tag_keys) > 1:
daily_items = r_serv_tags.sinter(tag_keys[0], *tag_keys[1:])
else:
daily_items = r_serv_tags.sinter(tag_keys[0])
tagged_pastes.extend(daily_items)
else :
return 'INCORRECT INPUT'
@ -113,7 +167,21 @@ def Tags_page():
allPastes = list(tagged_pastes)
paste_tags = []
for path in allPastes[0:50]: ######################moduleName
try:
page = int(request.args.get('page'))
except:
page = 1
if page <= 0:
page = 1
nb_page_max = len(tagged_pastes)/(max_tags_result)
if not nb_page_max.is_integer():
nb_page_max = int(nb_page_max)+1
if page > nb_page_max:
page = nb_page_max
start = max_tags_result*(page -1)
stop = max_tags_result*page
for path in allPastes[start:stop]: ######################moduleName
all_path.append(path)
paste = Paste.Paste(path)
content = paste.get_p_content()
@ -153,8 +221,9 @@ def Tags_page():
all_path=all_path,
tags=tags,
list_tag = list_tag,
date_from=current_date,
date_to=current_date,
date_from=dates['date_from'],
date_to=dates['date_to'],
page=page, nb_page_max=nb_page_max,
paste_tags=paste_tags,
bootstrap_label=bootstrap_label,
content=all_content,

42
var/www/modules/Tags/templates/Tags.html
View File

@ -105,7 +105,7 @@
<div>
{%if all_path%}
<table class="table table-bordered table-hover " id="myTable_">
<table class="table table-bordered table-hover" id="myTable_">
<thead class="thead-dark">
<tr>
<th>Date</th>
@ -142,6 +142,42 @@
</tbody>
</table>
<div class="d-flex justify-content-center border-top mt-2">
<nav class="mt-4" aria-label="...">
<ul class="pagination">
<li class="page-item {%if page==1%}disabled{%endif%}">
<a class="page-link" href="{{ url_for('Tags.Tags_page') }}?page={{page-1}}&date_from={{date_from}}&date_to={{date_to}}&ltags={{tags}}">Previous</a>
</li>
{%if page>3%}
<li class="page-item"><a class="page-link" href="{{ url_for('Tags.Tags_page') }}?page=1&date_from={{date_from}}&date_to={{date_to}}&ltags={{tags}}">1</a></li>
<li class="page-item disabled"><a class="page-link" aria-disabled="true" href="#">...</a></li>
<li class="page-item"><a class="page-link" href="{{ url_for('Tags.Tags_page') }}?page={{page-1}}&date_from={{date_from}}&date_to={{date_to}}&ltags={{tags}}">{{page-1}}</a></li>
<li class="page-item active"><a class="page-link" href="{{ url_for('Tags.Tags_page') }}?page={{page}}&date_from={{date_from}}&date_to={{date_to}}&ltags={{tags}}">{{page}}</a></li>
{%else%}
{%if page>2%}<li class="page-item"><a class="page-link" href="{{ url_for('Tags.Tags_page') }}?page={{page-2}}&date_from={{date_from}}&date_to={{date_to}}&ltags={{tags}}">{{page-2}}</a></li>{%endif%}
{%if page>1%}<li class="page-item"><a class="page-link" href="{{ url_for('Tags.Tags_page') }}?page={{page-1}}&date_from={{date_from}}&date_to={{date_to}}&ltags={{tags}}">{{page-1}}</a></li>{%endif%}
<li class="page-item active"><a class="page-link" href="{{ url_for('Tags.Tags_page') }}?page={{page}}&date_from={{date_from}}&date_to={{date_to}}&ltags={{tags}}">{{page}}</a></li>
{%endif%}
{%if nb_page_max-page>3%}
<li class="page-item"><a class="page-link" href="{{ url_for('Tags.Tags_page') }}?page={{page+1}}&date_from={{date_from}}&date_to={{date_to}}&ltags={{tags}}">{{page+1}}</a></li>
<li class="page-item disabled"><a class="page-link" aria-disabled="true" href="#">...</a></li>
<li class="page-item"><a class="page-link" href="{{ url_for('Tags.Tags_page') }}?page={{nb_page_max}}&date_from={{date_from}}&date_to={{date_to}}&ltags={{tags}}">{{nb_page_max}}</a></li>
{%else%}
{%if nb_page_max-page>2%}<li class="page-item"><a class="page-link" href="{{ url_for('Tags.Tags_page') }}?page={{nb_page_max-2}}&date_from={{date_from}}&date_to={{date_to}}&ltags={{tags}}">{{nb_page_max-2}}</a></li>{%endif%}
{%if nb_page_max-page>1%}<li class="page-item"><a class="page-link" href="{{ url_for('Tags.Tags_page') }}?page={{nb_page_max-1}}&date_from={{date_from}}&date_to={{date_to}}&ltags={{tags}}">{{nb_page_max-1}}</a></li>{%endif%}
{%if nb_page_max-page>0%}<li class="page-item"><a class="page-link" href="{{ url_for('Tags.Tags_page') }}?page={{nb_page_max}}&date_from={{date_from}}&date_to={{date_to}}&ltags={{tags}}">{{nb_page_max}}</a></li>{%endif%}
{%endif%}
<li class="page-item {%if page==nb_page_max%}disabled{%endif%}">
<a class="page-link" href="{{ url_for('Tags.Tags_page') }}?page={{page+1}}&date_from={{date_from}}&date_to={{date_to}}&ltags={{tags}}" aria-disabled="true">Next</a>
</li>
</ul>
</nav>
</div>
{%endif%}
</div>
@ -247,7 +283,9 @@ $(document).ready(function(){
<script>
function searchTags() {
var data = ltags.getValue();
window.location.replace("{{ url_for('Tags.Tags_page') }}?ltags=" + data);
var date_from = $('#date-range-from-input').val();
var date_to =$('#date-range-to-input').val();
window.location.replace("{{ url_for('Tags.Tags_page') }}?date_from="+date_from+"&date_to="+date_to+"&ltags=" + data);
}
function emptyTags() {
ltags.clear();

2
var/www/modules/showpaste/templates/show_saved_item_min.html
View File

@ -241,6 +241,7 @@
});
</script>
{% if crawler_metadata['get_metadata'] %}
<script>
var ctx = canvas.getContext('2d'), img = new Image();
@ -287,6 +288,7 @@
blocks.addEventListener('change', pixelate, false);
</script>
{% endif %}
<div id="container-show-more" class="text-center">