CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

chg: [chats] add pagination

dev
terrtia 2023-12-11 00:46:15 +01:00
parent 759bb9a2f0
commit 5fc9b1403f
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
9 changed files with 256 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
bin/lib/chats_viewer.py
View File

@ -331,7 +331,7 @@ def api_get_chat_service_instance(chat_instance_uuid):
return {"status": "error", "reason": "Unknown uuid"}, 404
return chat_instance.get_meta({'chats'}), 200
def api_get_chat(chat_id, chat_instance_uuid, translation_target=None):
def api_get_chat(chat_id, chat_instance_uuid, translation_target=None, nb=-1, page=-1):
chat = Chats.Chat(chat_id, chat_instance_uuid)
if not chat.exists():
return {"status": "error", "reason": "Unknown chat"}, 404
@ -341,7 +341,7 @@ def api_get_chat(chat_id, chat_instance_uuid, translation_target=None):
if meta['subchannels']:
meta['subchannels'] = get_subchannels_meta_from_global_id(meta['subchannels'])
else:
meta['messages'], meta['tags_messages'] = chat.get_messages(translation_target=translation_target)
meta['messages'], meta['pagination'], meta['tags_messages'] = chat.get_messages(translation_target=translation_target, nb=nb, page=page)
return meta, 200
def api_get_nb_message_by_week(chat_id, chat_instance_uuid):
@ -367,7 +367,7 @@ def api_get_chat_participants(chat_type, chat_subtype, chat_id):
meta['participants'] = chat_participants
return meta, 200
def api_get_subchannel(chat_id, chat_instance_uuid, translation_target=None):
def api_get_subchannel(chat_id, chat_instance_uuid, translation_target=None, nb=-1, page=-1):
subchannel = ChatSubChannels.ChatSubChannel(chat_id, chat_instance_uuid)
if not subchannel.exists():
return {"status": "error", "reason": "Unknown subchannel"}, 404
@ -378,17 +378,17 @@ def api_get_subchannel(chat_id, chat_instance_uuid, translation_target=None):
meta['threads'] = get_threads_metas(meta['threads'])
if meta.get('username'):
meta['username'] = get_username_meta_from_global_id(meta['username'])
meta['messages'], meta['tags_messages'] = subchannel.get_messages(translation_target=translation_target)
meta['messages'], meta['pagination'], meta['tags_messages'] = subchannel.get_messages(translation_target=translation_target, nb=nb, page=page)
return meta, 200
def api_get_thread(thread_id, thread_instance_uuid, translation_target=None):
def api_get_thread(thread_id, thread_instance_uuid, translation_target=None, nb=-1, page=-1):
thread = ChatThreads.ChatThread(thread_id, thread_instance_uuid)
if not thread.exists():
return {"status": "error", "reason": "Unknown thread"}, 404
meta = thread.get_meta({'chat', 'nb_messages', 'nb_participants'})
# if meta['chat']:
# meta['chat'] = get_chat_meta_from_global_id(meta['chat'])
meta['messages'], meta['tags_messages'] = thread.get_messages(translation_target=translation_target)
meta['messages'], meta['pagination'], meta['tags_messages'] = thread.get_messages(translation_target=translation_target, nb=nb, page=page)
return meta, 200
def api_get_message(message_id):

49
bin/lib/objects/abstract_chat_object.py
View File

@ -130,18 +130,36 @@ class AbstractChatObject(AbstractSubtypeObject, ABC):
def get_nb_messages(self):
return r_object.zcard(f'messages:{self.type}:{self.subtype}:{self.id}')
def _get_messages(self, nb=-1, page=1):
def _get_messages(self, nb=-1, page=-1):
if nb < 1:
return r_object.zrange(f'messages:{self.type}:{self.subtype}:{self.id}', 0, -1, withscores=True)
messages = r_object.zrange(f'messages:{self.type}:{self.subtype}:{self.id}', 0, -1, withscores=True)
nb_pages = 0
page = 1
total = len(messages)
nb_first = 1
nb_last = total
else:
total = r_object.zcard(f'messages:{self.type}:{self.subtype}:{self.id}')
nb_pages = total / nb
if not nb_pages.is_integer():
nb_pages = int(nb_pages) + 1
else:
nb_pages = int(nb_pages)
if page > nb_pages or page < 1:
page = nb_pages
if page > 1:
start = page - 1 + nb
start = (page - 1) * nb
else:
start = 0
messages = r_object.zrevrange(f'messages:{self.type}:{self.subtype}:{self.id}', start, start+nb-1, withscores=True)
if messages:
messages = reversed(messages)
return messages
messages = r_object.zrange(f'messages:{self.type}:{self.subtype}:{self.id}', start, start+nb-1, withscores=True)
# if messages:
# messages = reversed(messages)
nb_first = start+1
nb_last = start+nb
if nb_last > total:
nb_last = total
return messages, {'nb': nb, 'page': page, 'nb_pages': nb_pages, 'total': total, 'nb_first': nb_first, 'nb_last': nb_last}
def get_timestamp_first_message(self):
return r_object.zrange(f'messages:{self.type}:{self.subtype}:{self.id}', 0, 0, withscores=True)
@ -184,12 +202,23 @@ class AbstractChatObject(AbstractSubtypeObject, ABC):
meta = message.get_meta(options={'content', 'files-names', 'images', 'link', 'parent', 'parent_meta', 'reactions', 'thread', 'translation', 'user-account'}, timestamp=timestamp, translation_target=translation_target)
return meta
def get_messages(self, start=0, page=1, nb=500, unread=False, translation_target='en'): # threads ???? # TODO ADD last/first message timestamp + return page
def get_messages(self, start=0, page=-1, nb=500, unread=False, translation_target='en'): # threads ???? # TODO ADD last/first message timestamp + return page
# TODO return message meta
tags = {}
messages = {}
curr_date = None
for message in self._get_messages(nb=2000, page=1):
try:
nb = int(nb)
except TypeError:
nb = 500
if not page:
page = -1
try:
page = int(page)
except TypeError:
page = 1
mess, pagination = self._get_messages(nb=nb, page=page)
for message in mess:
timestamp = message[1]
date_day = datetime.fromtimestamp(timestamp).strftime('%Y/%m/%d')
if date_day != curr_date:
@ -203,7 +232,7 @@ class AbstractChatObject(AbstractSubtypeObject, ABC):
if tag not in tags:
tags[tag] = 0
tags[tag] += 1
return messages, tags
return messages, pagination, tags
# TODO REWRITE ADD OR ADD MESSAGE ????
# add

12
var/www/blueprints/chats_explorer.py
View File

@ -82,7 +82,9 @@ def chats_explorer_chat():
chat_id = request.args.get('id')
instance_uuid = request.args.get('uuid')
target = request.args.get('target')
chat = chats_viewer.api_get_chat(chat_id, instance_uuid, translation_target=target)
nb_messages = request.args.get('nb')
page = request.args.get('page')
chat = chats_viewer.api_get_chat(chat_id, instance_uuid, translation_target=target, nb=nb_messages, page=page)
if chat[1] != 200:
return create_json_response(chat[0], chat[1])
else:
@ -109,7 +111,9 @@ def objects_subchannel_messages():
subchannel_id = request.args.get('id')
instance_uuid = request.args.get('uuid')
target = request.args.get('target')
subchannel = chats_viewer.api_get_subchannel(subchannel_id, instance_uuid, translation_target=target)
nb_messages = request.args.get('nb')
page = request.args.get('page')
subchannel = chats_viewer.api_get_subchannel(subchannel_id, instance_uuid, translation_target=target, nb=nb_messages, page=page)
if subchannel[1] != 200:
return create_json_response(subchannel[0], subchannel[1])
else:
@ -124,7 +128,9 @@ def objects_thread_messages():
thread_id = request.args.get('id')
instance_uuid = request.args.get('uuid')
target = request.args.get('target')
thread = chats_viewer.api_get_thread(thread_id, instance_uuid, translation_target=target)
nb_messages = request.args.get('nb')
page = request.args.get('page')
thread = chats_viewer.api_get_thread(thread_id, instance_uuid, translation_target=target, nb=nb_messages, page=page)
if thread[1] != 200:
return create_json_response(thread[0], thread[1])
else:

32
var/www/templates/chats_explorer/SubChannelMessages.html
View File

@ -126,12 +126,12 @@
</li>
</ul>
{% with obj_type='chat', obj_id=subchannel['id'], obj_subtype=subchannel['subtype'] %}
{% include 'modals/investigations_register_obj.html' %}
{% endwith %}
<button type="button" class="btn btn-primary" data-toggle="modal" data-target="#investigations_register_obj_modal">
<i class="fas fa-microscope"></i> Investigations
</button>
{# {% with obj_type='chat', obj_id=subchannel['id'], obj_subtype=subchannel['subtype'] %}#}
{# {% include 'modals/investigations_register_obj.html' %}#}
{# {% endwith %}#}
{# <button type="button" class="btn btn-primary" data-toggle="modal" data-target="#investigations_register_obj_modal">#}
{# <i class="fas fa-microscope"></i> Investigations#}
{# </button>#}
</div>
</div>
@ -191,6 +191,18 @@
{% with translate_url=url_for('chats_explorer.objects_subchannel_messages', uuid=subchannel['subtype']), obj_id=subchannel['id'] %}
{% include 'chats_explorer/block_translation.html' %}
{% endwith %}
{% with obj_subtype=subchannel['subtype'], obj_id=subchannel['id'], url_endpoint=url_for("chats_explorer.objects_subchannel_messages"), nb=subchannel['pagination']['nb'] %}
{% set date_from=subchannel['first_seen'] %}
{% set date_to=subchannel['last_seen'] %}
{% include 'block_obj_time_search.html' %}
{% endwith %}
{% with endpoint_url=url_for('chats_explorer.objects_subchannel_messages', uuid=subchannel['subtype']), pagination=subchannel['pagination'] %}
{% set endpoint_url = endpoint_url + "&id=" + subchannel['id'] + "&nb=" + subchannel['pagination']['nb'] | string %}
{% if translation_target %}
{% set endpoint_url = endpoint_url + "&target=" + translation_target %}
{% endif %}
{% include 'chats_explorer/pagination.html' %}
{% endwith %}
<div class="position-relative">
<div class="chat-messages p-2">
@ -216,6 +228,14 @@
</div>
</div>
{% with endpoint_url=url_for('chats_explorer.objects_subchannel_messages', uuid=subchannel['subtype']), pagination=subchannel['pagination'] %}
{% set endpoint_url = endpoint_url + "&id=" + subchannel['id'] + "&nb=" + subchannel['pagination']['nb'] | string %}
{% if translation_target %}
{% set endpoint_url = endpoint_url + "&target=" + translation_target %}
{% endif %}
{% include 'chats_explorer/pagination.html' %}
{% endwith %}
</div>
</div>

20
var/www/templates/chats_explorer/ThreadMessages.html
View File

@ -141,6 +141,18 @@
{% with translate_url=url_for('chats_explorer.objects_thread_messages', uuid=meta['subtype']), obj_id=meta['id'] %}
{% include 'chats_explorer/block_translation.html' %}
{% endwith %}
{% with obj_subtype=meta['subtype'], obj_id=meta['id'], url_endpoint=url_for("chats_explorer.objects_thread_messages"), nb=meta['pagination']['nb'] %}
{% set date_from=meta['first_seen'] %}
{% set date_to=meta['last_seen'] %}
{% include 'block_obj_time_search.html' %}
{% endwith %}
{% with endpoint_url=url_for('chats_explorer.objects_thread_messages', uuid=meta['subtype']), pagination=meta['pagination'] %}
{% set endpoint_url = endpoint_url + "&id=" + meta['id'] + "&nb=" + meta['pagination']['nb'] | string %}
{% if translation_target %}
{% set endpoint_url = endpoint_url + "&target=" + translation_target %}
{% endif %}
{% include 'chats_explorer/pagination.html' %}
{% endwith %}
<div class="position-relative">
<div class="chat-messages p-2">
@ -166,6 +178,14 @@
</div>
</div>
{% with endpoint_url=url_for('chats_explorer.objects_thread_messages', uuid=meta['subtype']), pagination=meta['pagination'] %}
{% set endpoint_url = endpoint_url + "&id=" + meta['id'] + "&nb=" + meta['pagination']['nb'] | string %}
{% if translation_target %}
{% set endpoint_url = endpoint_url + "&target=" + translation_target %}
{% endif %}
{% include 'chats_explorer/pagination.html' %}
{% endwith %}
</div>
</div>

85
var/www/templates/chats_explorer/block_obj_time_search.html Normal file
View File

@ -0,0 +1,85 @@
<div class="card mb-3 mt-1">
<div class="card-body">
<h5 class="card-title">Filter by Time :</h5>
{# <div class="row mb-3">#}
{# <div class="col-md-6">#}
{# <div class="input-group" id="date-range-from">#}
{# <div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>#}
{# <input class="form-control" id="date-range-from-input" placeholder="yyyy-mm-dd" value="{{ date_from[0:4] }}-{{ date_from[4:6] }}-{{ date_from[6:8] }}" name="date_from" autocomplete="off">#}
{# </div>#}
{# <div class="input-group" id="time-range-from">#}
{# <div class="input-group-prepend"><span class="input-group-text"><i class="far fa-clock" aria-hidden="true"></i></span></div>#}
{# <input class="form-control" type="time" id="time-range-from-input" name="time_from" autocomplete="off">#}
{# </div>#}
{# </div>#}
{# <div class="col-md-6">#}
{# <div class="input-group" id="date-range-to">#}
{# <div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>#}
{# <input class="form-control" id="date-range-to-input" placeholder="yyyy-mm-dd" value="{{ date_to[0:4] }}-{{ date_to[4:6] }}-{{ date_to[6:8] }}" name="date_to" autocomplete="off">#}
{# </div>#}
{# <div class="input-group" id="time-range-to">#}
{# <div class="input-group-prepend"><span class="input-group-text"><i class="far fa-clock" aria-hidden="true"></i></span></div>#}
{# <input class="form-control" type="time" id="time-range-from-to" name="time_to" autocomplete="off">#}
{# </div>#}
{# </div>#}
{# </div>#}
<div class="input-group mb-3">
<div class="input-group">
<div class="input-group-prepend"><span class="input-group-text">Numbers by page</span></div>
<input id="nb_messages" name="nb_messages" type="number" value="{{ nb }}" class="form-control" aria-describedby="button-clear-tags" autocomplete="off">
</div>
</div>
<button class="btn btn-primary" type="button" id="button-search-tags" onclick="filter_by_time()">
<i class="fas fa-search"></i> Show
</button>
</div>
</div>
<link href="{{ url_for('static', filename='css/daterangepicker.min.css') }}" rel="stylesheet">
<script src="{{ url_for('static', filename='js/moment.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.daterangepicker.min.js') }}"></script>
<script>
function filter_by_time() {
let parameter = "?uuid={{ obj_subtype }}&id={{ obj_id }}";
//let date_from = $('#date-range-from-input').val();
//let date_to = $('#date-range-to-input').val();
let nb_messages = $('#nb_messages').val();
//parameter = parameter + "&date_from=" + date_from + "&date_to=" + date_to + "&nb=" + nb_messages {%if page%}+ "&page={{ page }}"{%endif%};
parameter = parameter + "&nb=" + nb_messages {%if page%}+ "&page={{ page }}"{%endif%};
window.location.replace("{{ url_endpoint }}" + parameter);
}
$('#date-range-from').dateRangePicker({
separator : ' to ',
getValue: function(){
if ($('#date-range-from-input').val() && $('#date-range-to-input').val() )
return $('#date-range-from-input').val() + ' to ' + $('#date-range-to-input').val();
else
return '';
},
setValue: function(s,s1,s2){
$('#date-range-from-input').val(s1);
$('#date-range-to-input').val(s2);
}
});
$('#date-range-to').dateRangePicker({
separator : ' to ',
getValue: function(){
if ($('#date-range-from-input').val() && $('#date-range-to-input').val() )
return $('#date-range-from-input').val() + ' to ' + $('#date-range-to-input').val();
else
return '';
},
setValue: function(s,s1,s2){
$('#date-range-from-input').val(s1);
$('#date-range-to-input').val(s2);
}
});
</script>

2
var/www/templates/chats_explorer/chat_participants.html
View File

@ -97,7 +97,7 @@
<th>info</th>
<th>First Seen</th>
<th>Last Seen</th>
<th>NB Messages</th>
<th><i class="fas fa-comment-dots"></i></th>
<th></th>
</tr>
</thead>

20
var/www/templates/chats_explorer/chat_viewer.html
View File

@ -164,6 +164,18 @@
{% with translate_url=url_for('chats_explorer.chats_explorer_chat', uuid=chat['subtype']), obj_id=chat['id'] %}
{% include 'chats_explorer/block_translation.html' %}
{% endwith %}
{% with obj_subtype=chat['subtype'], obj_id=chat['id'], url_endpoint=url_for("chats_explorer.chats_explorer_chat"), nb=chat['pagination']['nb'] %}
{% set date_from=chat['first_seen'] %}
{% set date_to=chat['last_seen'] %}
{% include 'block_obj_time_search.html' %}
{% endwith %}
{% with endpoint_url=url_for('chats_explorer.chats_explorer_chat', uuid=chat['subtype']), pagination=chat['pagination'] %}
{% set endpoint_url = endpoint_url + "&id=" + chat['id'] + "&nb=" + chat['pagination']['nb'] | string %}
{% if translation_target %}
{% set endpoint_url = endpoint_url + "&target=" + translation_target %}
{% endif %}
{% include 'chats_explorer/pagination.html' %}
{% endwith %}
<div class="position-relative">
<div class="chat-messages p-2">
@ -189,6 +201,14 @@
</div>
</div>
{% with endpoint_url=url_for('chats_explorer.chats_explorer_chat', uuid=chat['subtype']), pagination=chat['pagination'] %}
{% set endpoint_url = endpoint_url + "&id=" + chat['id'] + "&nb=" + chat['pagination']['nb'] | string %}
{% if translation_target %}
{% set endpoint_url = endpoint_url + "&target=" + translation_target %}
{% endif %}
{% include 'chats_explorer/pagination.html' %}
{% endwith %}
{% endif %}
</div>

50
var/www/templates/chats_explorer/pagination.html Normal file
View File

@ -0,0 +1,50 @@
<div class="my-2">
<div class="d-flex justify-content-center">
<nav class="mt-4" aria-label="...">
<ul class="pagination">
<li class="page-item {% if pagination['page']==1 %}disabled{%endif%}">
<a class="page-link" href="{{ endpoint_url }}&page={{pagination['page']-1}}">Previous</a>
</li>
{% if pagination['page']>3 %}
<li class="page-item"><a class="page-link" href="{{ endpoint_url }}&page=1">1</a></li>
<li class="page-item disabled"><a class="page-link" aria-disabled="true" href="#">...</a></li>
<li class="page-item"><a class="page-link" href="{{ endpoint_url }}&page={{ pagination['page'] - 1 }}">{{ pagination['page'] - 1 }}</a></li>
<li class="page-item active"><a class="page-link" href="{{ endpoint_url }}&page={{ pagination['page'] }}">{{ pagination['page'] }}</a></li>
{% else %}
{% if pagination['page']>2 %}<li class="page-item"><a class="page-link" href="{{ endpoint_url }}&page={{ pagination['page'] - 2 }}">{{ pagination['page'] - 2 }}</a></li>{%endif%}
{% if pagination['page']>1 %}<li class="page-item"><a class="page-link" href="{{ endpoint_url }}&page={{ pagination['page'] - 1 }}">{{ pagination['page'] - 1 }}</a></li>{%endif%}
<li class="page-item active"><a class="page-link" href="{{ endpoint_url }}&page={{ pagination['page'] }}">{{ pagination['page'] }}</a></li>
{% endif %}
{% if pagination['nb_pages']>3 and pagination['nb_pages'] != pagination['page'] %}
<li class="page-item"><a class="page-link" href="{{ endpoint_url }}&page={{ pagination['page'] + 1 }}">{{ pagination['page'] + 1 }}</a></li>
<li class="page-item disabled"><a class="page-link" aria-disabled="true" href="#">...</a></li>
<li class="page-item"><a class="page-link" href="{{ endpoint_url }}&page={{ pagination['nb_pages'] }}">{{ pagination['nb_pages'] }}</a></li>
{% else %}
{%if pagination['nb_pages'] - pagination['page']>2 %}<li class="page-item"><a class="page-link" href="{{ endpoint_url }}&page={{ pagination['nb_pages'] - 2 }}">{{ pagination['nb_pages'] - 2 }}</a></li>{%endif%}
{%if pagination['nb_pages'] - pagination['page']>1 %}<li class="page-item"><a class="page-link" href="{{ endpoint_url }}&page={{ pagination['nb_pages'] - 1 }}">{{ pagination['nb_pages'] - 1 }}</a></li>{%endif%}
{%if pagination['nb_pages'] - pagination['page']>0 %}<li class="page-item"><a class="page-link" href="{{ endpoint_url }}&page={{ pagination['nb_pages'] }}">{{ pagination['nb_pages'] }}</a></li>{%endif%}
{% endif %}
<li class="page-item {%if pagination['page']==pagination['nb_pages'] %}disabled{%endif%}">
<a class="page-link" href="{{ endpoint_url }}&page={{ pagination['page'] + 1 }}" aria-disabled="true">Next</a>
</li>
</ul>
</nav>
</div>
{%if pagination['total'] %}
<div class="d-flex justify-content-center">
<span class="badge badge-info text-white">
results:&nbsp;
<span class="badge badge-light">{{ pagination['nb_first'] }}-{{ pagination['nb_last'] }}</span>
<span> / </span>
<span class="badge badge-light">{{ pagination['total'] }}</span>
</span>
</div>
<br>
<br>
<br>
{%endif%}
</div>