CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

chg: [UI Tags search] bootstrap 4 migration

pull/342/head
Terrtia 2019-03-14 17:04:55 +01:00
parent 9e4813ca4b
commit 23d9cb2265
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
8 changed files with 307 additions and 118 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
OVERVIEW.md
View File

@ -14,7 +14,6 @@ Redis and ARDB overview
* ARDB on TCP port 6382
DB 1 - Curve
DB 2 - TermFreq
DB 3 - Trending
@ -24,7 +23,7 @@ Redis and ARDB overview
DB 7 - Metadata
DB 8 - Statistics
DB 9 - Crawler
* ARDB on TCP port <year>
- DB 0 - Lines duplicate
- DB 1 - Hashes
@ -36,32 +35,58 @@ Redis and ARDB overview
##### Hset:
| Key | Field | Value |
| ------ | ------ | ------ |
| daily_tags:**<date>** | **<tag>** | **<nb tagged this day>** |
| daily_tags:**date** | **tag** | **nb tagged this day** |
| | |
| tag_metadata:**<tag>** | first_seen | **<date>** |
| tag_metadata:**<tag>** | last_seen | **<date>** |
| tag_metadata:**tag** | first_seen | **date** |
| tag_metadata:**tag** | last_seen | **date** |
##### Set:
| Key | Value |
| ------ | ------ |
| list_tags | **<tag>** |
| active_taxonomies | **<taxonomie>** |
| active_galaxies | **<galaxie>** |
| active_tag_**<taxonomie or galaxy>** | **<tag>** |
| synonym_tag_misp-galaxy:**<galaxy>** | **<tag synonym>** |
| list_export_tags | **<user_tag>** |
| **<tag>**:**<date>** | **<paste>** |
| list_tags | **tag** |
| active_taxonomies | **taxonomie** |
| active_galaxies | **galaxie** |
| active_tag_**taxonomie or galaxy** | **tag** |
| synonym_tag_misp-galaxy:**galaxy** | **tag synonym** |
| list_export_tags | **user_tag** |
| **tag**:**date** | **paste** |
##### old:
| Key | Value |
| ------ | ------ |
| **<tag>** | **<paste>** |
| *tag* | *paste* |
## DB9 - Crawler:
##### Hset:
| Key | Field | Value |
| ------ | ------ | ------ |
| **service type**:**domain** | first_seen | **date** |
| | last_check | **date** |
| | paste_parent | **parent last crawling (can be auto or manual)** |
##### Zset:
| Key | Field | Value |
| ------ | ------ | ------ |
| crawler_history_**service type**:**domain** | **item root (first crawled item)** | **epoch (seconds)** |
##### Regular key:
| Key | Value |
| ------ | ------ |
| crawler_history_**service type**:**domain** | **json config** |
##### exemple json config:
```json
{
"closespider_pagecount": 1,
"time": 3600,
"depth_limit": 0,
"har": 0,
"png": 0
}
```
ARDB overview
---------------------------
@ -130,5 +155,3 @@ ARDB_DB
GET - 'base64_decoded:'+date nd_decoded
GET - 'binary_decoded:'+date nd_decoded

6
bin/Tags.py
View File

@ -75,12 +75,12 @@ if __name__ == '__main__':
if res == 1:
print("new tags added : {}".format(tag))
# add the path to the tag set
date = get_paste_date(path)
res = server.sadd('{}:{}'.format(tag, date), path)
curr_date = datetime.date.today().strftime("%Y%m%d")
res = server.sadd('{}:{}'.format(tag, curr_date), path)
if res == 1:
print("new paste: {}".format(path))
print(" tagged: {}".format(tag))
set_tag_metadata(tag, date)
set_tag_metadata(tag, curr_date)
server_metadata.sadd('tag:{}'.format(path), tag)
curr_date = datetime.date.today().strftime("%Y%m%d")

183
var/www/modules/Tags/Flask_Tags.py
View File

@ -8,7 +8,7 @@ import redis
from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for
import json
from datetime import datetime
import datetime
import ssdeep
import Paste
@ -71,9 +71,98 @@ def get_tags_with_synonyms(tag):
# ============= ROUTES ==============
@Tags.route("/Tags/", methods=['GET'])
@Tags.route("/tags/", methods=['GET'])
def Tags_page():
return render_template("Tags.html")
current_date = datetime.date.today().strftime("%Y-%m-%d")
tags = request.args.get('ltags')
if tags is None:
return render_template("Tags.html", date_from=current_date, date_to=current_date)
else:
tags = request.args.get('ltags')
list_tags = tags.split(',')
list_tag = []
for tag in list_tags:
list_tag.append(tag.replace('"','\"'))
# TODO verify input
if(type(list_tags) is list):
# no tag
if list_tags is False:
print('empty')
# 1 tag
elif len(list_tags) < 2:
tagged_pastes = r_serv_tags.smembers(list_tags[0])
# 2 tags or more
else:
tagged_pastes = r_serv_tags.sinter(list_tags[0], *list_tags[1:])
else :
return 'INCORRECT INPUT'
all_content = []
paste_date = []
paste_linenum = []
all_path = []
allPastes = list(tagged_pastes)
paste_tags = []
for path in allPastes[0:50]: ######################moduleName
all_path.append(path)
paste = Paste.Paste(path)
content = paste.get_p_content()
content_range = max_preview_char if len(content)>max_preview_char else len(content)-1
all_content.append(content[0:content_range].replace("\"", "\'").replace("\r", " ").replace("\n", " "))
curr_date = str(paste._get_p_date())
curr_date = curr_date[0:4]+'/'+curr_date[4:6]+'/'+curr_date[6:]
paste_date.append(curr_date)
paste_linenum.append(paste.get_lines_info()[0])
p_tags = r_serv_metadata.smembers('tag:'+path)
complete_tags = []
l_tags = []
for tag in p_tags:
complete_tag = tag
tag = tag.split('=')
if len(tag) > 1:
if tag[1] != '':
tag = tag[1][1:-1]
# no value
else:
tag = tag[0][1:-1]
# use for custom tags
else:
tag = tag[0]
l_tags.append( (tag,complete_tag) )
paste_tags.append(l_tags)
if len(allPastes) > 10:
finished = False
else:
finished = True
return render_template("Tags.html",
all_path=all_path,
tags=tags,
list_tag = list_tag,
date_from=current_date,
date_to=current_date,
paste_tags=paste_tags,
bootstrap_label=bootstrap_label,
content=all_content,
paste_date=paste_date,
paste_linenum=paste_linenum,
char_to_display=max_preview_modal,
finished=finished)
@Tags.route("/Tags/get_all_tags")
def get_all_tags():
@ -173,94 +262,6 @@ def get_tags_galaxy():
else:
return 'this galaxy is disable'
@Tags.route("/Tags/get_tagged_paste")
def get_tagged_paste():
tags = request.args.get('ltags')
list_tags = tags.split(',')
list_tag = []
for tag in list_tags:
list_tag.append(tag.replace('"','\"'))
# TODO verify input
if(type(list_tags) is list):
# no tag
if list_tags is False:
print('empty')
# 1 tag
elif len(list_tags) < 2:
tagged_pastes = r_serv_tags.smembers(list_tags[0])
# 2 tags or more
else:
tagged_pastes = r_serv_tags.sinter(list_tags[0], *list_tags[1:])
else :
return 'INCORRECT INPUT'
#TODO FIXME
currentSelectYear = int(datetime.now().year)
all_content = []
paste_date = []
paste_linenum = []
all_path = []
allPastes = list(tagged_pastes)
paste_tags = []
for path in allPastes[0:50]: ######################moduleName
all_path.append(path)
paste = Paste.Paste(path)
content = paste.get_p_content()
content_range = max_preview_char if len(content)>max_preview_char else len(content)-1
all_content.append(content[0:content_range].replace("\"", "\'").replace("\r", " ").replace("\n", " "))
curr_date = str(paste._get_p_date())
curr_date = curr_date[0:4]+'/'+curr_date[4:6]+'/'+curr_date[6:]
paste_date.append(curr_date)
paste_linenum.append(paste.get_lines_info()[0])
p_tags = r_serv_metadata.smembers('tag:'+path)
complete_tags = []
l_tags = []
for tag in p_tags:
complete_tag = tag
tag = tag.split('=')
if len(tag) > 1:
if tag[1] != '':
tag = tag[1][1:-1]
# no value
else:
tag = tag[0][1:-1]
# use for custom tags
else:
tag = tag[0]
l_tags.append( (tag,complete_tag) )
paste_tags.append(l_tags)
if len(allPastes) > 10:
finished = False
else:
finished = True
return render_template("tagged.html",
year=currentSelectYear,
all_path=all_path,
tags=tags,
list_tag = list_tag,
paste_tags=paste_tags,
bootstrap_label=bootstrap_label,
content=all_content,
paste_date=paste_date,
paste_linenum=paste_linenum,
char_to_display=max_preview_modal,
finished=finished)
@Tags.route("/Tags/remove_tag")
def remove_tag():

168
var/www/modules/Tags/templates/Tags.html
View File

@ -9,12 +9,15 @@
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/daterangepicker.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dataTables.bootstrap4.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/tags.css') }}" rel="stylesheet" type="text/css" />
<!-- JS -->
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
<script language="javascript" src="{{ url_for('static', filename='js/moment.min.js') }}"></script>
<script language="javascript" src="{{ url_for('static', filename='js/jquery.daterangepicker.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/tags.js') }}"></script>
@ -68,6 +71,48 @@
</div>
</div>
<div>
{%if all_path%}
<table class="table table-bordered table-hover " id="myTable_">
<thead class="thead-dark">
<tr>
<th>Date</th>
<th style="max-width: 800px;">Path</th>
<th># of lines</th>
<th>Action</th>
</tr>
</thead>
<tbody>
{% for path in all_path %}
<tr>
<td class="pb-0">{{ paste_date[loop.index0] }}</td>
<td class="pb-0"><a target="_blank" href="{{ url_for('showsavedpastes.showsavedpaste') }}?paste={{path}}" class="text-secondary">
<div style="line-height:0.9;">{{ path }}</div>
</a>
<div>
{% for tag in paste_tags[loop.index0] %}
<a href="{{ url_for('Tags.Tags_page') }}?ltags={{ tag[1] }}">
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag[0] }}</span>
</a>
{% endfor %}
</div>
</td class="pb-0">
<td class="pb-0">{{ paste_linenum[loop.index0] }}</td>
<td class="pb-0"><p>
<span class="fas fa-info-circle" data-toggle="tooltip" data-placement="left" title="{{ content[loop.index0] }} "></span>
<button type="button" class="btn btn-light" data-num="{{ loop.index0 + 1 }}" data-toggle="modal" data-target="#mymodal" data-url="{{ url_for('showsavedpastes.showsavedpaste') }}?paste={{ path }}&num={{ loop.index0+1 }}" data-path="{{ path }}">
<span class="fas fa-search-plus"></span>
</button></p>
</td>
</tr>
{% endfor %}
</tbody>
</table>
{%endif%}
</div>
<div>
<br></br>
<a class="btn btn-light text-secondary" href="{{ url_for('Tags.taxonomies') }}" target="_blank">
@ -98,14 +143,38 @@
</body>
<script>
var ltags
var ltags;
var search_table;
var last_clicked_paste;
var can_change_modal_content = true;
$("#myTable_").attr('data-numElem', "{{ all_path|length }}");
$(document).ready(function(){
$("#page-Browse-Items").addClass("active");
search_table = $('#myTable_').DataTable({ "order": [[ 0, "asc" ]] });
// Use to bind the button with the new displayed data
// (The bind do not happens if the dataTable is in tabs and the clicked data is in another page)
search_table.on( 'draw.dt', function () {
// Bind tooltip each time we draw a new page
$('[data-toggle="tooltip"]').tooltip();
// On click, get html content from url and update the corresponding modal
$("[data-toggle='modal']").off('click.openmodal').on("click.openmodal", function (event) {
get_html_and_update_modal(event, $(this));
});
} );
var valueData = [
{% for tag in list_tag %}
'{{tag|safe}}',
{% endfor %}
];
$.getJSON("{{ url_for('Tags.get_all_tags') }}",
function(data) {
ltags = $('#ltags').tagSuggest({
data: data,
value: valueData,
sortOrder: 'name',
maxDropHeight: 200,
name: 'ltags'
@ -144,7 +213,7 @@ $(document).ready(function(){
<script>
function searchTags() {
var data = ltags.getValue();
window.location.replace("{{ url_for('Tags.get_tagged_paste') }}?ltags=" + data);
window.location.replace("{{ url_for('Tags.Tags_page') }}?ltags=" + data);
}
function emptyTags() {
ltags.clear();
@ -165,4 +234,99 @@ function toggle_sidebar(){
}
</script>
<!-- Dynamically update the modal -->
<script>
// static data
var alert_message = '<div class="alert alert-info alert-dismissable"><button type="button" class="close" data-dismiss="alert" aria-hidden="true">×</button><strong>No more data.</strong> Full paste displayed.</div>';
var complete_paste = null;
var char_to_display = {{ char_to_display }};
var start_index = 0;
// When the modal goes out, refresh it to normal content
$("#mymodal").on('hidden.bs.modal', function () {
can_change_modal_content = true;
$("#mymodalbody").html("<p>Loading paste information...</p>");
var loading_gif = "<img id='loading-gif-modal' class='img-center' src=\"{{url_for('static', filename='image/loading.gif') }}\" height='26' width='26' style='margin: 4px;'>";
$("#mymodalbody").append(loading_gif); // Show the loading GIF
$("#button_show_path").attr('href', '');
$("#button_show_path").hide();
complete_paste = null;
start_index = 0;
});
// Update the paste preview in the modal
function update_preview() {
if (start_index + char_to_display > complete_paste.length-1){ // end of paste reached
var final_index = complete_paste.length-1;
var flag_stop = true;
} else {
var final_index = start_index + char_to_display;
}
if (final_index != start_index){ // still have data to display
// Append the new content using text() and not append (XSS)
$("#mymodalbody").find("#paste-holder").text($("#mymodalbody").find("#paste-holder").text()+complete_paste.substring(start_index+1, final_index+1));
start_index = final_index;
if (flag_stop)
nothing_to_display();
} else {
nothing_to_display();
}
}
// Update the modal when there is no more data
function nothing_to_display() {
var new_content = $(alert_message).hide();
$("#mymodalbody").find("#panel-body").append(new_content);
new_content.show('fast');
$("#load-more-button").hide();
}
function get_html_and_update_modal(event, truemodal) {
event.preventDefault();
var modal=truemodal;
var url = " {{ url_for('showsavedpastes.showpreviewpaste') }}?paste=" + modal.attr('data-path') + "&num=" + modal.attr('data-num');
last_clicked_paste = modal.attr('data-num');
$.get(url, function (data) {
// verify that the reveived data is really the current clicked paste. Otherwise, ignore it.
var received_num = parseInt(data.split("|num|")[1]);
if (received_num == last_clicked_paste && can_change_modal_content) {
can_change_modal_content = false;
// clear data by removing html, body, head tags. prevent dark modal background stack bug.
var cleared_data = data.split("<body>")[1].split("</body>")[0];
$("#mymodalbody").html(cleared_data);
var button = $('<button type="button" id="load-more-button" class="btn btn-info btn-xs center-block" data-url="' + $(modal).attr('data-path') +'" data-toggle="tooltip" data-placement="bottom" title="Load more content"><span class="glyphicon glyphicon-download"></span></button>');
button.tooltip();
$("#mymodalbody").children(".panel-default").append(button);
$("#button_show_path").attr('href', $(modal).attr('data-url'));
$("#button_show_path").show('fast');
$("#loading-gif-modal").css("visibility", "hidden"); // Hide the loading GIF
if ($("[data-initsize]").attr('data-initsize') < char_to_display) { // All the content is displayed
nothing_to_display();
}
// On click, donwload all paste's content
$("#load-more-button").on("click", function (event) {
if (complete_paste == null) { //Donwload only once
$.get("{{ url_for('showsavedpastes.getmoredata') }}"+"?paste="+$(modal).attr('data-path'), function(data, status){
complete_paste = data;
update_preview();
});
} else {
update_preview();
}
});
} else if (can_change_modal_content) {
$("#mymodalbody").html("Ignoring previous not finished query of paste #" + received_num);
}
});
}
</script>
</html>

2
var/www/modules/hiddenServices/templates/header_hiddenServices.html
View File

@ -1 +1 @@
<li id='page-hiddenServices'><a href="{{ url_for('hiddenServices.hiddenServices_page') }}"><i class="fa fa-user-secret"></i> hidden Services </a></li>
<li id='page-hiddenServices'><a href="{{ url_for('hiddenServices.dashboard') }}"><i class="fa fa-user-secret"></i> hidden Services </a></li>

7
var/www/modules/hiddenServices/templates/showDomain.html
View File

@ -7,6 +7,7 @@
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/daterangepicker.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dataTables.bootstrap4.min.css') }}" rel="stylesheet">
<!-- JS -->
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
@ -70,7 +71,7 @@
{%endif%}
<div>
{% for tag in origin_paste_tags %}
<a href="{{ url_for('Tags.get_tagged_paste') }}?ltags={{ tag[1] }}">
<a href="{{ url_for('Tags.Tags_page') }}?ltags={{ tag[1] }}">
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag[0] }}</span>
</a>
{% endfor %}
@ -82,7 +83,7 @@
<div>
{% for tag in domain_tags %}
<a href="{{ url_for('Tags.get_tagged_paste') }}?ltags={{ tag }}">
<a href="{{ url_for('Tags.Tags_page') }}?ltags={{ tag }}">
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }} <i>{{ domain_tags[tag] }}</i></span>
</a>
{% endfor %}
@ -106,7 +107,7 @@
</a>
<div>
{% for tag in paste_tags[loop.index0] %}
<a href="{{ url_for('Tags.get_tagged_paste') }}?ltags={{ tag[1] }}">
<a href="{{ url_for('Tags.Tags_page') }}?ltags={{ tag[1] }}">
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag[0] }}</span>
</a>
{% endfor %}

4
var/www/modules/search/templates/search.html
View File

@ -101,7 +101,7 @@
<td><a target="_blank" href="{{ url_for('showsavedpastes.showsavedpaste') }}?paste={{path}}">{{ path }}</a>
<div>
{% for tag in paste_tags[loop.index0] %}
<a href="{{ url_for('Tags.get_tagged_paste') }}?ltags={{ tag[1] }}">
<a href="{{ url_for('Tags.Tags_page') }}?ltags={{ tag[1] }}">
<span class="label label-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag[0] }}</span>
</a>
{% endfor %}
@ -201,7 +201,7 @@
var curr_preview = data.preview_array[i].replace(/\"/g, "\'");
var tag = ""
for(j=0; j<data.list_tags[i].length; j++) {
tag = tag + "<a href=\"{{ url_for('Tags.get_tagged_paste') }}?ltags=" + data.list_tags[i][j][1] + ">"
tag = tag + "<a href=\"{{ url_for('Tags.Tags_page') }}?ltags=" + data.list_tags[i][j][1] + ">"
+ "<span class=\"label label-" + data.bootstrap_label[j % 5] + " pull-left\">" + data.list_tags[i][j][0]
+ "</span>" + "</a>"
}

2
var/www/templates/nav_bar.html
View File

@ -22,7 +22,7 @@
<a class="nav-link" href="#" aria-disabled="true"><i class="fas fa-crosshairs"></i> Leaks Hunter</a>
</li>
<li class="nav-item mr-3">
<a class="nav-link" id="page-Crawler" href="#" tabindex="-1" href="#" aria-disabled="true"><i class="fas fa-spider"></i> Crawlers</a>
<a class="nav-link" id="page-Crawler" href="{{ url_for('hiddenServices.dashboard') }}" tabindex="-1" aria-disabled="true"><i class="fas fa-spider"></i> Crawlers</a>
</li>
<li class="nav-item mr-3">
<a class="nav-link" href="#" aria-disabled="true"><i class="fas fa-lock-open"></i> Decoded</a>