D4
/
d4-core
mirror of https://github.com/D4-project/d4-core
2
0
Fork 0
Code Issues Releases Wiki Activity

chg: [UI v0.2] manage extended accepted types + extended types analyzers

visu-type
Terrtia 2019-03-14 11:28:33 +01:00
parent b6c18f606b
commit 365f542d1e
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
3 changed files with 173 additions and 12 deletions
server
web
workers/workers_2/meta_types_modules

56
server/web/Flask_server.py
View File

@ -238,7 +238,25 @@ def server_management():
list_accepted_types.append({"id": int(type), "description": description, 'list_analyzer_uuid': list_analyzer_uuid})
return render_template("server_management.html", list_accepted_types=list_accepted_types,
list_accepted_extended_types = []
for extended_type in redis_server_metadata.smembers('server:accepted_extended_type'):
list_analyzer_uuid = []
for analyzer_uuid in redis_server_metadata.smembers('analyzer:254:{}'.format(extended_type)):
size_limit = redis_server_metadata.hget('analyzer:{}'.format(analyzer_uuid), 'max_size')
if size_limit is None:
size_limit = analyzer_list_max_default_size
last_updated = redis_server_metadata.hget('analyzer:{}'.format(analyzer_uuid), 'last_updated')
if last_updated is None:
last_updated = 'Never'
len_queue = redis_server_analyzer.llen('analyzer:{}:{}'.format(extended_type, analyzer_uuid))
if len_queue is None:
len_queue = 0
list_analyzer_uuid.append({'uuid': analyzer_uuid, 'size_limit': size_limit,'last_updated': last_updated, 'length': len_queue})
list_accepted_extended_types.append({"name": extended_type, 'list_analyzer_uuid': list_analyzer_uuid})
return render_template("server_management.html", list_accepted_types=list_accepted_types, list_accepted_extended_types=list_accepted_extended_types,
default_analyzer_max_line_len=default_analyzer_max_line_len,
blacklisted_ip=blacklisted_ip, unblacklisted_ip=unblacklisted_ip,
blacklisted_uuid=blacklisted_uuid, unblacklisted_uuid=unblacklisted_uuid)
@ -362,10 +380,12 @@ def uuid_change_stream_max_size():
else:
return 'Invalid uuid'
# # TODO: check analyser uuid dont exist
@app.route('/add_new_analyzer')
def add_new_analyzer():
type = request.args.get('type')
user = request.args.get('redirect')
metatype_name = request.args.get('metatype_name')
analyzer_uuid = request.args.get('analyzer_uuid')
if is_valid_uuid_v4(analyzer_uuid):
try:
@ -374,7 +394,11 @@ def add_new_analyzer():
return 'type, Invalid Integer'
except:
return 'type, Invalid Integer'
redis_server_metadata.sadd('analyzer:{}'.format(type), analyzer_uuid)
if type == 254:
# # TODO: check metatype_name
redis_server_metadata.sadd('analyzer:{}:{}'.format(type, metatype_name), analyzer_uuid)
else:
redis_server_metadata.sadd('analyzer:{}'.format(type), analyzer_uuid)
if user:
return redirect(url_for('server_management'))
else:
@ -384,6 +408,7 @@ def add_new_analyzer():
def empty_analyzer_queue():
analyzer_uuid = request.args.get('analyzer_uuid')
type = request.args.get('type')
metatype_name = request.args.get('metatype_name')
user = request.args.get('redirect')
if is_valid_uuid_v4(analyzer_uuid):
try:
@ -392,7 +417,10 @@ def empty_analyzer_queue():
return 'type, Invalid Integer'
except:
return 'type, Invalid Integer'
redis_server_analyzer.delete('analyzer:{}:{}'.format(type, analyzer_uuid))
if type == 254:
redis_server_analyzer.delete('analyzer:{}:{}'.format(metatype_name, analyzer_uuid))
else:
redis_server_analyzer.delete('analyzer:{}:{}'.format(type, analyzer_uuid))
if user:
return redirect(url_for('server_management'))
else:
@ -402,6 +430,7 @@ def empty_analyzer_queue():
def remove_analyzer():
analyzer_uuid = request.args.get('analyzer_uuid')
type = request.args.get('type')
metatype_name = request.args.get('metatype_name')
user = request.args.get('redirect')
if is_valid_uuid_v4(analyzer_uuid):
try:
@ -410,8 +439,12 @@ def remove_analyzer():
return 'type, Invalid Integer'
except:
return 'type, Invalid Integer'
redis_server_metadata.srem('analyzer:{}'.format(type), analyzer_uuid)
redis_server_analyzer.delete('analyzer:{}:{}'.format(type, analyzer_uuid))
if type == 254:
redis_server_metadata.srem('analyzer:{}:{}'.format(type, metatype_name), analyzer_uuid)
redis_server_analyzer.delete('analyzer:{}:{}'.format(metatype_name, analyzer_uuid))
else:
redis_server_metadata.srem('analyzer:{}'.format(type), analyzer_uuid)
redis_server_analyzer.delete('analyzer:{}:{}'.format(type, analyzer_uuid))
redis_server_metadata.delete('analyzer:{}'.format(analyzer_uuid))
if user:
return redirect(url_for('server_management'))
@ -556,10 +589,17 @@ def unblacklist_ip_by_uuid():
@app.route('/add_accepted_type')
def add_accepted_type():
type = request.args.get('type')
extended_type_name = request.args.get('extended_type_name')
user = request.args.get('redirect')
json_type_description = get_json_type_description()
try:
type = int(type)
except:
return 'Invalid type'
if json_type_description[int(type)]:
redis_server_metadata.sadd('server:accepted_type', type)
if type == 254:
redis_server_metadata.sadd('server:accepted_extended_type', extended_type_name)
if user:
return redirect(url_for('server_management'))
else:
@ -577,6 +617,12 @@ def remove_accepted_type():
else:
return 'Invalid type'
@app.route('/remove_accepted_extended_type')
def remove_accepted_extended_type():
type_name = request.args.get('type_name')
redis_server_metadata.srem('server:accepted_extended_type', type_name)
return redirect(url_for('server_management'))
# demo function
@app.route('/delete_data')
def delete_data():

125
server/web/templates/server_management.html
View File

@ -175,13 +175,40 @@
{% endfor %}
</tbody>
</table>
<div class="mt-3">
<table class="table table-striped table-bordered table-hover mt-3" id="table_accepted_extended_type">
<thead class="thead-dark">
<tr>
<th>Type Name</th>
<th>Description</th>
<th>Remove Type</th>
</tr>
</thead>
<tbody id="table_accepted_extended_type_tbody">
{% for type in list_accepted_extended_types %}
<tr>
<td>{{type['name']}}</td>
<td>{{type['description']}}</td>
<td>
<a href="{{ url_for('remove_accepted_extended_type') }}?type_name={{type['name']}}">
<button type="button" class="btn btn-outline-danger">Remove Extended Type</button>
</a>
</td>
</tr>
{% endfor %}
</tbody>
</table>
</div>
</div>
<div class="col-md-4">
<div class="card border-dark mt-3" style="max-width: 18rem;">
<div class="card-body text-dark">
<h5 class="card-title">Add New Types</h5>
<input class="form-control" type="number" id="accepted_type" value="1" min="1" max="8" required>
<button type="button" class="btn btn-outline-primary mt-1" onclick="window.location.href ='{{ url_for('add_accepted_type') }}?redirect=1&type='+$('#accepted_type').val();">Add New Type</button>
<input class="form-control" type="number" id="accepted_type" value="1" min="1" max="254" required>
<input class="form-control" type="text" id="extended_type_name" placeholder="Type Name">
<button type="button" class="btn btn-outline-primary mt-1" onclick="window.location.href ='{{ url_for('add_accepted_type') }}?redirect=1&type='+$('#accepted_type').val()+'&extended_type_name='+$('#extended_type_name').val();">Add New Type</button>
</div>
</div>
</div>
@ -245,14 +272,62 @@
{% endfor %}
</tbody>
</table>
<div class="mt-3">
<table class="table table-striped table-bordered table-hover" id="analyzer_accepted_extended_types">
<thead class="thead-dark">
<tr>
<th>Type Name</th>
<th style="max-width: 800px;">uuid</th>
<th style="max-width: 800px;">last updated</th>
<th style="max-width: 800px;">Change max size limit</th>
<th style="max-width: 800px;">Analyzer Queue</th>
</tr>
</thead>
<tbody id="analyzer_accepted_extended_types_tbody">
{% for type in list_accepted_extended_types %}
{% if type['list_analyzer_uuid'] %}
{% for analyzer in type['list_analyzer_uuid'] %}
<tr>
<td>{{type['name']}}</td>
<td>
<div class="d-flex">
{{analyzer['uuid']}}
<a href="{{ url_for('remove_analyzer') }}?redirect=1&type=254&metatype_name={{type['name']}}&analyzer_uuid={{analyzer['uuid']}}" class="ml-auto">
<button type="button" class="btn btn-outline-danger"><i class="fa fa-trash"></i></button>
</a>
</div>
</td>
<td>{{analyzer['last_updated']}}</td>
<td>
<div class="d-xl-flex justify-content-xl-center">
<input class="form-control mr-lg-1" style="max-width: 100px;" type="number" id="max_size_analyzer_{{analyzer['uuid']}}" value="{{analyzer['size_limit']}}" min="0" required="">
<button type="button" class="btn btn-outline-secondary" onclick="window.location.href ='{{ url_for('analyzer_change_max_size') }}?analyzer_uuid={{analyzer['uuid']}}&redirect=0&max_size_analyzer='+$('#max_size_analyzer_{{analyzer['uuid']}}').val();">Change Max Size</button>
</div>
</td>
<td>
<a href="{{ url_for('empty_analyzer_queue') }}?redirect=1&type=254&metatype_name={{type['name']}}&analyzer_uuid={{analyzer['uuid']}}">
<button type="button" class="btn btn-outline-danger"><i class="fa fa-eraser"></i></button>
</a>
<button type="button" class="btn btn-outline-info ml-xl-3" onclick="get_analyser_sample('{{type['name']}}', '{{analyzer['uuid']}}');"><i class="fa fa-database"></i> {{analyzer['length']}}</button>
</td>
</tr>
{% endfor %}
{% endif %}
{% endfor %}
</tbody>
</table>
</div>
</div>
<div class="col-xl-4">
<div class="card border-dark mt-3" style="max-width: 18rem;">
<div class="card-body text-dark">
<h5 class="card-title">Add New Analyzer Queue</h5>
<input class="form-control" type="number" id="analyzer_type" value="1" min="1" max="8" required>
<input class="form-control" type="text" id="analyzer_uuid" required>
<button type="button" class="btn btn-outline-primary mt-1" onclick="window.location.href ='{{ url_for('add_new_analyzer') }}?redirect=1&type='+$('#analyzer_type').val()+'&analyzer_uuid='+$('#analyzer_uuid').val();">Add New Analyzer</button>
<input class="form-control" type="number" id="analyzer_type" value="1" min="1" max="254" required>
<input class="form-control" type="text" id="analyzer_metatype_name" placeholder="Meta Type Name">
<input class="form-control" type="text" id="analyzer_uuid" required placeholder="Analyzer uuid">
<button type="button" class="btn btn-outline-primary mt-1" onclick="window.location.href ='{{ url_for('add_new_analyzer') }}?redirect=1&type='+$('#analyzer_type').val()+'&analyzer_uuid='+$('#analyzer_uuid').val()+'&metatype_name='+$('#analyzer_metatype_name').val();">Add New Analyzer</button>
</div>
</div>
</div>
@ -295,6 +370,8 @@
<script>
var table
$(document).ready(function(){
$('#extended_type_name').hide()
$('#analyzer_metatype_name').hide()
table = $('#myTable_').DataTable(
{
@ -314,6 +391,44 @@ $(document).ready(function(){
});
var tbody = $("#table_accepted_extended_type_tbody");
if (tbody.children().length == 0) {
$("#table_accepted_extended_type").hide();
} else {
table = $('#table_accepted_extended_type').DataTable(
{
"order": [[ 0, "asc" ]]
}
);
}
var tbody = $("#analyzer_accepted_extended_types_tbody");
if (tbody.children().length == 0) {
$("#analyzer_accepted_extended_types").hide();
} else {
table = $('#analyzer_accepted_extended_types').DataTable(
{
"order": [[ 0, "asc" ]]
}
);
}
$('#accepted_type').on('input', function() {
if ($('#accepted_type').val() == 254){
$('#extended_type_name').show()
} else {
$('#extended_type_name').hide()
}
});
$('#analyzer_type').on('input', function() {
if ($('#analyzer_type').val() == 254){
$('#analyzer_metatype_name').show()
} else {
$('#analyzer_metatype_name').hide()
}
});
function get_analyser_sample(type, analyzer_uuid, max_line_len){
$.getJSON( "{{url_for('get_analyser_sample')}}?type="+type+"&analyzer_uuid="+analyzer_uuid+"&max_line_len="+max_line_len, function( data ) {

4
server/workers/workers_2/meta_types_modules/MetaTypesDefault.py
View File

@ -153,12 +153,12 @@ class MetaTypesDefault:
## save full path in anylyzer queue
for analyzer_uuid in redis_server_metadata.smembers('analyzer:{}:{}'.format(TYPE, self.get_type_name())):
analyzer_uuid = analyzer_uuid.decode()
redis_server_analyzer.lpush('analyzer:{}:{}'.format(TYPE, analyzer_uuid), data_to_send)
redis_server_analyzer.lpush('analyzer:{}:{}'.format(self.get_type_name(), analyzer_uuid), data_to_send)
redis_server_metadata.hset('analyzer:{}'.format(analyzer_uuid), 'last_updated', time.time())
analyser_queue_max_size = redis_server_metadata.hget('analyzer:{}'.format(analyzer_uuid), 'max_size')
if analyser_queue_max_size is None:
analyser_queue_max_size = analyzer_list_max_default_size
redis_server_analyzer.ltrim('analyzer:{}:{}'.format(TYPE, analyzer_uuid), 0, analyser_queue_max_size)
redis_server_analyzer.ltrim('analyzer:{}:{}'.format(self.get_type_name(), analyzer_uuid), 0, analyser_queue_max_size)
######## GET FUNCTIONS ########