D4
/
d4-core
mirror of https://github.com/D4-project/d4-core
2
0
Fork 0
Code Issues Releases Wiki Activity

chg: [UI v0.3 uuid_management] add type/uuid barchart

pull/23/head
Terrtia 2019-04-04 13:55:56 +02:00
parent 5d85903bf2
commit 45546fd4a2
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
3 changed files with 297 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
server/web/Flask_server.py
View File

@ -105,6 +105,18 @@ def get_whois_ouput(ip):
else:
return ''
def get_substract_date_range(num_day, date_from=None):
if date_from is None:
date_from = datetime.datetime.now()
else:
date_from = datetime.date(int(date_from[0:4]), int(date_from[4:6]), int(date_from[6:8]))
l_date = []
for i in range(num_day):
date = date_from - datetime.timedelta(days=i)
l_date.append( date.strftime('%Y%m%d') )
return list(reversed(l_date))
# ========== ERRORS ============
@app.errorhandler(404)
@ -315,6 +327,17 @@ def uuid_management():
if uuid_key is None:
uuid_key = redis_server_metadata.get('server:hmac_default_key')
uuid_all_type_list = []
uuid_all_type = redis_server_metadata.smembers('all_types_by_uuid:{}'.format(uuid_sensor))
for type in uuid_all_type:
type_first_seen = redis_server_metadata.hget('metadata_type_by_uuid:{}:{}'.format(uuid_sensor, type), 'first_seen')
type_last_seen = redis_server_metadata.hget('metadata_type_by_uuid:{}:{}'.format(uuid_sensor, type), 'last_seen')
if type_first_seen:
type_first_seen = datetime.datetime.fromtimestamp(float(type_first_seen)).strftime('%Y-%m-%d %H:%M:%S')
if type_last_seen:
type_last_seen = datetime.datetime.fromtimestamp(float(type_last_seen)).strftime('%Y-%m-%d %H:%M:%S')
uuid_all_type_list.append({'type': type, 'first_seen':type_first_seen, 'last_seen': type_last_seen})
list_ip = redis_server_metadata.lrange('list_uuid_ip:{}'.format(uuid_sensor), 0, -1)
all_ip = []
for elem in list_ip:
@ -322,7 +345,8 @@ def uuid_management():
all_ip.append({'ip': ip,'datetime': '{}/{}/{} - {}:{}.{}'.format(d_time[0:4], d_time[5:6], d_time[6:8], d_time[8:10], d_time[10:12], d_time[12:14])})
return render_template("uuid_management.html", uuid_sensor=uuid_sensor, active_connection=active_connection,
uuid_key=uuid_key, data_uuid=data_uuid, max_uuid_stream=max_uuid_stream, all_ip=all_ip)
uuid_key=uuid_key, data_uuid=data_uuid, uuid_all_type=uuid_all_type_list,
max_uuid_stream=max_uuid_stream, all_ip=all_ip)
else:
return 'Invalid uuid'
@ -710,5 +734,38 @@ def get_analyser_sample():
else:
return jsonify('Incorrect UUID')
@app.route('/get_uuid_type_history_json')
def get_uuid_type_history_json():
uuid_sensor = request.args.get('uuid_sensor')
if is_valid_uuid_v4(uuid_sensor):
num_day_type = 7
date_range = get_substract_date_range(num_day_type)
type_history = []
range_decoder = []
all_type = set()
for date in date_range:
type_day = redis_server_metadata.zrange('stat_uuid_type:{}:{}'.format(date, uuid_sensor), 0, -1, withscores=True)
for type in type_day:
all_type.add(type[0])
range_decoder.append((date, type_day))
default_dict_type = {}
for type in all_type:
default_dict_type[type] = 0
for row in range_decoder:
day_type = default_dict_type.copy()
date = row[0]
day_type['date']= date[0:4] + '-' + date[4:6] + '-' + date[6:8]
for type in row[1]:
day_type[type[0]]= type[1]
type_history.append(day_type)
return jsonify(type_history)
else:
return jsonify('Incorrect UUID')
if __name__ == "__main__":
app.run(host='0.0.0.0', port=7000, threaded=True)

231
server/web/templates/uuid_management.html
View File

@ -7,10 +7,15 @@
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
<!-- JS -->
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/d3v5.min.js')}}"></script>
</head>
@ -136,8 +141,44 @@
</div>
</div>
<div>
<div class="card text-center mt-3 mx-3">
<div class="card-header bg-dark text-white">
Types Used:
</div>
<div class="row ml-0 mr-0">
<div class="col-lg-4">
<div class="mt-2">
<table class="table table-striped table-bordered table-hover" id="myTable_1">
<thead class="thead-dark">
<tr>
<th>Type</th>
<th style="max-width: 800px;">first seen</th>
<th style="max-width: 800px;">last seen</th>
</tr>
</thead>
<tbody>
{% for type in uuid_all_type %}
<tr>
<td>{{type['type']}}</td>
<td>{{type['first_seen']}}</td>
<td>{{type['last_seen']}}</td>
</tr>
{% endfor %}
</tbody>
</table>
</div>
</div>
<div class="col-lg-8">
<div id="barchart_type">
</div>
</div>
</div>
</div>
</div>
<div class="row ml-0 mr-0">
<div class="col-6">
<div class="col-lg-6">
<div class="card text-center mt-3">
<div class="card-header bg-dark text-white">
Last IP Used:
@ -151,7 +192,7 @@
</ul>
</div>
</div>
<div class="col-6">
<div class="col-lg-6">
<div class="d-none card mt-3 mb-3" id="whois_data">
<div class="card-header bg-dark text-center text-white">
Whois Info:
@ -166,13 +207,197 @@
</body>
<script>
var chart = {};
$(document).ready(function(){
table = $('#myTable_1').DataTable(
{
"aLengthMenu": [[5, 10, 15, 20, -1], [5, 10, 15, 20, "All"]],
"iDisplayLength": 10,
"order": [[ 0, "asc" ]]
}
);
chart.stackBarChart =barchart_type_stack("{{ url_for('get_uuid_type_history_json') }}?uuid_sensor={{uuid_sensor}}", 'id');
chart.onResize();
$(window).on("resize", function() {
chart.onResize();
});
$('[data-toggle="popover"]').popover({
placement: 'top',
container: 'body',
html : true,
});
});
function get_whois_data(ip){
$.getJSON( "{{url_for('whois_data')}}?ip="+ip, function( data ) {
console.log(data)
$( "#whois_data" ).removeClass( "d-none" );
$( "#whois_output" ).text(data);
});
}
</script>
<script>
var margin = {top: 20, right: 90, bottom: 55, left: 0},
width = parseInt(d3.select('#barchart_type').style('width'), 10);
width = 1000 - margin.left - margin.right,
height = 500 - margin.top - margin.bottom;
var x = d3.scaleBand().rangeRound([0, width]).padding(0.1);
var y = d3.scaleLinear().rangeRound([height, 0]);
var xAxis = d3.axisBottom(x);
var yAxis = d3.axisLeft(y);
var color = d3.scaleOrdinal(d3.schemeSet3);
var svg = d3.select("#barchart_type").append("svg")
.attr("id", "thesvg")
.attr("viewBox", "0 0 "+width+" 500")
.attr("width", width + margin.left + margin.right)
.attr("height", height + margin.top + margin.bottom)
.append("g")
.attr("transform", "translate(" + margin.left + "," + margin.top + ")");
function barchart_type_stack(url, id) {
d3.json(url)
.then(function(data){
var labelVar = 'date'; //A
var varNames = d3.keys(data[0])
.filter(function (key) { return key !== labelVar;}); //B
data.forEach(function (d) { //D
var y0 = 0;
d.mapping = varNames.map(function (name) {
return {
name: name,
label: d[labelVar],
y0: y0,
y1: y0 += +d[name]
};
});
d.total = d.mapping[d.mapping.length - 1].y1;
});
x.domain(data.map(function (d) { return (d.date); })); //E
y.domain([0, d3.max(data, function (d) { return d.total; })]);
svg.append("g")
.attr("class", "x axis")
.attr("transform", "translate(0," + height + ")")
.call(xAxis)
.selectAll("text")
.attr("class", "bar")
.on("click", function (d) { window.location.href = "#" })
.attr("transform", "rotate(-18)" )
//.attr("transform", "rotate(-40)" )
.style("text-anchor", "end");
svg.append("g")
.attr("class", "y axis")
.call(yAxis)
.append("text")
.attr("transform", "rotate(-90)")
.attr("y", 6)
.attr("dy", ".71em")
.style("text-anchor", "end");
var selection = svg.selectAll(".series")
.data(data)
.enter().append("g")
.attr("class", "series")
.attr("transform", function (d) { return "translate(" + x((d.date)) + ",0)"; });
selection.selectAll("rect")
.data(function (d) { return d.mapping; })
.enter().append("rect")
.attr("class", "bar_stack")
.attr("width", x.bandwidth())
.attr("y", function (d) { return y(d.y1); })
.attr("height", function (d) { return y(d.y0) - y(d.y1); })
.style("fill", function (d) { return color(d.name); })
.style("stroke", "grey")
.on("mouseover", function (d) { showPopover.call(this, d); })
.on("mouseout", function (d) { removePopovers(); })
.on("click", function(d){ window.location.href = "#" });
data.forEach(function(d) {
if(d.total != 0){
svg.append("text")
.attr("class", "bar")
.attr("dy", "-.35em")
.attr('x', x(d.date) + x.bandwidth()/2)
.attr('y', y(d.total))
.on("click", function () {window.location.href = "#" })
.style("text-anchor", "middle")
.text(d.total);
}
});
drawLegend(varNames);
});
}
function drawLegend (varNames) {
var legend = svg.selectAll(".legend")
.data(varNames.slice().reverse())
.enter().append("g")
.attr("class", "legend")
.attr("transform", function (d, i) { return "translate(0," + i * 20 + ")"; });
legend.append("rect")
.attr("x", 943)
.attr("width", 10)
.attr("height", 10)
.style("fill", color)
.style("stroke", "grey");
legend.append("text")
.attr("class", "svgText")
.attr("x", 941)
.attr("y", 6)
.attr("dy", ".35em")
.style("text-anchor", "end")
.text(function (d) { return d; });
}
function removePopovers () {
$('.popover').each(function() {
$(this).remove();
});
}
function showPopover (d) {
$(this).popover({
title: d.name,
placement: 'top',
container: 'body',
trigger: 'manual',
html : true,
content: function() {
return d.label +
"<br/>num: " + d3.format(",")(d.value ? d.value: d.y1 - d.y0); }
});
$(this).popover('show')
}
chart.onResize = function () {
var aspect = width / height, chart = $("#thesvg");
var targetWidth = chart.parent().width();
chart.attr("width", targetWidth);
chart.attr("height", targetWidth / 2);
}
window.chart = chart;
</script>

11
server/web/update_web.sh
View File

@ -5,6 +5,7 @@ set -e
BOOTSTRAP_VERSION='4.2.1'
FONT_AWESOME_VERSION='4.7.0'
D3_JS_VERSION='4.13.0'
D3_JS_VERSIONv5='5.9.2'
if [ ! -d static/css ]; then
mkdir static/css
@ -19,10 +20,14 @@ fi
rm -rf temp
mkdir temp
mkdir temp/d3v5/
wget https://github.com/twbs/bootstrap/releases/download/v${BOOTSTRAP_VERSION}/bootstrap-${BOOTSTRAP_VERSION}-dist.zip -O temp/bootstrap${BOOTSTRAP_VERSION}.zip
wget https://github.com/FortAwesome/Font-Awesome/archive/v${FONT_AWESOME_VERSION}.zip -O temp/FONT_AWESOME_${FONT_AWESOME_VERSION}.zip
wget https://github.com/d3/d3/releases/download/v${D3_JS_VERSION}/d3.zip -O temp/d3_${D3_JS_VERSION}.zip
wget https://github.com/d3/d3/releases/download/v${D3_JS_VERSIONv5}/d3.zip -O temp/d3v5/d3_${D3_JS_VERSIONv5}.zip
wget https://github.com/FezVrasta/popper.js/archive/v1.14.3.zip -O temp/popper.zip
# dateRangePicker
wget https://github.com/moment/moment/archive/2.22.2.zip -O temp/moment_2.22.2.zip
@ -32,6 +37,8 @@ wget https://github.com/longbill/jquery-date-range-picker/archive/v0.18.0.zip -O
unzip temp/bootstrap${BOOTSTRAP_VERSION}.zip -d temp/
unzip temp/FONT_AWESOME_${FONT_AWESOME_VERSION}.zip -d temp/
unzip temp/d3_${D3_JS_VERSION}.zip -d temp/
unzip temp/d3v5/d3_${D3_JS_VERSIONv5}.zip -d temp/d3v5/
unzip temp/popper.zip -d temp/
unzip temp/moment_2.22.2.zip -d temp/
unzip temp/daterangepicker_v0.18.0.zip -d temp/
@ -40,6 +47,8 @@ mv temp/bootstrap-${BOOTSTRAP_VERSION}-dist/js/bootstrap.min.js ./static/js/
mv temp/bootstrap-${BOOTSTRAP_VERSION}-dist/css/bootstrap.min.css ./static/css/
mv temp/bootstrap-${BOOTSTRAP_VERSION}-dist/css/bootstrap.min.css.map ./static/css/
mv temp/popper.js-1.14.3/dist/umd/popper.min.js ./static/js/
mv temp/popper.js-1.14.3/dist/umd/popper.min.js.map ./static/js/
mv temp/Font-Awesome-${FONT_AWESOME_VERSION} temp/font-awesome
@ -49,6 +58,8 @@ mv temp/font-awesome/ ./static/
mv temp/jquery-date-range-picker-0.18.0/dist/daterangepicker.min.css ./static/css/
mv temp/d3.min.js ./static/js/
cp temp/d3v5/d3.min.js ./static/js/d3v5.min.js
mv temp/moment-2.22.2/min/moment.min.js ./static/js/
mv temp/jquery-date-range-picker-0.18.0/dist/jquery.daterangepicker.min.js ./static/js/