mirror of https://github.com/CIRCL/AIL-framework
chg: [UI crawler] add splash dashboard
Terrtia
parent
2a1cd4a009
commit
4ecff21194
|
|
@ -222,7 +222,17 @@ def delete_auto_crawler(url):
|
|||
|
||||
@hiddenServices.route("/crawlers/", methods=['GET'])
|
||||
def dashboard():
|
||||
return render_template("Crawler_dashboard.html")
|
||||
crawler_metadata_onion = get_crawler_splash_status('onion')
|
||||
crawler_metadata_regular = get_crawler_splash_status('regular')
|
||||
|
||||
now = datetime.datetime.now()
|
||||
date = now.strftime("%Y%m%d")
|
||||
statDomains_onion = get_stats_last_crawled_domains('onion', date)
|
||||
statDomains_regular = get_stats_last_crawled_domains('regular', date)
|
||||
|
||||
return render_template("Crawler_dashboard.html", crawler_metadata_onion = crawler_metadata_onion,
|
||||
crawler_metadata_regular=crawler_metadata_regular,
|
||||
statDomains_onion=statDomains_onion, statDomains_regular=statDomains_regular)
|
||||
|
||||
@hiddenServices.route("/hiddenServices/2", methods=['GET'])
|
||||
def hiddenServices_page_test():
|
||||
|
|
@ -482,6 +492,21 @@ def remove_auto_crawler():
|
|||
delete_auto_crawler(url)
|
||||
return redirect(url_for('hiddenServices.auto_crawler', page=page))
|
||||
|
||||
@hiddenServices.route("/crawlers/crawler_dashboard_json", methods=['GET'])
|
||||
def crawler_dashboard_json():
|
||||
|
||||
crawler_metadata_onion = get_crawler_splash_status('onion')
|
||||
crawler_metadata_regular = get_crawler_splash_status('regular')
|
||||
|
||||
now = datetime.datetime.now()
|
||||
date = now.strftime("%Y%m%d")
|
||||
|
||||
statDomains_onion = get_stats_last_crawled_domains('onion', date)
|
||||
statDomains_regular = get_stats_last_crawled_domains('regular', date)
|
||||
|
||||
return jsonify({'statDomains_onion': statDomains_onion, 'statDomains_regular': statDomains_regular,
|
||||
'crawler_metadata_onion':crawler_metadata_onion, 'crawler_metadata_regular':crawler_metadata_regular})
|
||||
|
||||
# # TODO: refractor
|
||||
@hiddenServices.route("/hiddenServices/last_crawled_domains_with_stats_json", methods=['GET'])
|
||||
def last_crawled_domains_with_stats_json():
|
||||
|
|
|
|||
|
|
@ -26,17 +26,81 @@
|
|||
|
||||
<div class="col-12 col-lg-10" id="core_content">
|
||||
|
||||
<div class="row">
|
||||
<div class="col-xl-6">
|
||||
|
||||
<div class="card mt-1 mb-1">
|
||||
<div class="card-header text-white bg-dark">
|
||||
<h5><a class="text-info" href="{{ url_for('hiddenServices.Crawler_Splash_last_by_type')}}?type=onion"><i class="fas fa-user-secret"></i> Onions Crawlers</a></h5>
|
||||
<div class="row">
|
||||
<div class="col-6">
|
||||
<span class="badge badge-success" id="stat_onion_domain_up">{{ statDomains_onion['domains_up'] }}</span> UP
|
||||
<span class="badge badge-danger ml-md-3" id="stat_onion_domain_down">{{ statDomains_onion['domains_down'] }}</span> DOWN
|
||||
</div>
|
||||
<div class="col-6">
|
||||
<span class="badge badge-success" id="stat_onion_total">{{ statDomains_onion['total'] }}</span> Crawled
|
||||
<span class="badge badge-warning ml-md-3" id="stat_onion_queue">{{ statDomains_onion['domains_queue'] }}</span> Queue
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="card-body px-0 py-0 ">
|
||||
<table class="table">
|
||||
<tbody id="tbody_crawler_onion_info">
|
||||
{% for crawler in crawler_metadata_onion %}
|
||||
<tr>
|
||||
<td>
|
||||
<i class="fas fa-{%if crawler['status']%}check{%else%}times{%endif%}-circle" style="color:{%if crawler['status']%}Green{%else%}Red{%endif%};"></i> {{crawler['crawler_info']}}
|
||||
</td>
|
||||
<td>
|
||||
{{crawler['crawling_domain']}}
|
||||
</td>
|
||||
<td style="color:{%if crawler['status']%}Green{%else%}Red{%endif%};">
|
||||
{{crawler['status_info']}}
|
||||
</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div >
|
||||
<pre>
|
||||
--------------
|
||||
|
||||
|
||||
|
||||
--------------
|
||||
|
||||
</pre>
|
||||
</div>
|
||||
<div class="col-xl-6">
|
||||
<div class="card mt-1 mb-1">
|
||||
<div class="card-header text-white bg-dark">
|
||||
<h5><a class="text-info" href="{{ url_for('hiddenServices.Crawler_Splash_last_by_type')}}?type=regular"><i class="fab fa-html5"></i> Regular Crawlers</a></h5>
|
||||
<div class="row">
|
||||
<div class="col-6">
|
||||
<span class="badge badge-success" id="stat_regular_domain_up">{{ statDomains_regular['domains_up'] }}</span> UP
|
||||
<span class="badge badge-danger ml-md-3" id="stat_regular_domain_down">{{ statDomains_regular['domains_down'] }}</span> DOWN
|
||||
</div>
|
||||
<div class="col-6">
|
||||
<span class="badge badge-success" id="stat_regular_total">{{ statDomains_regular['total'] }}</span> Crawled
|
||||
<span class="badge badge-warning ml-md-3" id="stat_regular_queue">{{ statDomains_regular['domains_queue'] }}</span> Queue
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="card-body px-0 py-0 ">
|
||||
<table class="table">
|
||||
<tbody id="tbody_crawler_regular_info">
|
||||
{% for crawler in crawler_metadata_regular %}
|
||||
<tr>
|
||||
<td>
|
||||
<i class="fas fa-{%if crawler['status']%}check{%else%}times{%endif%}-circle" style="color:{%if crawler['status']%}Green{%else%}Red{%endif%};"></i> {{crawler['crawler_info']}}
|
||||
</td>
|
||||
<td>
|
||||
{{crawler['crawling_domain']}}
|
||||
</td>
|
||||
<td style="color:{%if crawler['status']%}Green{%else%}Red{%endif%};">
|
||||
{{crawler['status_info']}}
|
||||
</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
|
@ -47,8 +111,20 @@
|
|||
</body>
|
||||
|
||||
<script>
|
||||
var to_refresh = false
|
||||
$(document).ready(function(){
|
||||
$("#page-Crawler").addClass("active");
|
||||
$("#nav_dashboard").addClass("active");
|
||||
$( window ).focus(function() {
|
||||
to_refresh = true
|
||||
refresh_crawler_status();
|
||||
});
|
||||
$( window ).blur(function() {
|
||||
to_refresh = false
|
||||
});
|
||||
|
||||
to_refresh = true
|
||||
refresh_crawler_status();
|
||||
});
|
||||
|
||||
function toggle_sidebar(){
|
||||
|
|
@ -64,4 +140,83 @@ function toggle_sidebar(){
|
|||
$('#core_content').addClass('col-lg-10')
|
||||
}
|
||||
}
|
||||
|
||||
function refresh_crawler_status(){
|
||||
|
||||
$.getJSON("{{ url_for('hiddenServices.crawler_dashboard_json') }}",
|
||||
function(data) {
|
||||
|
||||
$('#stat_onion_domain_up').text(data.statDomains_onion['domains_up']);
|
||||
$('#stat_onion_domain_down').text(data.statDomains_onion['domains_down']);
|
||||
$('#stat_onion_total').text(data.statDomains_onion['total']);
|
||||
$('#stat_onion_queue').text(data.statDomains_onion['domains_queue']);
|
||||
|
||||
$('#stat_regular_domain_up').text(data.statDomains_regular['domains_up']);
|
||||
$('#stat_regular_domain_down').text(data.statDomains_regular['domains_down']);
|
||||
$('#stat_regular_total').text(data.statDomains_regular['total']);
|
||||
$('#stat_regular_queue').text(data.statDomains_regular['domains_queue']);
|
||||
|
||||
if(data.crawler_metadata_onion.length!=0){
|
||||
$("#tbody_crawler_onion_info").empty();
|
||||
var tableRef = document.getElementById('tbody_crawler_onion_info');
|
||||
for (var i = 0; i < data.crawler_metadata_onion.length; i++) {
|
||||
var crawler = data.crawler_metadata_onion[i];
|
||||
var newRow = tableRef.insertRow(tableRef.rows.length);
|
||||
var text_color;
|
||||
var icon;
|
||||
if(crawler['status']){
|
||||
text_color = 'Green';
|
||||
icon = 'check';
|
||||
} else {
|
||||
text_color = 'Red';
|
||||
icon = 'times';
|
||||
}
|
||||
|
||||
var newCell = newRow.insertCell(0);
|
||||
newCell.innerHTML = "<td><i class=\"fas fa-"+icon+"-circle\" style=\"color:"+text_color+";\"></i> "+crawler['crawler_info']+"</td>";
|
||||
|
||||
newCell = newRow.insertCell(1);
|
||||
newCell.innerHTML = "<td>"+crawler['crawling_domain']+"</td>";
|
||||
|
||||
newCell = newRow.insertCell(2);
|
||||
newCell.innerHTML = "<td><div style=\"color:"+text_color+";\">"+crawler['status_info']+"</div></td>";
|
||||
|
||||
//$("#panel_crawler").show();
|
||||
}
|
||||
}
|
||||
if(data.crawler_metadata_regular.length!=0){
|
||||
$("#tbody_crawler_regular_info").empty();
|
||||
var tableRef = document.getElementById('tbody_crawler_regular_info');
|
||||
for (var i = 0; i < data.crawler_metadata_regular.length; i++) {
|
||||
var crawler = data.crawler_metadata_regular[i];
|
||||
var newRow = tableRef.insertRow(tableRef.rows.length);
|
||||
var text_color;
|
||||
var icon;
|
||||
if(crawler['status']){
|
||||
text_color = 'Green';
|
||||
icon = 'check';
|
||||
} else {
|
||||
text_color = 'Red';
|
||||
icon = 'times';
|
||||
}
|
||||
|
||||
var newCell = newRow.insertCell(0);
|
||||
newCell.innerHTML = "<td><i class=\"fas fa-"+icon+"-circle\" style=\"color:"+text_color+";\"></i> "+crawler['crawler_info']+"</td>";
|
||||
|
||||
newCell = newRow.insertCell(1);
|
||||
newCell.innerHTML = "<td>"+crawler['crawling_domain']+"</td>";
|
||||
|
||||
newCell = newRow.insertCell(2);
|
||||
newCell.innerHTML = "<td><div style=\"color:"+text_color+";\">"+crawler['status_info']+"</div></td>";
|
||||
|
||||
//$("#panel_crawler").show();
|
||||
}
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
if (to_refresh) {
|
||||
setTimeout("refresh_crawler_status()", 10000);
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
|
|
|||
Loading…
Reference in New Issue