mirror of https://github.com/CIRCL/AIL-framework
chg: [hiddenServices] auto update last crawled domains
Terrtia
parent
b3b75ccbea
commit
46c1105483
|
|
@ -102,7 +102,40 @@ def hiddenServices_page():
|
|||
metadata_onion['status_icon'] = 'fa-times-circle'
|
||||
list_onion.append(metadata_onion)
|
||||
|
||||
return render_template("hiddenServices.html", last_onions=list_onion, statDomains=statDomains)
|
||||
date_string = '{}-{}-{}'.format(date[0:4], date[4:6], date[6:8])
|
||||
return render_template("hiddenServices.html", last_onions=list_onion, statDomains=statDomains, date_from=date_string, date_to=date_string)
|
||||
|
||||
@hiddenServices.route("/hiddenServices/last_crawled_domains_with_stats_jon", methods=['GET'])
|
||||
def last_crawled_domains_with_stats_jon():
|
||||
last_onions = r_serv_onion.lrange('last_onion', 0 ,-1)
|
||||
list_onion = []
|
||||
|
||||
now = datetime.datetime.now()
|
||||
date = '{}{}{}'.format(now.strftime("%Y"), now.strftime("%m"), now.strftime("%d"))
|
||||
statDomains = {}
|
||||
statDomains['domains_up'] = r_serv_onion.scard('onion_up:{}'.format(date))
|
||||
statDomains['domains_down'] = r_serv_onion.scard('onion_down:{}'.format(date))
|
||||
statDomains['total'] = statDomains['domains_up'] + statDomains['domains_down']
|
||||
statDomains['domains_queue'] = r_serv_onion.scard('onion_domain_crawler_queue')
|
||||
|
||||
for onion in last_onions:
|
||||
metadata_onion = {}
|
||||
metadata_onion['domain'] = onion
|
||||
metadata_onion['last_check'] = r_serv_onion.hget('onion_metadata:{}'.format(onion), 'last_check')
|
||||
metadata_onion['first_seen'] = r_serv_onion.hget('onion_metadata:{}'.format(onion), 'first_seen')
|
||||
if get_onion_status(onion, metadata_onion['last_check']):
|
||||
metadata_onion['status_text'] = 'UP'
|
||||
metadata_onion['status_color'] = 'Green'
|
||||
metadata_onion['status_icon'] = 'fa-check-circle'
|
||||
else:
|
||||
metadata_onion['status_text'] = 'DOWN'
|
||||
metadata_onion['status_color'] = 'Red'
|
||||
metadata_onion['status_icon'] = 'fa-times-circle'
|
||||
list_onion.append(metadata_onion)
|
||||
|
||||
date_string = '{}-{}-{}'.format(date[0:4], date[4:6], date[6:8])
|
||||
|
||||
return jsonify({'last_onions': list_onion, 'statDomains': statDomains})
|
||||
|
||||
@hiddenServices.route("/hiddenServices/get_onions_by_daterange", methods=['POST'])
|
||||
def get_onions_by_daterange():
|
||||
|
|
|
|||
|
|
@ -66,7 +66,7 @@
|
|||
<th>Status</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tbody id="tbody_last_crawled">
|
||||
{% for metadata_onion in last_onions %}
|
||||
<tr>
|
||||
<td><a target="_blank" href="{{ url_for('hiddenServices.onion_domain') }}?onion_domain={{ metadata_onion['domain'] }}">{{ metadata_onion['domain'] }}</a></td>
|
||||
|
|
@ -152,37 +152,53 @@
|
|||
<tbody>
|
||||
<tr>
|
||||
<td>
|
||||
<div style="color:Green; display:inline-block">
|
||||
<i class="fa fa-check-circle fa-2x"></i>
|
||||
Domains UP
|
||||
</div>
|
||||
<a target="_blank" href="{{ url_for('hiddenServices.show_domains_by_daterange') }}?date_from={{ date_from }}&domains_up=True">
|
||||
<div style="color:Green; display:inline-block">
|
||||
<i class="fa fa-check-circle fa-2x"></i>
|
||||
Domains UP
|
||||
</div>
|
||||
</a>
|
||||
</td>
|
||||
<td>
|
||||
<div style="color:Green; display:inline-block">
|
||||
{{ statDomains['domains_up'] }}
|
||||
</div>
|
||||
<a target="_blank" href="{{ url_for('hiddenServices.show_domains_by_daterange') }}?date_from={{ date_from }}&domains_up=True">
|
||||
<div style="color:Green; display:inline-block">
|
||||
<div id="text_domain_up">{{ statDomains['domains_up'] }}</div>
|
||||
</div>
|
||||
</a>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
<div style="color:Red; display:inline-block">
|
||||
<i class="fa fa-times-circle fa-2x"></i>
|
||||
Domains DOWN
|
||||
</div>
|
||||
<a target="_blank" href="{{ url_for('hiddenServices.show_domains_by_daterange') }}?date_from={{ date_from }}&domains_down=True">
|
||||
<div style="color:Red; display:inline-block">
|
||||
<i class="fa fa-times-circle fa-2x"></i>
|
||||
Domains DOWN
|
||||
</div>
|
||||
</a>
|
||||
</td>
|
||||
<td>
|
||||
<div style="color:Red; display:inline-block">
|
||||
{{ statDomains['domains_down'] }}
|
||||
</div>
|
||||
<a target="_blank" href="{{ url_for('hiddenServices.show_domains_by_daterange') }}?date_from={{ date_from }}&domains_down=True">
|
||||
<div style="color:Red; display:inline-block">
|
||||
<div id="text_domain_down">{{ statDomains['domains_down'] }}</div>
|
||||
</div>
|
||||
</a>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Crawled Domains</td>
|
||||
<td>{{ statDomains['total'] }}</td>
|
||||
<td>
|
||||
<a target="_blank" href="{{ url_for('hiddenServices.show_domains_by_daterange') }}?date_from={{ date_from }}&domains_up=True&domains_down=True">
|
||||
Crawled Domains
|
||||
</a>
|
||||
</td>
|
||||
<td>
|
||||
<a target="_blank" href="{{ url_for('hiddenServices.show_domains_by_daterange') }}?date_from={{ date_from }}&domains_up=True&domains_down=True">
|
||||
<div id="text_total_domains">{{ statDomains['total'] }}</div>
|
||||
</a>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Domains in Queue</td>
|
||||
<td>{{ statDomains['domains_queue'] }}</td>
|
||||
<td><div id="text_domain_queue">{{ statDomains['domains_queue'] }}</div></td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
|
|
@ -196,6 +212,7 @@
|
|||
|
||||
<script>
|
||||
var all_graph = {};
|
||||
var to_refresh = false
|
||||
$(document).ready(function(){
|
||||
activePage = "page-hiddenServices"
|
||||
$("#"+activePage).addClass("active");
|
||||
|
|
@ -232,10 +249,22 @@
|
|||
});
|
||||
|
||||
all_graph.line_chart = create_line_chart('graph_line', "{{ url_for('hiddenServices.domain_crawled_7days_json') }}?type=onion");
|
||||
});
|
||||
$(window).on("resize", function() {
|
||||
all_graph.onResize();
|
||||
|
||||
$( window ).focus(function() {
|
||||
to_refresh = true
|
||||
refresh_list_crawled();
|
||||
});
|
||||
$( window ).blur(function() {
|
||||
to_refresh = false
|
||||
});
|
||||
|
||||
to_refresh = true
|
||||
refresh_list_crawled();
|
||||
});
|
||||
$(window).on("resize", function() {
|
||||
all_graph.onResize();
|
||||
});
|
||||
|
||||
</script>
|
||||
|
||||
<script>
|
||||
|
|
@ -319,6 +348,49 @@ d3.json(url)
|
|||
});
|
||||
});
|
||||
}
|
||||
</script>
|
||||
|
||||
<script>
|
||||
|
||||
function refresh_list_crawled(){
|
||||
|
||||
$.getJSON("{{ url_for('hiddenServices.last_crawled_domains_with_stats_jon') }}",
|
||||
function(data) {
|
||||
|
||||
var tableRef = document.getElementById('tbody_last_crawled');
|
||||
$("#tbody_last_crawled").empty()
|
||||
|
||||
for (var i = 0; i < data.last_onions.length; i++) {
|
||||
var data_domain = data.last_onions[i]
|
||||
var newRow = tableRef.insertRow(tableRef.rows.length);
|
||||
|
||||
var newCell = newRow.insertCell(0);
|
||||
newCell.innerHTML = "<td><a target=\"_blank\" href=\"{{ url_for('hiddenServices.onion_domain') }}?onion_domain="+data_domain['domain']+"\">"+data_domain['domain']+"</a></td>";
|
||||
|
||||
newCell = newRow.insertCell(1);
|
||||
newCell.innerHTML = "<td>"+data_domain['first_seen'].substr(0, 4)+"/"+data_domain['first_seen'].substr(4, 2)+"/"+data_domain['first_seen'].substr(6, 2)+"</td>"
|
||||
|
||||
newCell = newRow.insertCell(2);
|
||||
newCell.innerHTML = "<td>"+data_domain['last_check'].substr(0, 4)+"/"+data_domain['last_check'].substr(4, 2)+"/"+data_domain['last_check'].substr(6, 2)+"</td>"
|
||||
|
||||
newCell = newRow.insertCell(3);
|
||||
newCell.innerHTML = "<td><div style=\"color:"+data_domain['status_color']+"; display:inline-block\"><i class=\"fa "+data_domain['status_icon']+" fa-2x\"></i>"+data_domain['status_text']+"</div></td>"
|
||||
|
||||
}
|
||||
var statDomains = data.statDomains
|
||||
document.getElementById('text_domain_up').innerHTML = statDomains['domains_up']
|
||||
document.getElementById('text_domain_down').innerHTML = statDomains['domains_down']
|
||||
document.getElementById('text_domain_queue').innerHTML = statDomains['domains_queue']
|
||||
document.getElementById('text_total_domains').innerHTML = statDomains['total']
|
||||
}
|
||||
);
|
||||
|
||||
if (to_refresh) {
|
||||
setTimeout("refresh_list_crawled()", 10000);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
</script>
|
||||
</body>
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue