CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

chg: [crawler] show all crawlers type on dashboard

pull/559/head
Terrtia 2020-08-17 21:52:57 +02:00
parent 8901ffe989
commit 3ea14b29b8
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
9 changed files with 371 additions and 306 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
bin/Crawler.py
View File

@ -349,7 +349,7 @@ if __name__ == '__main__':
'user_agent': p.config.get("Crawler", "default_crawler_user_agent")}
# Track launched crawler
r_cache.sadd('all_crawler', splash_url)
r_cache.sadd('all_splash_crawlers', splash_url)
r_cache.hset('metadata_crawler:{}'.format(splash_url), 'status', 'Waiting')
r_cache.hset('metadata_crawler:{}'.format(splash_url), 'started_time', datetime.datetime.now().strftime("%Y/%m/%d - %H:%M.%S"))
@ -385,7 +385,7 @@ if __name__ == '__main__':
'epoch': int(time.time())}
# Update crawler status type
r_cache.sadd('{}_crawlers'.format(to_crawl['type_service']), splash_url)
r_cache.hset('metadata_crawler:{}'.format(splash_url), 'type', to_crawl['type_service'])
crawler_config = load_crawler_config(to_crawl['type_service'], url_data['domain'], to_crawl['paste'], to_crawl['url'], date)
# check if default crawler
@ -437,7 +437,7 @@ if __name__ == '__main__':
r_cache.hdel('metadata_crawler:{}'.format(splash_url), 'crawling_domain')
# Update crawler status type
r_cache.srem('{}_crawlers'.format(to_crawl['type_service']), splash_url)
r_cache.hdel('metadata_crawler:{}'.format(splash_url), 'type', to_crawl['type_service'])
# add next auto Crawling in queue:
if to_crawl['paste'] == 'auto':

4
bin/core/Crawler_manager.py
View File

@ -35,6 +35,8 @@ def launch_crawlers():
print('Please launch {} additional {} Dockers'.format( nb_crawlers - len(all_crawler_urls), splash_name))
nb_crawlers = len(all_crawler_urls)
crawlers.reset_all_spash_crawler_status()
for i in range(0, int(nb_crawlers)):
splash_url = all_crawler_urls[i]
print(all_crawler_urls[i])
@ -59,6 +61,8 @@ if __name__ == '__main__':
while True:
# # TODO: avoid multiple ping
# check if manager is connected
if int(time.time()) - last_check > 60:
is_manager_connected = crawlers.is_splash_manager_connected()

74
bin/lib/crawlers.py
View File

@ -46,9 +46,20 @@ config_loader = None
faup = Faup()
# # # # # # # #
# #
# COMMON #
# #
# # # # # # # #
def generate_uuid():
return str(uuid.uuid4()).replace('-', '')
def get_current_date():
return datetime.now().strftime("%Y%m%d")
##-- COMMON --#
################################################################################
# # TODO: handle prefix cookies
@ -377,6 +388,55 @@ def api_create_cookie(user_id, cookiejar_uuid, cookie_dict):
#### ####
# # # # # # # #
# #
# CRAWLER #
# #
# # # # # # # #
#### CRAWLER GLOBAL ####
def get_all_spash_crawler_status():
crawler_metadata = []
all_crawlers = r_cache.smembers('all_splash_crawlers')
for crawler in all_crawlers:
crawler_metadata.append(get_splash_crawler_status(crawler))
return crawler_metadata
def reset_all_spash_crawler_status():
r_cache.delete('all_splash_crawlers')
def get_splash_crawler_status(spash_url):
crawler_type = r_cache.hget('metadata_crawler:{}'.format(spash_url), 'type')
crawling_domain = r_cache.hget('metadata_crawler:{}'.format(spash_url), 'crawling_domain')
started_time = r_cache.hget('metadata_crawler:{}'.format(spash_url), 'started_time')
status_info = r_cache.hget('metadata_crawler:{}'.format(spash_url), 'status')
crawler_info = '{} - {}'.format(spash_url, started_time)
if status_info=='Waiting' or status_info=='Crawling':
status=True
else:
status=False
return {'crawler_info': crawler_info, 'crawling_domain': crawling_domain, 'status_info': status_info, 'status': status, 'type': crawler_type}
def get_stats_last_crawled_domains(crawler_types, date):
statDomains = {}
for crawler_type in crawler_types:
stat_type = {}
stat_type['domains_up'] = r_serv_onion.scard('{}_up:{}'.format(crawler_type, date))
stat_type['domains_down'] = r_serv_onion.scard('{}_down:{}'.format(crawler_type, date))
stat_type['total'] = stat_type['domains_up'] + stat_type['domains_down']
stat_type['domains_queue'] = get_nb_elem_to_crawl_by_type(crawler_type)
statDomains[crawler_type] = stat_type
return statDomains
# # TODO: handle custom proxy
def get_splash_crawler_latest_stats():
now = datetime.now()
date = now.strftime("%Y%m%d")
return get_stats_last_crawled_domains(['onion', 'regular'], date)
##-- CRAWLER GLOBAL --##
#### CRAWLER TASK ####
def create_crawler_task(url, screenshot=True, har=True, depth_limit=1, max_pages=100, auto_crawler=False, crawler_delta=3600, cookiejar_uuid=None, user_agent=None):
@ -587,10 +647,20 @@ def get_elem_to_crawl_by_queue_type(l_queue_type):
return {'url': url, 'paste': item_id, 'type_service': queue_type, 'original_message': message}
return None
def get_nb_elem_to_crawl_by_type(queue_type):
nb = r_serv_onion.scard('{}_crawler_priority_queue'.format(queue_type))
nb += r_serv_onion.scard('{}_crawler_discovery_queue'.format(queue_type))
nb += r_serv_onion.scard('{}_crawler_queue'.format(queue_type))
return nb
#### ---- ####
# # # # # # # # # # # #
# #
# SPLASH MANAGER #
# #
# # # # # # # # # # # #
#### SPLASH MANAGER ####
def get_splash_manager_url(reload=False): # TODO: add in db config
return splash_manager_url
@ -636,6 +706,8 @@ def ping_splash_manager():
return True
else:
print(req.json())
update_splash_manager_connection_status(False)
return False
except requests.exceptions.ConnectionError:
pass
# splash manager unreachable

32
var/www/blueprints/crawler_splash.py
View File

@ -48,6 +48,31 @@ def create_json_response(data, status_code):
return Response(json.dumps(data, indent=2, sort_keys=True), mimetype='application/json'), status_code
# ============= ROUTES ==============
@crawler_splash.route("/crawlers/dashboard", methods=['GET'])
@login_required
@login_read_only
def crawlers_dashboard():
# # TODO: get splash manager status
crawler_enabled = crawlers.ping_splash_manager()
all_splash_crawler_status = crawlers.get_all_spash_crawler_status()
splash_crawlers_latest_stats = crawlers.get_splash_crawler_latest_stats()
date = crawlers.get_current_date()
return render_template("dashboard_splash_crawler.html", all_splash_crawler_status = all_splash_crawler_status,
crawler_enabled=crawler_enabled, date=date,
splash_crawlers_latest_stats=splash_crawlers_latest_stats)
@crawler_splash.route("/crawlers/crawler_dashboard_json", methods=['GET'])
@login_required
@login_read_only
def crawler_dashboard_json():
all_splash_crawler_status = crawlers.get_all_spash_crawler_status()
splash_crawlers_latest_stats = crawlers.get_splash_crawler_latest_stats()
return jsonify({'all_splash_crawler_status': all_splash_crawler_status,
'splash_crawlers_latest_stats':splash_crawlers_latest_stats})
@crawler_splash.route("/crawlers/manual", methods=['GET'])
@login_required
@login_read_only
@ -403,4 +428,11 @@ def crawler_cookiejar_cookie_json_add_post():
return redirect(url_for('crawler_splash.crawler_cookiejar_cookie_add', cookiejar_uuid=cookiejar_uuid))
@crawler_splash.route('/crawler/cookiejar/cookie/json_add_post', methods=['GET'])
@login_required
@login_analyst
def crawler_splash_setings():
return render_template("settings_splash_crawler.html", cookiejar_uuid=True, cookie_uuid=False)
## - - ##

49
var/www/modules/hiddenServices/Flask_hiddenServices.py
View File

@ -231,22 +231,22 @@ def delete_auto_crawler(url):
# ============= ROUTES ==============
@hiddenServices.route("/crawlers/", methods=['GET'])
@login_required
@login_read_only
def dashboard():
crawler_metadata_onion = get_crawler_splash_status('onion')
crawler_metadata_regular = get_crawler_splash_status('regular')
now = datetime.datetime.now()
date = now.strftime("%Y%m%d")
statDomains_onion = get_stats_last_crawled_domains('onion', date)
statDomains_regular = get_stats_last_crawled_domains('regular', date)
return render_template("Crawler_dashboard.html", crawler_metadata_onion = crawler_metadata_onion,
crawler_enabled=crawler_enabled, date=date,
crawler_metadata_regular=crawler_metadata_regular,
statDomains_onion=statDomains_onion, statDomains_regular=statDomains_regular)
# @hiddenServices.route("/crawlers/", methods=['GET'])
# @login_required
# @login_read_only
# def dashboard():
# crawler_metadata_onion = get_crawler_splash_status('onion')
# crawler_metadata_regular = get_crawler_splash_status('regular')
#
# now = datetime.datetime.now()
# date = now.strftime("%Y%m%d")
# statDomains_onion = get_stats_last_crawled_domains('onion', date)
# statDomains_regular = get_stats_last_crawled_domains('regular', date)
#
# return render_template("Crawler_dashboard.html", crawler_metadata_onion = crawler_metadata_onion,
# crawler_enabled=crawler_enabled, date=date,
# crawler_metadata_regular=crawler_metadata_regular,
# statDomains_onion=statDomains_onion, statDomains_regular=statDomains_regular)
@hiddenServices.route("/crawlers/crawler_splash_onion", methods=['GET'])
@login_required
@ -439,23 +439,6 @@ def remove_auto_crawler():
delete_auto_crawler(url)
return redirect(url_for('hiddenServices.auto_crawler', page=page))
@hiddenServices.route("/crawlers/crawler_dashboard_json", methods=['GET'])
@login_required
@login_read_only
def crawler_dashboard_json():
crawler_metadata_onion = get_crawler_splash_status('onion')
crawler_metadata_regular = get_crawler_splash_status('regular')
now = datetime.datetime.now()
date = now.strftime("%Y%m%d")
statDomains_onion = get_stats_last_crawled_domains('onion', date)
statDomains_regular = get_stats_last_crawled_domains('regular', date)
return jsonify({'statDomains_onion': statDomains_onion, 'statDomains_regular': statDomains_regular,
'crawler_metadata_onion':crawler_metadata_onion, 'crawler_metadata_regular':crawler_metadata_regular})
# # TODO: refractor
@hiddenServices.route("/hiddenServices/last_crawled_domains_with_stats_json", methods=['GET'])
@login_required

267
var/www/modules/hiddenServices/templates/Crawler_dashboard.html
View File

@ -1,267 +0,0 @@
<!DOCTYPE html>
<html>
<head>
<title>AIL-Framework</title>
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png')}}">
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
<!-- JS -->
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
</head>
<body>
{% include 'nav_bar.html' %}
<div class="container-fluid">
<div class="row">
{% include 'crawler/menu_sidebar.html' %}
<div class="col-12 col-lg-10" id="core_content">
{% include 'crawler/crawler_disabled.html' %}
<div class="row">
<div class="col-xl-6">
<div class="card mt-1 mb-1">
<div class="card-header text-white bg-dark">
<h5><a class="text-info" href="{{ url_for('hiddenServices.Crawler_Splash_last_by_type')}}?type=onion"><i class="fas fa-user-secret"></i> Onions Crawlers</a></h5>
<div class="row">
<div class="col-6">
<a href="{{ url_for('hiddenServices.show_domains_by_daterange') }}?service_type=onion&domains_up=True&date_from={{date}}&date_to={{date}}" class="badge badge-success" id="stat_onion_domain_up">{{ statDomains_onion['domains_up'] }}</a> UP
<a href="{{ url_for('hiddenServices.show_domains_by_daterange') }}?service_type=onion&domains_down=True&date_from={{date}}&date_to={{date}}" class="badge badge-danger ml-md-3" id="stat_onion_domain_down">{{ statDomains_onion['domains_down'] }}</a> DOWN
</div>
<div class="col-6">
<a href="{{ url_for('hiddenServices.show_domains_by_daterange') }}?service_type=onion&domains_up=True&domains_down=True&date_from={{date}}&date_to={{date}}" class="badge badge-success" id="stat_onion_total">{{ statDomains_onion['total'] }}</a> Crawled
<span class="badge badge-warning ml-md-3" id="stat_onion_queue">{{ statDomains_onion['domains_queue'] }}</span> Queue
</div>
</div>
</div>
<div class="card-body px-0 py-0 ">
<table class="table">
<tbody id="tbody_crawler_onion_info">
{% for crawler in crawler_metadata_onion %}
<tr>
<td>
<i class="fas fa-{%if crawler['status']%}check{%else%}times{%endif%}-circle" style="color:{%if crawler['status']%}Green{%else%}Red{%endif%};"></i> {{crawler['crawler_info']}}
</td>
<td>
{{crawler['crawling_domain']}}
</td>
<td style="color:{%if crawler['status']%}Green{%else%}Red{%endif%};">
{{crawler['status_info']}}
</td>
</tr>
{% endfor %}
</tbody>
</table>
</div>
</div>
</div>
<div class="col-xl-6">
<div class="card mt-1 mb-1">
<div class="card-header text-white bg-dark">
<h5><a class="text-info" href="{{ url_for('hiddenServices.Crawler_Splash_last_by_type')}}?type=regular"><i class="fab fa-html5"></i> Regular Crawlers</a></h5>
<div class="row">
<div class="col-6">
<a href="{{ url_for('hiddenServices.show_domains_by_daterange') }}?service_type=regular&domains_up=True&date_from={{date}}&date_to={{date}}" class="badge badge-success" id="stat_regular_domain_up">{{ statDomains_regular['domains_up'] }}</a> UP
<a href="{{ url_for('hiddenServices.show_domains_by_daterange') }}?service_type=regular&domains_down=True&date_from={{date}}&date_to={{date}}" class="badge badge-danger ml-md-3" id="stat_regular_domain_down">{{ statDomains_regular['domains_down'] }}</a> DOWN
</div>
<div class="col-6">
<a href="{{ url_for('hiddenServices.show_domains_by_daterange') }}?service_type=regular&domains_up=True&domains_down=True&date_from={{date}}&date_to={{date}}" class="badge badge-success" id="stat_regular_total">{{ statDomains_regular['total'] }}</a> Crawled
<span class="badge badge-warning ml-md-3" id="stat_regular_queue">{{ statDomains_regular['domains_queue'] }}</span> Queue
</div>
</div>
</div>
<div class="card-body px-0 py-0 ">
<table class="table">
<tbody id="tbody_crawler_regular_info">
{% for crawler in crawler_metadata_regular %}
<tr>
<td>
<i class="fas fa-{%if crawler['status']%}check{%else%}times{%endif%}-circle" style="color:{%if crawler['status']%}Green{%else%}Red{%endif%};"></i> {{crawler['crawler_info']}}
</td>
<td>
{{crawler['crawling_domain']}}
</td>
<td style="color:{%if crawler['status']%}Green{%else%}Red{%endif%};">
{{crawler['status_info']}}
</td>
</tr>
{% endfor %}
</tbody>
</table>
</div>
</div>
</div>
</div>
<div class="d-flex justify-content-center my-4">
<div class="card border-secondary" style="max-width: 40rem;">
<div class="card-body text-dark">
<h5 class="card-title">Show Domain:</h5>
<form class="" action="{{url_for('crawler_splash.showDomain')}}" method="post">
<div class="input-group mb-3">
<input type="text" class="form-control" style="min-width: 30rem;" placeholder="Domain name" aria-label="Domain name" aria-describedby="btn_show_domain" id="in_show_domain" , name="in_show_domain">
<div class="input-group-append">
<button class="btn btn-info" type="submit" id="btn_show_domain">
<i class="fas fa-search"></i>
</button>
</div>
</div>
</form>
</div>
</div>
</div>
<hr>
<div class="row mb-3">
<div class="col-xl-6">
<div class="text-center">
<a class="btn btn-secondary" href="{{url_for('crawler_splash.domains_explorer_onion')}}" role="button">
<i class="fas fa-user-secret"></i> Onion Domain Explorer
</a>
</div>
</div>
<div class="col-xl-6">
<div class="text-center">
<a class="btn btn-secondary" href="{{url_for('crawler_splash.domains_explorer_web')}}" role="button">
<i class="fab fa-html5"></i> Web Domain Explorer
</a>
</div>
</div>
</div>
{% with object_type='domain' %}
{% include 'tags/block_obj_tags_search.html' %}
{% endwith %}
</div>
</div>
</div>
</body>
<script>
var to_refresh = false
$(document).ready(function(){
$("#page-Crawler").addClass("active");
$("#nav_dashboard").addClass("active");
$( window ).on("focus", function() {
to_refresh = true
refresh_crawler_status();
});
$( window ).on("blur", function() {
to_refresh = false
});
to_refresh = true
refresh_crawler_status();
});
function toggle_sidebar(){
if($('#nav_menu').is(':visible')){
$('#nav_menu').hide();
$('#side_menu').removeClass('border-right')
$('#side_menu').removeClass('col-lg-2')
$('#core_content').removeClass('col-lg-10')
}else{
$('#nav_menu').show();
$('#side_menu').addClass('border-right')
$('#side_menu').addClass('col-lg-2')
$('#core_content').addClass('col-lg-10')
}
}
function refresh_crawler_status(){
$.getJSON("{{ url_for('hiddenServices.crawler_dashboard_json') }}",
function(data) {
$('#stat_onion_domain_up').text(data.statDomains_onion['domains_up']);
$('#stat_onion_domain_down').text(data.statDomains_onion['domains_down']);
$('#stat_onion_total').text(data.statDomains_onion['total']);
$('#stat_onion_queue').text(data.statDomains_onion['domains_queue']);
$('#stat_regular_domain_up').text(data.statDomains_regular['domains_up']);
$('#stat_regular_domain_down').text(data.statDomains_regular['domains_down']);
$('#stat_regular_total').text(data.statDomains_regular['total']);
$('#stat_regular_queue').text(data.statDomains_regular['domains_queue']);
if(data.crawler_metadata_onion.length!=0){
$("#tbody_crawler_onion_info").empty();
var tableRef = document.getElementById('tbody_crawler_onion_info');
for (var i = 0; i < data.crawler_metadata_onion.length; i++) {
var crawler = data.crawler_metadata_onion[i];
var newRow = tableRef.insertRow(tableRef.rows.length);
var text_color;
var icon;
if(crawler['status']){
text_color = 'Green';
icon = 'check';
} else {
text_color = 'Red';
icon = 'times';
}
var newCell = newRow.insertCell(0);
newCell.innerHTML = "<td><i class=\"fas fa-"+icon+"-circle\" style=\"color:"+text_color+";\"></i> "+crawler['crawler_info']+"</td>";
newCell = newRow.insertCell(1);
newCell.innerHTML = "<td>"+crawler['crawling_domain']+"</td>";
newCell = newRow.insertCell(2);
newCell.innerHTML = "<td><div style=\"color:"+text_color+";\">"+crawler['status_info']+"</div></td>";
//$("#panel_crawler").show();
}
}
if(data.crawler_metadata_regular.length!=0){
$("#tbody_crawler_regular_info").empty();
var tableRef = document.getElementById('tbody_crawler_regular_info');
for (var i = 0; i < data.crawler_metadata_regular.length; i++) {
var crawler = data.crawler_metadata_regular[i];
var newRow = tableRef.insertRow(tableRef.rows.length);
var text_color;
var icon;
if(crawler['status']){
text_color = 'Green';
icon = 'check';
} else {
text_color = 'Red';
icon = 'times';
}
var newCell = newRow.insertCell(0);
newCell.innerHTML = "<td><i class=\"fas fa-"+icon+"-circle\" style=\"color:"+text_color+";\"></i> "+crawler['crawler_info']+"</td>";
newCell = newRow.insertCell(1);
newCell.innerHTML = "<td>"+crawler['crawling_domain']+"</td>";
newCell = newRow.insertCell(2);
newCell.innerHTML = "<td><div style=\"color:"+text_color+";\">"+crawler['status_info']+"</div></td>";
//$("#panel_crawler").show();
}
}
}
);
if (to_refresh) {
setTimeout("refresh_crawler_status()", 10000);
}
}
</script>

235
var/www/templates/crawler/crawler_splash/dashboard_splash_crawler.html Normal file
View File

@ -0,0 +1,235 @@
<!DOCTYPE html>
<html>
<head>
<title>AIL-Framework</title>
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png')}}">
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
<!-- JS -->
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
</head>
<body>
{% include 'nav_bar.html' %}
<div class="container-fluid">
<div class="row">
{% include 'crawler/menu_sidebar.html' %}
<div class="col-12 col-lg-10" id="core_content">
{% include 'crawler/crawler_disabled.html' %}
<div class="row">
<div class="col-xl-6">
<div class="card mt-1 mb-1">
<div class="card-header text-white bg-dark">
<h5><a class="text-info" href="{{ url_for('hiddenServices.Crawler_Splash_last_by_type')}}?type=onion"><i class="fas fa-user-secret"></i> Onions Crawlers</a></h5>
<div class="row">
<div class="col-6">
<a href="{{ url_for('hiddenServices.show_domains_by_daterange') }}?service_type=onion&domains_up=True&date_from={{date}}&date_to={{date}}" class="badge badge-success" id="stat_onion_domain_up">{{ splash_crawlers_latest_stats['onion']['domains_up'] }}</a> UP
<a href="{{ url_for('hiddenServices.show_domains_by_daterange') }}?service_type=onion&domains_down=True&date_from={{date}}&date_to={{date}}" class="badge badge-danger ml-md-3" id="stat_onion_domain_down">{{ splash_crawlers_latest_stats['onion']['domains_down'] }}</a> DOWN
</div>
<div class="col-6">
<a href="{{ url_for('hiddenServices.show_domains_by_daterange') }}?service_type=onion&domains_up=True&domains_down=True&date_from={{date}}&date_to={{date}}" class="badge badge-success" id="stat_onion_total">{{ splash_crawlers_latest_stats['onion']['total'] }}</a> Crawled
<span class="badge badge-warning ml-md-3" id="stat_onion_queue">{{ splash_crawlers_latest_stats['onion']['domains_queue'] }}</span> Queue
</div>
</div>
</div>
</div>
</div>
<div class="col-xl-6">
<div class="card mt-1 mb-1">
<div class="card-header text-white bg-dark">
<h5><a class="text-info" href="{{ url_for('hiddenServices.Crawler_Splash_last_by_type')}}?type=regular"><i class="fab fa-html5"></i> Regular Crawlers</a></h5>
<div class="row">
<div class="col-6">
<a href="{{ url_for('hiddenServices.show_domains_by_daterange') }}?service_type=regular&domains_up=True&date_from={{date}}&date_to={{date}}" class="badge badge-success" id="stat_regular_domain_up">{{ splash_crawlers_latest_stats['regular']['domains_up'] }}</a> UP
<a href="{{ url_for('hiddenServices.show_domains_by_daterange') }}?service_type=regular&domains_down=True&date_from={{date}}&date_to={{date}}" class="badge badge-danger ml-md-3" id="stat_regular_domain_down">{{ splash_crawlers_latest_stats['regular']['domains_down'] }}</a> DOWN
</div>
<div class="col-6">
<a href="{{ url_for('hiddenServices.show_domains_by_daterange') }}?service_type=regular&domains_up=True&domains_down=True&date_from={{date}}&date_to={{date}}" class="badge badge-success" id="stat_regular_total">{{ splash_crawlers_latest_stats['regular']['total'] }}</a> Crawled
<span class="badge badge-warning ml-md-3" id="stat_regular_queue">{{ splash_crawlers_latest_stats['regular']['domains_queue'] }}</span> Queue
</div>
</div>
</div>
</div>
</div>
</div>
<table class="table">
<tbody id="tbody_crawler_onion_info">
{% for splash_crawler in all_splash_crawler_status %}
<tr>
<td>
<i class="fas fa-{%if splash_crawler['status']%}check{%else%}times{%endif%}-circle" style="color:{%if splash_crawler['status']%}Green{%else%}Red{%endif%};"></i> {{splash_crawler['crawler_info']}}
</td>
<td>
{%if splash_crawler['type']=='onion'%}
<i class="fas fa-user-secret"></i>
{%else%}
<i class="fab fa-html5">
{%endif%}
</td>
<td>
{{splash_crawler['crawling_domain']}}
</td>
<td style="color:{%if splash_crawler['status']%}Green{%else%}Red{%endif%};">
{{splash_crawler['status_info']}}
</td>
</tr>
{% endfor %}
</tbody>
</table>
<div class="d-flex justify-content-center my-4">
<div class="card border-secondary" style="max-width: 40rem;">
<div class="card-body text-dark">
<h5 class="card-title">Show Domain:</h5>
<form class="" action="{{url_for('crawler_splash.showDomain')}}" method="post">
<div class="input-group mb-3">
<input type="text" class="form-control" style="min-width: 30rem;" placeholder="Domain name" aria-label="Domain name" aria-describedby="btn_show_domain" id="in_show_domain" , name="in_show_domain">
<div class="input-group-append">
<button class="btn btn-info" type="submit" id="btn_show_domain">
<i class="fas fa-search"></i>
</button>
</div>
</div>
</form>
</div>
</div>
</div>
<hr>
<div class="row mb-3">
<div class="col-xl-6">
<div class="text-center">
<a class="btn btn-secondary" href="{{url_for('crawler_splash.domains_explorer_onion')}}" role="button">
<i class="fas fa-user-secret"></i> Onion Domain Explorer
</a>
</div>
</div>
<div class="col-xl-6">
<div class="text-center">
<a class="btn btn-secondary" href="{{url_for('crawler_splash.domains_explorer_web')}}" role="button">
<i class="fab fa-html5"></i> Web Domain Explorer
</a>
</div>
</div>
</div>
{% with object_type='domain' %}
{% include 'tags/block_obj_tags_search.html' %}
{% endwith %}
</div>
</div>
</div>
</body>
<script>
var to_refresh = false
$(document).ready(function(){
$("#page-Crawler").addClass("active");
$("#nav_dashboard").addClass("active");
$( window ).on("focus", function() {
to_refresh = true
refresh_crawler_status();
});
$( window ).on("blur", function() {
to_refresh = false
});
to_refresh = true
refresh_crawler_status();
});
function toggle_sidebar(){
if($('#nav_menu').is(':visible')){
$('#nav_menu').hide();
$('#side_menu').removeClass('border-right')
$('#side_menu').removeClass('col-lg-2')
$('#core_content').removeClass('col-lg-10')
}else{
$('#nav_menu').show();
$('#side_menu').addClass('border-right')
$('#side_menu').addClass('col-lg-2')
$('#core_content').addClass('col-lg-10')
}
}
function refresh_crawler_status(){
$.getJSON("{{ url_for('crawler_splash.crawler_dashboard_json') }}",
function(data) {
$('#stat_onion_domain_up').text(data.splash_crawlers_latest_stats['onion']['domains_up']);
$('#stat_onion_domain_down').text(data.splash_crawlers_latest_stats['onion']['domains_down']);
$('#stat_onion_total').text(data.splash_crawlers_latest_stats['onion']['total']);
$('#stat_onion_queue').text(data.splash_crawlers_latest_stats['onion']['domains_queue']);
$('#stat_regular_domain_up').text(data.splash_crawlers_latest_stats['regular']['domains_up']);
$('#stat_regular_domain_down').text(data.splash_crawlers_latest_stats['regular']['domains_down']);
$('#stat_regular_total').text(data.splash_crawlers_latest_stats['regular']['total']);
$('#stat_regular_queue').text(data.splash_crawlers_latest_stats['regular']['domains_queue']);
if(data.all_splash_crawler_status.length!=0){
$("#tbody_crawler_onion_info").empty();
var tableRef = document.getElementById('tbody_crawler_onion_info');
for (var i = 0; i < data.all_splash_crawler_status.length; i++) {
var crawler = data.all_splash_crawler_status[i];
var newRow = tableRef.insertRow(tableRef.rows.length);
var text_color;
var icon;
if(crawler['status']){
text_color = 'Green';
icon = 'check';
} else {
text_color = 'Red';
icon = 'times';
}
if(crawler['type'] === 'onion'){
icon_t = 'fas fa-user-secret';
} else {
icon_t = 'fab fa-html5';
}
var newCell = newRow.insertCell(0);
newCell.innerHTML = "<td><i class=\"fas fa-"+icon+"-circle\" style=\"color:"+text_color+";\"></i> "+crawler['crawler_info']+"</td>";
var newCell = newRow.insertCell(1);
newCell.innerHTML = "<td><i class=\""+icon_t+"\"></i></td>";
newCell = newRow.insertCell(2);
newCell.innerHTML = "<td>"+crawler['crawling_domain']+"</td>";
newCell = newRow.insertCell(3);
newCell.innerHTML = "<td><div style=\"color:"+text_color+";\">"+crawler['status_info']+"</div></td>";
//$("#panel_crawler").show();
}
}
}
);
if (to_refresh) {
setTimeout("refresh_crawler_status()", 10000);
}
}
</script>

8
var/www/templates/crawler/menu_sidebar.html
View File

@ -14,7 +14,7 @@
</h5>
<ul class="nav flex-md-column flex-row navbar-nav justify-content-between w-100"> <!--nav-pills-->
<li class="nav-item">
<a class="nav-link" href="{{url_for('hiddenServices.dashboard')}}" id="nav_dashboard">
<a class="nav-link" href="{{url_for('crawler_splash.crawlers_dashboard')}}" id="nav_dashboard">
<i class="fas fa-search"></i>
<span>Dashboard</span>
</a>
@ -43,6 +43,12 @@
Automatic Crawler
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="{{url_for('hiddenServices.auto_crawler')}}" id="nav_settings">
<i class="fas fa-cog"></i>
Settings
</a>
</li>
</ul>
<h5 class="d-flex text-muted w-100" id="nav_title_domains_explorer">

2
var/www/templates/nav_bar.html
View File

@ -22,7 +22,7 @@
<a class="nav-link" id="page-Tracker" href="{{ url_for('hunter.tracked_menu') }}" aria-disabled="true"><i class="fas fa-crosshairs"></i> Leaks Hunter</a>
</li>
<li class="nav-item mr-3">
<a class="nav-link" id="page-Crawler" href="{{ url_for('hiddenServices.dashboard') }}" tabindex="-1" aria-disabled="true"><i class="fas fa-spider"></i> Crawlers</a>
<a class="nav-link" id="page-Crawler" href="{{ url_for('crawler_splash.crawlers_dashboard') }}" tabindex="-1" aria-disabled="true"><i class="fas fa-spider"></i> Crawlers</a>
</li>
<li class="nav-item mr-3">
<a class="nav-link" id="page-Decoded" href="{{ url_for('hashDecoded.hashDecoded_page') }}" aria-disabled="true"><i class="fas fa-cube"></i> Objects</a>