CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

fix: [search domains] fix search domains by daterange

pull/594/head
Terrtia 2023-04-24 13:35:55 +02:00
parent 10fbf228c1
commit 8f65fd5fe7
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
5 changed files with 55 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
bin/lib/objects/Domains.py
View File

@ -531,16 +531,19 @@ def get_domains_by_daterange(date_from, date_to, domain_type, up=True, down=Fals
domains.extend(get_domains_down_by_date(date, domain_type)) domains.extend(get_domains_down_by_date(date, domain_type))
return domains return domains
def get_domains_dates_by_daterange(date_from, date_to, domain_type, up=True, down=False): def get_domains_dates_by_daterange(date_from, date_to, domain_types, up=True, down=False):
if not domain_types:
domain_types = get_all_domains_types()
date_domains = {} date_domains = {}
for date in Date.substract_date(date_from, date_to): for date in Date.substract_date(date_from, date_to):
domains = [] domains = []
if up: for domain_type in domain_types:
domains.extend(get_domains_up_by_date(date, domain_type)) if up:
if down: domains.extend(get_domains_up_by_date(date, domain_type))
domains.extend(get_domains_down_by_date(date, domain_type)) if down:
if domains: domains.extend(get_domains_down_by_date(date, domain_type))
date_domains[date] = list(domains) if domains:
date_domains[date] = list(domains)
return date_domains return date_domains
def get_domains_meta(domains): def get_domains_meta(domains):

28
var/www/blueprints/crawler_splash.py
View File

@ -68,6 +68,7 @@ def crawlers_dashboard():
return render_template("dashboard_crawler.html", date=date, return render_template("dashboard_crawler.html", date=date,
is_manager_connected=is_manager_connected, is_manager_connected=is_manager_connected,
crawlers_status=crawlers_status, crawlers_status=crawlers_status,
filter_up=True,
crawlers_latest_stats=crawlers_latest_stats) crawlers_latest_stats=crawlers_latest_stats)
@ -471,6 +472,9 @@ def domains_search_name():
except: except:
page = 1 page = 1
if not name:
return create_json_response({'error': 'Mandatory args name not provided'}, 400)
domains_types = request.args.getlist('domain_types') domains_types = request.args.getlist('domain_types')
if domains_types: if domains_types:
domains_types = domains_types[0].split(',') domains_types = domains_types[0].split(',')
@ -487,13 +491,25 @@ def domains_search_name():
@login_analyst @login_analyst
def domains_search_date(): def domains_search_date():
# TODO sanitize type + date # TODO sanitize type + date
domain_type = request.args.get('type') dom_types = request.args.get('type')
date_from = request.args.get('date_from') date_from = request.args.get('date_from')
date_to = request.args.get('date_to') date_to = request.args.get('date_to')
down = bool(request.args.get('down', False))
up = bool(request.args.get('up'))
# page = request.args.get('page') # page = request.args.get('page')
all_types = Domains.get_all_domains_types()
if dom_types == 'all':
domain_types = all_types
elif dom_types in Domains.get_all_domains_types():
domain_types = [dom_types]
else:
dom_types = 'all'
domain_types = all_types
date = Date.sanitise_date_range(date_from, date_to) date = Date.sanitise_date_range(date_from, date_to)
domains_date = Domains.get_domains_dates_by_daterange(date['date_from'], date['date_to'], domain_type) domains_date = Domains.get_domains_dates_by_daterange(date['date_from'], date['date_to'], domain_types,
up=up, down=down)
dict_domains = {} dict_domains = {}
for d in domains_date: for d in domains_date:
dict_domains[d] = Domains.get_domains_meta(domains_date[d]) dict_domains[d] = Domains.get_domains_meta(domains_date[d])
@ -502,7 +518,8 @@ def domains_search_date():
return render_template("domains_daterange.html", date_from=date_from, date_to=date_to, return render_template("domains_daterange.html", date_from=date_from, date_to=date_to,
bootstrap_label=bootstrap_label, bootstrap_label=bootstrap_label,
dict_domains=dict_domains, type=domain_type) filter_down=down, filter_up=up,
dict_domains=dict_domains, type=dom_types)
@crawler_splash.route('/domains/date/post', methods=['POST']) @crawler_splash.route('/domains/date/post', methods=['POST'])
@ -512,7 +529,10 @@ def domains_search_date_post():
domain_type = request.form.get('type') domain_type = request.form.get('type')
date_from = request.form.get('date_from') date_from = request.form.get('date_from')
date_to = request.form.get('date_to') date_to = request.form.get('date_to')
return redirect(url_for('crawler_splash.domains_search_date', date_from=date_from, date_to=date_to, type=domain_type)) down = request.form.get('down')
up = request.form.get('up')
return redirect(url_for('crawler_splash.domains_search_date', date_from=date_from, date_to=date_to,
type=domain_type, down=down, up=up))
##-- --## ##-- --##

12
var/www/templates/crawler/crawler_splash/dashboard_crawler.html
View File

@ -39,11 +39,11 @@
<h5><a class="text-info" href="{{ url_for('crawler_splash.crawlers_last_domains')}}?type=onion"><i class="fas fa-user-secret"></i> Onions Crawlers</a></h5> <h5><a class="text-info" href="{{ url_for('crawler_splash.crawlers_last_domains')}}?type=onion"><i class="fas fa-user-secret"></i> Onions Crawlers</a></h5>
<div class="row"> <div class="row">
<div class="col-6"> <div class="col-6">
<a href="{{ url_for('crawler_splash.domains_search_date') }}?service_type=onion&domains_up=True&date_from={{date}}&date_to={{date}}" class="badge badge-success" id="stat_onion_domain_up">{{ crawlers_latest_stats['onion']['up'] }}</a> UP <a href="{{ url_for('crawler_splash.domains_search_date') }}?type=onion&date_from={{date}}&date_to={{date}}" class="badge badge-success" id="stat_onion_domain_up">{{ crawlers_latest_stats['onion']['up'] }}</a> UP
<a href="{{ url_for('crawler_splash.domains_search_date') }}?service_type=onion&domains_down=True&date_from={{date}}&date_to={{date}}" class="badge badge-danger ml-md-3" id="stat_onion_domain_down">{{ crawlers_latest_stats['onion']['down'] }}</a> DOWN <a href="{{ url_for('crawler_splash.domains_search_date') }}?type=onion&down=True&up=False&date_from={{date}}&date_to={{date}}" class="badge badge-danger ml-md-3" id="stat_onion_domain_down">{{ crawlers_latest_stats['onion']['down'] }}</a> DOWN
</div> </div>
<div class="col-6"> <div class="col-6">
<a href="{{ url_for('crawler_splash.domains_search_date') }}?service_type=onion&domains_up=True&domains_down=True&date_from={{date}}&date_to={{date}}" class="badge badge-success" id="stat_onion_total">{{ crawlers_latest_stats['onion']['crawled'] }}</a> Crawled <a href="{{ url_for('crawler_splash.domains_search_date') }}?type=onion&up=True&down=True&date_from={{date}}&date_to={{date}}" class="badge badge-success" id="stat_onion_total">{{ crawlers_latest_stats['onion']['crawled'] }}</a> Crawled
<span class="badge badge-warning ml-md-3" id="stat_onion_queue">{{ crawlers_latest_stats['onion']['queue'] }}</span> Queue <span class="badge badge-warning ml-md-3" id="stat_onion_queue">{{ crawlers_latest_stats['onion']['queue'] }}</span> Queue
</div> </div>
</div> </div>
@ -57,11 +57,11 @@
<h5><a class="text-info" href="{{ url_for('crawler_splash.crawlers_last_domains')}}?type=web"><i class="fab fa-html5"></i> Web Crawlers</a></h5> <h5><a class="text-info" href="{{ url_for('crawler_splash.crawlers_last_domains')}}?type=web"><i class="fab fa-html5"></i> Web Crawlers</a></h5>
<div class="row"> <div class="row">
<div class="col-6"> <div class="col-6">
<a href="{{ url_for('crawler_splash.domains_search_date') }}?service_type=web&domains_up=True&date_from={{date}}&date_to={{date}}" class="badge badge-success" id="stat_web_domain_up">{{ crawlers_latest_stats['web']['up'] }}</a> UP <a href="{{ url_for('crawler_splash.domains_search_date') }}?type=web&date_from={{date}}&date_to={{date}}" class="badge badge-success" id="stat_web_domain_up">{{ crawlers_latest_stats['web']['up'] }}</a> UP
<a href="{{ url_for('crawler_splash.domains_search_date') }}?service_type=web&domains_down=True&date_from={{date}}&date_to={{date}}" class="badge badge-danger ml-md-3" id="stat_web_domain_down">{{ crawlers_latest_stats['web']['down'] }}</a> DOWN <a href="{{ url_for('crawler_splash.domains_search_date') }}?type=web&down=True&up=False&date_from={{date}}&date_to={{date}}" class="badge badge-danger ml-md-3" id="stat_web_domain_down">{{ crawlers_latest_stats['web']['down'] }}</a> DOWN
</div> </div>
<div class="col-6"> <div class="col-6">
<a href="{{ url_for('crawler_splash.domains_search_date') }}?service_type=web&domains_up=True&domains_down=True&date_from={{date}}&date_to={{date}}" class="badge badge-success" id="stat_web_total">{{ crawlers_latest_stats['web']['crawled'] }}</a> Crawled <a href="{{ url_for('crawler_splash.domains_search_date') }}?type=web&up=True&down=True&date_from={{date}}&date_to={{date}}" class="badge badge-success" id="stat_web_total">{{ crawlers_latest_stats['web']['crawled'] }}</a> Crawled
<span class="badge badge-warning ml-md-3" id="stat_web_queue">{{ crawlers_latest_stats['web']['queue'] }}</span> Queue <span class="badge badge-warning ml-md-3" id="stat_web_queue">{{ crawlers_latest_stats['web']['queue'] }}</span> Queue
</div> </div>
</div> </div>

12
var/www/templates/crawler/show_domains_by_daterange.html
View File

@ -3,11 +3,11 @@
<div class="card-header"> <div class="card-header">
<div class="row"> <div class="row">
<div class="col-6"> <div class="col-6">
<a href="{{ url_for('crawler_splash.domains_search_date') }}?service_type={{type}}&domains_up=True&date_from={{date_from}}&date_to={{date_to}}" class="badge badge-success">{{ crawler_stats[type]['up'] }}</a> UP <a href="{{ url_for('crawler_splash.domains_search_date') }}?type={{type}}&date_from={{date_from}}&date_to={{date_to}}" class="badge badge-success">{{ crawler_stats[type]['up'] }}</a> UP
<a href="{{ url_for('crawler_splash.domains_search_date') }}?service_type={{type}}&domains_down=True&date_from={{date_from}}&date_to={{date_to}}" class="badge badge-danger ml-md-3">{{ crawler_stats[type]['down'] }}</a> DOWN <a href="{{ url_for('crawler_splash.domains_search_date') }}?type={{type}}&down=True&up=False&date_from={{date_from}}&date_to={{date_to}}" class="badge badge-danger ml-md-3">{{ crawler_stats[type]['down'] }}</a> DOWN
</div> </div>
<div class="col-6"> <div class="col-6">
<a href="{{ url_for('crawler_splash.domains_search_date') }}?service_type={{type}}&domains_up=True&domains_down=True&date_from={{date_from}}&date_to={{date_to}}" class="badge badge-success">{{ crawler_stats[type]['crawled'] }}</a> Crawled <a href="{{ url_for('crawler_splash.domains_search_date') }}?type={{type}}&up=True&down=True&date_from={{date_from}}&date_to={{date_to}}" class="badge badge-success">{{ crawler_stats[type]['crawled'] }}</a> Crawled
<span class="badge badge-warning ml-md-3">{{ crawler_stats[type]['queue'] }}</span> Queue <span class="badge badge-warning ml-md-3">{{ crawler_stats[type]['queue'] }}</span> Queue
</div> </div>
</div> </div>
@ -18,7 +18,7 @@
<form action="{{ url_for('crawler_splash.domains_search_date_post') }}" id="hash_selector_form" method='post'> <form action="{{ url_for('crawler_splash.domains_search_date_post') }}" id="hash_selector_form" method='post'>
<div class="row"> <div class="row">
<div class="col-6"> <div class="col-6">
<input type="hidden" id="type" name="type" value="{{type}}"> <input type="hidden" id="type" name="type" value="{% if type %}{{type}}{% else %}all{% endif %}">
<div class="input-group" id="date-range-from"> <div class="input-group" id="date-range-from">
<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div> <div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
<input class="form-control" id="date-range-from-input" placeholder="yyyy-mm-dd" value="{{ date_from }}" name="date_from" autocomplete="off"> <input class="form-control" id="date-range-from-input" placeholder="yyyy-mm-dd" value="{{ date_from }}" name="date_from" autocomplete="off">
@ -30,13 +30,13 @@
</div> </div>
<div class="col-6"> <div class="col-6">
<div class="custom-control custom-switch"> <div class="custom-control custom-switch">
<input class="custom-control-input" type="checkbox" name="domains_up" value="True" id="domains_up_id" checked> <input class="custom-control-input" type="checkbox" name="up" value="True" id="domains_up_id" {% if filter_up is none or filter_up %}checked{% endif %}>
<label class="custom-control-label" for="domains_up_id"> <label class="custom-control-label" for="domains_up_id">
<span class="badge badge-success"><i class="fas fa-check-circle"></i> Domains UP </span> <span class="badge badge-success"><i class="fas fa-check-circle"></i> Domains UP </span>
</label> </label>
</div> </div>
<div class="custom-control custom-switch"> <div class="custom-control custom-switch">
<input class="custom-control-input" type="checkbox" name="domains_down" value="True" id="domains_down_id"> <input class="custom-control-input" type="checkbox" name="down" value="True" id="domains_down_id" {% if filter_down %}checked{% endif %}>
<label class="custom-control-label" for="domains_down_id"> <label class="custom-control-label" for="domains_down_id">
<span class="badge badge-danger"><i class="fas fa-times-circle"></i> Domains DOWN</span> <span class="badge badge-danger"><i class="fas fa-times-circle"></i> Domains DOWN</span>
</label> </label>

19
var/www/templates/domains/block_domains_name_search.html
View File

@ -31,20 +31,19 @@
</div> </div>
<script> <script>
function searchDomainName() { function searchDomainName() {
var all_domain_types = ['onion', 'regular'] // TODO: load from flask var all_domain_types = ['onion', 'regular'] // TODO: load from flask
var l_domains_types = []; // var l_domains_types = [];
console.log(document.getElementById('in_search_name'));
console.log(document.getElementById('in_search_name')); var data = document.getElementById('in_search_name').value;
for (var i = 0; i < all_domain_types.length; i++) {
var data = document.getElementById('in_search_name').value; if (document.getElementById('domain_'+ all_domain_types[i] +'_switch').checked) {
for (var i = 0; i < all_domain_types.length; i++) { l_domains_types.push(all_domain_types[i])
if (document.getElementById('domain_'+ all_domain_types[i] +'_switch').checked) {
l_domains_types.push(all_domain_types[i])
} }
} }
var parameter = "?name=" + data + "&domain_types=" + l_domains_types +"{%if page%}&page={{ page }}{%endif%}"; var parameter = "?name=" + data + "&domain_types=" + l_domains_types +"{%if page%}&page={{ page }}{%endif%}";
window.location.href = "{{ url_for('crawler_splash.domains_search_name') }}" + parameter; window.location.href = "{{ url_for('crawler_splash.domains_search_name') }}" + parameter;
} }
</script> </script>