From b63fa51166b57c6aa7a6b0a5dd291546f3fa064a Mon Sep 17 00:00:00 2001 From: Terrtia Date: Fri, 24 Jan 2020 15:03:04 +0100 Subject: [PATCH] chg: [domain explorer] domains explorer v2, filter domains by daterange --- bin/lib/Domain.py | 61 ++++- var/www/blueprints/crawler_splash.py | 61 ++++- .../crawler_splash/domain_explorer.html | 146 +++++------ var/www/templates/crawler/menu_sidebar.html | 20 ++ .../templates/domains/card_img_domain.html | 52 ++++ .../templates/domains/domain_explorer.html | 239 ++++++++++++++++++ var/www/templates/domains/filter_domains.html | 41 +++ 7 files changed, 525 insertions(+), 95 deletions(-) create mode 100644 var/www/templates/domains/card_img_domain.html create mode 100644 var/www/templates/domains/domain_explorer.html create mode 100644 var/www/templates/domains/filter_domains.html diff --git a/bin/lib/Domain.py b/bin/lib/Domain.py index 8c1c8b40..aa1b51cf 100755 --- a/bin/lib/Domain.py +++ b/bin/lib/Domain.py @@ -48,20 +48,29 @@ def get_db_keys_domain_up(domain_type, date_type): # sanitise domain_type def get_list_db_keys_domain_up(domain_type, l_dates, date_type): l_keys_name = [] - key_name = get_db_keys_domain_up(domain_type, date_type) - if key_name: - for str_date in l_dates: - l_keys_name.append(key_name.format(str_date)) + if domain_type=='all': + domains_types = get_all_domains_type() + else: + domains_types = [domain_type] + + for dom_type in domains_types: + key_name = get_db_keys_domain_up(dom_type, date_type) + if key_name: + for str_date in l_dates: + l_keys_name.append(key_name.format(str_date)) return l_keys_name ######## UTIL ######## def sanitize_domain_type(domain_type): - if domain_type in ['onion', 'regular']: + if domain_type in get_all_domains_type(): return domain_type else: return 'regular' ######## DOMAINS ######## +def get_all_domains_type(): + return ['onion', 'regular'] + def get_all_domains_up(domain_type, r_list=True): ''' Get all domain up (at least one time) @@ -116,7 +125,7 @@ def get_domains_up_by_daterange(date_from, date_to, domain_type): ''' Get all domain up (at least one time) by daterange - :param domain_type: date YYYYMMDD + :param domain_type: domain_type :type domain_type: str :return: list of domain @@ -184,14 +193,35 @@ def domains_up_by_page(domain_type, nb_obj=28, page=1): ''' domains = sorted(get_all_domains_up(domain_type, r_list=False)) domains = paginate_iterator(domains, nb_obj=nb_obj, page=page) - - # # TODO: get tags + root_screenshot + metadata - l_domains = [] - for domain in domains['list_elem']: - l_domains.append(get_domain_metadata(domain, domain_type, first_seen=True, last_ckeck=True, status=True, ports=True, tags=True, screenshot=True)) - domains['list_elem'] = l_domains + domains['list_elem'] = create_domains_metadata_list(domains['list_elem'], domain_type) return domains +def get_domains_up_by_filers(domain_type, date_from=None, date_to=None, tags=[], nb_obj=28, page=1): + if not tags: + if not date_from and not date_to: + return domains_up_by_page(domain_type, nb_obj=nb_obj, page=page) + else: + domains = sorted(get_domains_up_by_daterange(date_from, date_to, domain_type)) + domains = paginate_iterator(domains, nb_obj=nb_obj, page=page) + domains['list_elem'] = create_domains_metadata_list(domains['list_elem'], domain_type) + domains['domain_type'] = domain_type + domains['date_from'] = date_from + domains['date_to'] = date_to + return domains + else: + return None + +def create_domains_metadata_list(list_domains, domain_type): + l_domains = [] + for domain in list_domains: + if domain_type=='all': + dom_type = get_domain_type(domain) + else: + dom_type = domain_type + l_domains.append(get_domain_metadata(domain, dom_type, first_seen=True, last_ckeck=True, status=True, + ports=True, tags=True, screenshot=True, tags_safe=True)) + return l_domains + ######## DOMAIN ######## def get_domain_type(domain): @@ -439,7 +469,7 @@ def get_domain_random_screenshot(domain): ''' return Screenshot.get_randon_domain_screenshot(domain) -def get_domain_metadata(domain, domain_type, first_seen=True, last_ckeck=True, status=True, ports=True, tags=False, screenshot=False): +def get_domain_metadata(domain, domain_type, first_seen=True, last_ckeck=True, status=True, ports=True, tags=False, tags_safe=False, screenshot=False): ''' Get Domain basic metadata @@ -471,6 +501,11 @@ def get_domain_metadata(domain, domain_type, first_seen=True, last_ckeck=True, s dict_metadata['ports'] = get_domain_all_ports(domain, domain_type) if tags: dict_metadata['tags'] = get_domain_tags(domain) + if tags_safe: + if tags: + dict_metadata['is_tags_safe'] = Tag.is_tags_safe(dict_metadata['tags']) + else: + dict_metadata['is_tags_safe'] = Tag.is_tags_safe(get_domain_tags(domain)) if screenshot: dict_metadata['screenshot'] = get_domain_random_screenshot(domain) return dict_metadata diff --git a/var/www/blueprints/crawler_splash.py b/var/www/blueprints/crawler_splash.py index 1cedd8e6..ee2e68ba 100644 --- a/var/www/blueprints/crawler_splash.py +++ b/var/www/blueprints/crawler_splash.py @@ -80,28 +80,79 @@ def showDomain(): return render_template("showDomain.html", dict_domain=dict_domain, bootstrap_label=bootstrap_label, modal_add_tags=Tag.get_modal_add_tags(dict_domain['domain'], object_type="domain")) -@crawler_splash.route('/domains/explorer/onion', methods=['GET', 'POST']) +@crawler_splash.route('/domains/explorer/domain_type_post', methods=['POST']) @login_required @login_read_only -def domains_explorer_onion(): +def domains_explorer_post_filter(): + domain_onion = request.form.get('domain_onion_switch') + domain_regular = request.form.get('domain_regular_switch') + date_from = request.form.get('date_from') + date_to = request.form.get('date_to') + + if date_from and date_to: + date_from = date_from.replace('-', '') + date_to = date_to.replace('-', '') + else: + date_from = None + date_to = None + + if domain_onion and domain_regular: + if date_from and date_to: + return redirect(url_for('crawler_splash.domains_explorer_all', date_from=date_from, date_to=date_to)) + else: + return redirect(url_for('crawler_splash.domains_explorer_all')) + elif domain_regular: + if date_from and date_to: + return redirect(url_for('crawler_splash.domains_explorer_web', date_from=date_from, date_to=date_to)) + else: + return redirect(url_for('crawler_splash.domains_explorer_web')) + else: + if date_from and date_to: + return redirect(url_for('crawler_splash.domains_explorer_onion', date_from=date_from, date_to=date_to)) + else: + return redirect(url_for('crawler_splash.domains_explorer_onion')) + +@crawler_splash.route('/domains/explorer/all', methods=['GET']) +@login_required +@login_read_only +def domains_explorer_all(): page = request.args.get('page') + date_from = request.args.get('date_from') + date_to = request.args.get('date_to') try: page = int(page) except: page = 1 - dict_data = Domain.domains_up_by_page('onion', page=page) + dict_data = Domain.get_domains_up_by_filers('all', page=page, date_from=date_from, date_to=date_to) + return render_template("domain_explorer.html", dict_data=dict_data, bootstrap_label=bootstrap_label, domain_type='all') + +@crawler_splash.route('/domains/explorer/onion', methods=['GET']) +@login_required +@login_read_only +def domains_explorer_onion(): + page = request.args.get('page') + date_from = request.args.get('date_from') + date_to = request.args.get('date_to') + try: + page = int(page) + except: + page = 1 + + dict_data = Domain.get_domains_up_by_filers('onion', page=page, date_from=date_from, date_to=date_to) return render_template("domain_explorer.html", dict_data=dict_data, bootstrap_label=bootstrap_label, domain_type='onion') -@crawler_splash.route('/domains/explorer/web', methods=['GET', 'POST']) +@crawler_splash.route('/domains/explorer/web', methods=['GET']) @login_required @login_read_only def domains_explorer_web(): page = request.args.get('page') + date_from = request.args.get('date_from') + date_to = request.args.get('date_to') try: page = int(page) except: page = 1 dict_data = Domain.domains_up_by_page('regular', page=page) - return render_template("domain_explorer.html", dict_data=dict_data, bootstrap_label=bootstrap_label, domain_type='onion') + return render_template("domain_explorer.html", dict_data=dict_data, bootstrap_label=bootstrap_label, domain_type='regular') diff --git a/var/www/templates/crawler/crawler_splash/domain_explorer.html b/var/www/templates/crawler/crawler_splash/domain_explorer.html index e80bdb7f..007da8f0 100644 --- a/var/www/templates/crawler/crawler_splash/domain_explorer.html +++ b/var/www/templates/crawler/crawler_splash/domain_explorer.html @@ -6,10 +6,13 @@ + + + + + + + + {% include 'nav_bar.html' %} + +
+
+ + {% include 'crawler/menu_sidebar.html' %} + +
+ +
+
+
+
+ +
+
+ +
+
+ +
+
+
+
+ + {% for dict_domain in dict_data['list_elem'] %} + + {% if loop.index0 % 4 == 0 %} +
+ {% endif %} + +
+
+ + +
+
+
+ + {{dict_domain["id"]}} + +
+

+ + First seen: {{dict_domain["first_seen"]}}
+ Last_seen: {{dict_domain["first_seen"]}}
+ Ports: {{dict_domain["ports"]}} + +

+ Status: + {% if dict_domain["status"] %} + + + UP + + {% else %} + + + DOWN + + {% endif %} +
+ {% for tag in dict_domain['tags'] %} + + {{ tag }} + + {% endfor %} +
+
+
+ + + {% if loop.index0 % 4 == 3 %} +
+ {% endif %} + + {% endfor %} + + {% if dict_data['list_elem']|length % 4 != 0 %} +
+ {% endif %} + +
+
+ + {%if 'list_elem' in dict_data%} + {% with page=dict_data['page'], nb_page_max=dict_data['nb_pages'], nb_first_elem=dict_data['nb_first_elem'], nb_last_elem=dict_data['nb_last_elem'], nb_all_elem=dict_data['nb_all_elem'] %} + {% set object_name="domain" %} + {%if domain_type=='onion'%} + {% set target_url=url_for('crawler_splash.domains_explorer_onion') + "?domain_type=onion" %} + {%else%} + {% set target_url=url_for('crawler_splash.domains_explorer_web') + "?domain_type=regular" %} + {%endif%} + {% include 'pagination.html' %} + {% endwith %} + {%endif%} + + + + + +
+
+ + + + + + + + + + + diff --git a/var/www/templates/domains/filter_domains.html b/var/www/templates/domains/filter_domains.html new file mode 100644 index 00000000..b8d638ae --- /dev/null +++ b/var/www/templates/domains/filter_domains.html @@ -0,0 +1,41 @@ +
+
+
Filter domains by date range :
+

You can filter domains by a specified daterange. Display all domains if empty.

+
+
+
+ +
+
+ +
+
+
+ +
+
+
+
+ + +
+
+ + +
+
+
+
+ + +
+
+
+