diff --git a/bin/lib/Domain.py b/bin/lib/Domain.py index 8da5d960..fd8ac372 100755 --- a/bin/lib/Domain.py +++ b/bin/lib/Domain.py @@ -56,6 +56,7 @@ def get_link_tree(): pass + def get_domain_tags(domain): ''' Retun all tags of a given domain. @@ -103,11 +104,119 @@ def get_domain_all_correlation(domain, correlation_type=None): domain_correl['pgp'] = res return domain_correl + # TODO: handle port +def get_domain_history(domain, domain_type, port): # TODO: add date_range: from to + nb_elem + ''' + Retun . + + :param domain: crawled domain + :type domain: str + + :return: + :rtype: list of tuple (item_core, epoch) + ''' + return r_serv_onion.zrange('crawler_history_{}:{}:{}'.format(domain_type, domain, port), 0, -1, withscores=True) + +def get_domain_history_with_status(domain, domain_type, port): # TODO: add date_range: from to + nb_elem + ''' + Retun . + + :param domain: crawled domain + :type domain: str + + :return: + :rtype: list of dict (epoch, date: %Y/%m/%d - %H:%M.%S, boolean status) + ''' + l_history = [] + history = get_domain_history(domain, domain_type, port) + for root_item, epoch_val in history: + epoch_val = int(epoch_val) # force int + # domain down, root_item==epoch_val + try: + int(root_item) + status = False + # domain up, root_item=str + except ValueError: + status = True + l_history.append({"epoch": epoch_val, "date": time.strftime('%Y/%m/%d - %H:%M.%S', time.gmtime(epoch_val)), "status": status}) + return l_history + class Domain(object): """docstring for Domain.""" def __init__(self, domain, port=80): self.domain = str(domain) - ## TODO: handle none port self.type = get_domain_type(domain) + + def get_domain_first_seen(self): + ''' + Get domain first seen date + + :return: domain first seen date + :rtype: str + ''' + first_seen = r_serv_onion.hget('{}_metadata:{}'.format(self.type, self.domain), 'first_seen') + if first_seen is not None: + first_seen = '{}/{}/{}'.format(first_seen[0:4], first_seen[4:6], first_seen[6:8]) + return first_seen + + def get_domain_last_check(self):# # TODO: add epoch ??? + ''' + Get domain last check date + + :return: domain last check date + :rtype: str + ''' + last_check = r_serv_onion.hget('{}_metadata:{}'.format(self.type, self.domain), 'last_check') + if last_check is not None: + last_check = '{}/{}/{}'.format(last_check[0:4], last_check[4:6], last_check[6:8]) + return last_check + + #def get_domain_all_ports(self): + # pass + + def get_domain_metadata(self, first_seen=True, last_ckeck=True, ports=True): + ''' + Get Domain basic metadata + + :param first_seen: get domain first_seen + :type first_seen: boolean + :param last_ckeck: get domain last_check + :type last_ckeck: boolean + :param ports: get all domain ports + :type ports: boolean + + :return: a dict of all metadata for a given domain + :rtype: dict + ''' + dict_metadata = {} + if first_seen: + res = self.get_domain_first_seen() + if res is not None: + dict_metadata['first_seen'] = res + if last_ckeck: + res = self.get_domain_last_check() + if res is not None: + dict_metadata['last_check'] = res + return dict_metadata + + def get_domain_tags(self): + ''' + Retun all tags of a given domain. + + :param domain: crawled domain + ''' + return get_domain_tags(self.domain) + + def get_domain_correlation(self): + ''' + Retun all cryptocurrencies of a given domain. + ''' + return get_domain_all_correlation(self.domain) + + def get_domain_history_with_status(self): + ''' + Retun the full history of a given domain and port. + ''' + return get_domain_history_with_status(self.domain, self.type, 80) diff --git a/bin/packages/Correlation.py b/bin/packages/Correlation.py index 27b6dc24..ee339fb2 100755 --- a/bin/packages/Correlation.py +++ b/bin/packages/Correlation.py @@ -108,6 +108,24 @@ class Correlation(object): else: return [] + def _get_correlation_obj_domain(self, field_name, correlation_type): + ''' + Return all domains that contain this correlation. + + :param domain: field name + :type domain: str + :param correlation_type: correlation type + :type correlation_type: str + + :return: a list of correlation + :rtype: list + ''' + res = r_serv_metadata.smembers('set_domain_{}_{}:{}'.format(self.correlation_name, correlation_type, field_name)) + if res: + return list(res) + else: + return [] + def get_domain_correlation_dict(self, domain, correlation_type=None): ''' Return all correlation of a given domain. diff --git a/var/www/blueprints/crawler_splash.py b/var/www/blueprints/crawler_splash.py new file mode 100644 index 00000000..364a84d9 --- /dev/null +++ b/var/www/blueprints/crawler_splash.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +''' + Blueprint Flask: crawler splash endpoints: dashboard, onion crawler ... +''' + +import os +import sys + +from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response +from flask_login import login_required, current_user, login_user, logout_user + +sys.path.append('modules') +import Flask_config + +# Import Role_Manager +from Role_Manager import create_user_db, check_password_strength, check_user_role_integrity +from Role_Manager import login_admin, login_analyst + +sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib')) +import Domain + +r_cache = Flask_config.r_cache +r_serv_db = Flask_config.r_serv_db +r_serv_tags = Flask_config.r_serv_tags +bootstrap_label = Flask_config.bootstrap_label + +# ============ BLUEPRINT ============ +crawler_splash = Blueprint('crawler_splash', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/crawler/crawler_splash')) + +# ============ VARIABLES ============ + + + +# ============ FUNCTIONS ============ + + + +# ============= ROUTES ============== +@crawler_splash.route('/crawlers/showDomain') +#@login_required +#@login_analyst +def showDomain(): + domain_name = request.args.get('domain') + epoch = request.args.get('epoch') + port = request.args.get('port') + + domain = Domain.Domain(domain_name) + + dict_domain = domain.get_domain_metadata() + dict_domain = {**dict_domain, **domain.get_domain_correlation()} + dict_domain['domain'] = domain_name + dict_domain['tags'] = domain.get_domain_tags() + dict_domain['history'] = domain.get_domain_history_with_status() + + print(dict_domain) + + return render_template("showDomain.html", dict_domain=dict_domain, bootstrap_label=bootstrap_label, screenshot={'item': None, '':None}, dict_links={}) diff --git a/var/www/modules/hiddenServices/templates/showDomain.html b/var/www/templates/crawler/crawler_splash/showDomain.html similarity index 52% rename from var/www/modules/hiddenServices/templates/showDomain.html rename to var/www/templates/crawler/crawler_splash/showDomain.html index 4230ec5a..72c9e155 100644 --- a/var/www/modules/hiddenServices/templates/showDomain.html +++ b/var/www/templates/crawler/crawler_splash/showDomain.html @@ -6,10 +6,10 @@ - - + + @@ -45,7 +45,7 @@ {% endif %} -

{{ domain }} :

+

{{ dict_domain['domain'] }} :

@@ -58,23 +58,22 @@ - - + +
{{ first_seen }}{{ last_check }}{%if "first_seen" in dict_domain%}{{ dict_domain['first_seen'] }}{%endif%}{%if "last_check" in dict_domain%}{{ dict_domain['last_check'] }}{%endif%} {{ ports }}
- Origin Paste: {% if origin_paste_name=='manual' or origin_paste_name=='auto' %} {{ origin_paste_name }} {%else%} {{ origin_paste_name }} {%endif%}
- {% for tag in origin_paste_tags %} - - {{ tag[0] }} + {% for tag in dict_domain['tags'] %} + + {{ tag }} {% endfor %}
@@ -83,17 +82,122 @@
-
- {% for tag in domain_tags %} - - {{ tag }} {{ domain_tags[tag] }} - - {% endfor %} -
-
-
+ + {% if 'pgp' in dict_domain%} +
+
+
+
+
+
+ PGP Dumps   +
{{l_64|length}}
+
+
+
+ +
+
+
+
+
+ + + + + + + + + {% for dict_key in dict_domain['pgp']%} + {% if dict_key=="mail" %} + {% set var_icon = "fas fa-at" %} + {% elif dict_key=="name" %} + {% set var_icon = "fas fa-user-tag" %} + {% else %} + {% set var_icon = "fas fa-key" %} + {% endif %} + {% for key_id in dict_domain['pgp'][dict_key]%} + + + + + {% endfor %} + {% endfor %} + +
PGP TypeKey ID
+ +   {{ dict_key }} + {{ key_id }}
+
+
+
+
+ {% endif %} + + + {% if 'cryptocurrency' in dict_domain%} +
+
+
+
+
+
+ Cryptocurrencies   +
{{l_64|length}}
+
+
+
+ +
+
+
+
+
+ + + + + + + + + {% for dict_key in dict_domain['cryptocurrency']%} + {% if dict_key=="bitcoin" %} + {% set var_icon = "fab fa-bitcoin" %} + {% elif dict_key=="monero" %} + {% set var_icon = "fab fa-monero" %} + {% else %} + {% set var_icon = "fas fa-coins" %} + {% endif %} + {% for key_id in dict_domain['cryptocurrency'][dict_key]%} + + + + + {% endfor %} + {% endfor %} + +
Currencyaddress
+ +   {{ dict_key }} + {{ key_id }}
+
+
+
+
+ {% endif %} + + + {% if l_pastes %} +
+ @@ -129,7 +233,7 @@ - {% if domain_history %} + {% if dict_domain["domain_history"] %}
@@ -142,7 +246,7 @@
-
{{domain}}
+
{{dict_domain["domain"]}}
{% if epoch_item[2] %}
UP
{% else %} @@ -177,11 +281,13 @@
+
@@ -196,13 +302,19 @@