diff --git a/.gitignore b/.gitignore index f9d2344e..b12d5a13 100644 --- a/.gitignore +++ b/.gitignore @@ -46,6 +46,7 @@ configs/core.cfg.backup configs/update.cfg update/current_version files +configs/d4client_passiveDNS_conf/uuid # Trackers bin/trackers/yara/custom-rules/* diff --git a/bin/DomClassifier.py b/bin/DomClassifier.py index 1ae5ba13..aee83ca9 100755 --- a/bin/DomClassifier.py +++ b/bin/DomClassifier.py @@ -9,13 +9,18 @@ The DomClassifier modules extract and classify Internet domains/hostnames/IP add the out output of the Global module. """ +import os +import sys import time -from packages import Paste from pubsublogger import publisher import DomainClassifier.domainclassifier from Helper import Process +sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib')) +import d4 +import item_basic + def main(): publisher.port = 6380 @@ -35,35 +40,42 @@ def main(): while True: try: - message = p.get_from_set() + item_id = p.get_from_set() - if message is not None: - PST = Paste.Paste(message) - else: + if item_id is None: publisher.debug("Script DomClassifier is idling 1s") time.sleep(1) continue - paste = PST.get_p_content() - mimetype = PST._get_p_encoding() - if mimetype == "text/plain": - c.text(rawtext=paste) + item_content = item_basic.get_item_content(item_id) + mimetype = item_basic.get_item_mimetype(item_id) + item_basename = item_basic.get_basename(item_id) + item_source = item_basic.get_source(item_id) + item_date = item_basic.get_item_date(item_id) + + if mimetype.split('/')[0] == "text": + c.text(rawtext=item_content) c.potentialdomain() - c.validdomain(rtype=['A'], extended=True) + c.validdomain(rtype=['A'], passive_dns=True, extended=False) + print(c.vdomain) + + if c.vdomain and d4.is_passive_dns_enabled(): + for dns_record in c.vdomain: + p.populate_set_out(dns_record) + localizeddomains = c.include(expression=cc_tld) if localizeddomains: print(localizeddomains) - publisher.warning('DomainC;{};{};{};Checked {} located in {};{}'.format( - PST.p_source, PST.p_date, PST.p_name, localizeddomains, cc_tld, PST.p_rel_path)) + publisher.warning(f"DomainC;{item_source};{item_date};{item_basename};Checked {localizeddomains} located in {cc_tld};{item_id}") localizeddomains = c.localizedomain(cc=cc) + if localizeddomains: print(localizeddomains) - publisher.warning('DomainC;{};{};{};Checked {} located in {};{}'.format( - PST.p_source, PST.p_date, PST.p_name, localizeddomains, cc, PST.p_rel_path)) + publisher.warning(f"DomainC;{item_source};{item_date};{item_basename};Checked {localizeddomains} located in {cc};{item_id}") + except IOError: - print("CRC Checksum Failed on :", PST.p_rel_path) - publisher.error('Duplicate;{};{};{};CRC Checksum Failed'.format( - PST.p_source, PST.p_date, PST.p_name)) + print("CRC Checksum Failed on :", item_id) + publisher.error(f"Duplicate;{item_source};{item_date};{item_basename};CRC Checksum Failed") if __name__ == "__main__": main() diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index e231966c..874108b7 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -152,6 +152,8 @@ function launching_scripts { sleep 0.1 screen -S "Script_AIL" -X screen -t "Crawler_manager" bash -c "cd ${AIL_BIN}/core; ${ENV_PY} ./Crawler_manager.py; read x" sleep 0.1 + screen -S "Script_AIL" -X screen -t "D4_client" bash -c "cd ${AIL_BIN}/core; ${ENV_PY} ./D4_client.py; read x" + sleep 0.1 screen -S "Script_AIL" -X screen -t "ModuleInformation" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./ModulesInformationV2.py -k 0 -c 1; read x" diff --git a/bin/core/D4_client.py b/bin/core/D4_client.py new file mode 100755 index 00000000..ab5f99d2 --- /dev/null +++ b/bin/core/D4_client.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +""" +The D4_Client Module +============================ + +The D4_Client modules send all DNS records to a D4 Server. +Data produced by D4 sensors are ingested into +a Passive DNS server which can be queried later to search for the Passive DNS records. +""" + +import os +import sys +import time +from pubsublogger import publisher +sys.path.append(os.environ['AIL_BIN']) +from Helper import Process + +sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib')) +import ConfigLoader +import d4 + +# # TODO: lauch me in core screen +# # TODO: check if already launched in core screen + +if __name__ == '__main__': + publisher.port = 6380 + publisher.channel = "Script" + + config_section = 'D4_client' + p = Process(config_section) + publisher.info("""D4_client is Running""") + + last_refresh = time.time() + d4_client = d4.create_d4_client() + + while True: + if last_refresh < d4.get_config_last_update_time(): + d4_client = d4.create_d4_client() + last_refresh = time.time() + print('D4 Client: config updated') + + dns_record = p.get_from_set() + if dns_record is None: + publisher.debug("Script D4_client is idling 1s") + time.sleep(1) + continue + + if d4_client: + # Send DNS Record to D4Server + d4_client.send_manual_data(dns_record) diff --git a/bin/lib/d4.py b/bin/lib/d4.py new file mode 100755 index 00000000..e3305f75 --- /dev/null +++ b/bin/lib/d4.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +import os +import sys +import time +import redis +import d4_pyclient + +sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib')) +import ConfigLoader + +config_loader = ConfigLoader.ConfigLoader() +r_serv_db = config_loader.get_redis_conn("ARDB_DB") +r_cache = config_loader.get_redis_conn("Redis_Cache") +config_loader = None + +def get_ail_uuid(): + return r_serv_db.get('ail:uuid') + +def get_d4_client_config_dir(): + return os.path.join(os.environ['AIL_HOME'], 'configs', 'd4client_passiveDNS_conf') + +def create_d4_config_file(filename, content): + if not os.path.isfile(filename): + with open(filename, 'a') as f: + f.write(content) + +def get_d4_client_config(): + d4_client_config = get_d4_client_config_dir() + filename = os.path.join(d4_client_config, 'uuid') + if not os.path.isfile(filename): + create_d4_config_file(filename, get_ail_uuid()) + return d4_client_config + +def is_passive_dns_enabled(cache=True): + if cache: + res = r_cache.get('d4:passivedns:enabled') + if res is None: + res = r_serv_db.hget('d4:passivedns', 'enabled') == 'True' + r_cache.set('d4:passivedns:enabled', res) + return res + else: + return res == 'True' + else: + return r_serv_db.hget('d4:passivedns', 'enabled') == 'True' + +def change_passive_dns_state(new_state): + old_state = is_passive_dns_enabled(cache=False) + if old_state != new_state: + r_serv_db.hset('d4:passivedns', 'enabled', bool(new_state)) + r_cache.set('d4:passivedns:enabled', bool(new_state)) + update_time = time.time() + r_serv_db.hset('d4:passivedns', 'update_time', update_time) + r_cache.set('d4:passivedns:last_update_time', update_time) + return True + return False + +def get_config_last_update_time(): + last_update_time = r_cache.get('d4:passivedns:last_update_time') + if not last_update_time: + last_update_time = r_serv_db.hget('d4:passivedns', 'update_time') + if not last_update_time: + last_update_time = 0 + last_update_time = float(last_update_time) + r_cache.set('d4:passivedns:last_update_time', last_update_time) + return float(last_update_time) + +def create_d4_client(): + if is_passive_dns_enabled(): + d4_client = d4_pyclient.D4Client(get_d4_client_config(), False) + return d4_client + else: + return None diff --git a/bin/lib/item_basic.py b/bin/lib/item_basic.py index 010e30b6..1b7e0de5 100755 --- a/bin/lib/item_basic.py +++ b/bin/lib/item_basic.py @@ -5,6 +5,8 @@ import os import sys import gzip +import magic + sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) import ConfigLoader @@ -35,6 +37,9 @@ def get_item_date(item_id, add_separator=False): else: return '{}{}{}'.format(l_directory[-4], l_directory[-3], l_directory[-2]) +def get_basename(item_id): + return os.path.basename(item_id) + def get_source(item_id): return item_id.split('/')[-5] @@ -63,6 +68,9 @@ def get_item_content(item_id): item_content = '' return str(item_content) +def get_item_mimetype(item_id): + return magic.from_buffer(get_item_content(item_id), mime=True) + #### TREE CHILD/FATHER #### def is_father(item_id): return r_serv_metadata.exists('paste_children:{}'.format(item_id)) diff --git a/bin/packages/modules.cfg b/bin/packages/modules.cfg index 302b68af..c29ff926 100644 --- a/bin/packages/modules.cfg +++ b/bin/packages/modules.cfg @@ -21,6 +21,10 @@ subscribe = Redis_Global [DomClassifier] subscribe = Redis_Global +publish = Redis_D4_client + +[D4_client] +subscribe = Redis_D4_client [TermTrackerMod] subscribe = Redis_Global diff --git a/configs/d4client_passiveDNS_conf/destination b/configs/d4client_passiveDNS_conf/destination new file mode 100644 index 00000000..69cef238 --- /dev/null +++ b/configs/d4client_passiveDNS_conf/destination @@ -0,0 +1 @@ +d4pdns.circl.lu:4443 diff --git a/configs/d4client_passiveDNS_conf/key b/configs/d4client_passiveDNS_conf/key new file mode 100644 index 00000000..2c8a76aa --- /dev/null +++ b/configs/d4client_passiveDNS_conf/key @@ -0,0 +1 @@ +ail passivedns sensor key diff --git a/configs/d4client_passiveDNS_conf/snaplen b/configs/d4client_passiveDNS_conf/snaplen new file mode 100644 index 00000000..801c306e --- /dev/null +++ b/configs/d4client_passiveDNS_conf/snaplen @@ -0,0 +1 @@ +4096 diff --git a/configs/d4client_passiveDNS_conf/source b/configs/d4client_passiveDNS_conf/source new file mode 100644 index 00000000..cf52303b --- /dev/null +++ b/configs/d4client_passiveDNS_conf/source @@ -0,0 +1 @@ +stdin diff --git a/configs/d4client_passiveDNS_conf/type b/configs/d4client_passiveDNS_conf/type new file mode 100644 index 00000000..45a4fb75 --- /dev/null +++ b/configs/d4client_passiveDNS_conf/type @@ -0,0 +1 @@ +8 diff --git a/configs/d4client_passiveDNS_conf/version b/configs/d4client_passiveDNS_conf/version new file mode 100644 index 00000000..d00491fd --- /dev/null +++ b/configs/d4client_passiveDNS_conf/version @@ -0,0 +1 @@ +1 diff --git a/var/www/modules/settings/Flask_settings.py b/var/www/modules/settings/Flask_settings.py index 6b8cdb09..68ccd517 100644 --- a/var/www/modules/settings/Flask_settings.py +++ b/var/www/modules/settings/Flask_settings.py @@ -14,6 +14,7 @@ import json import datetime import git_status +import d4 # ============ VARIABLES ============ import Flask_config @@ -21,8 +22,6 @@ import Flask_config app = Flask_config.app baseUrl = Flask_config.baseUrl r_serv_db = Flask_config.r_serv_db -max_preview_char = Flask_config.max_preview_char -max_preview_modal = Flask_config.max_preview_modal REPO_ORIGIN = Flask_config.REPO_ORIGIN dict_update_description = Flask_config.dict_update_description email_regex = Flask_config.email_regex @@ -274,5 +273,20 @@ def get_background_update_stats_json(): else: return jsonify({}) +@settings.route("/settings/passivedns", methods=['GET']) +@login_required +@login_read_only +def passive_dns(): + passivedns_enabled = d4.is_passive_dns_enabled() + return render_template("passive_dns.html", passivedns_enabled=passivedns_enabled) + +@settings.route("/settings/passivedns/change_state", methods=['GET']) +@login_required +@login_admin +def passive_dns_change_state(): + new_state = request.args.get('state') == 'enable' + passivedns_enabled = d4.change_passive_dns_state(new_state) + return redirect(url_for('settings.passive_dns')) + # ========= REGISTRATION ========= app.register_blueprint(settings, url_prefix=baseUrl) diff --git a/var/www/modules/settings/templates/passive_dns.html b/var/www/modules/settings/templates/passive_dns.html new file mode 100644 index 00000000..0b21c82f --- /dev/null +++ b/var/www/modules/settings/templates/passive_dns.html @@ -0,0 +1,98 @@ + + + + + Passive DNS - AIL + + + + + + + + + + + + + + + + + + {% include 'nav_bar.html' %} + +
+
+ + {% include 'settings/menu_sidebar.html' %} + +
+ +
+ + D4 project + +
+ +

+ Passive DNS or pDNS is a service which records domain name system server (DNS) answers to DNS client requests.
+ In order to see the evolution of records over time, a history is recorded.
+ Various sources can be used to build a large sensor network.
+
+ Enabling the D4 passive DNS sensor in AIL will contribute resolved domains and host to the global Passive DNS community operated by + + CIRCL.lu + +
+
+ (if you want to have access to the global Passive DNS community + + https://www.circl.lu/services/passive-dns + + ) +

+ + + {% if passivedns_enabled %} + + + + {% else %} + + + + {% endif %} +
+
+
+ + + + + + diff --git a/var/www/static/image/d4-logo.png b/var/www/static/image/d4-logo.png new file mode 100644 index 00000000..63fe5a09 Binary files /dev/null and b/var/www/static/image/d4-logo.png differ diff --git a/var/www/templates/settings/menu_sidebar.html b/var/www/templates/settings/menu_sidebar.html index ce63965f..b88f7c9e 100644 --- a/var/www/templates/settings/menu_sidebar.html +++ b/var/www/templates/settings/menu_sidebar.html @@ -17,6 +17,17 @@ + +