chg: [passiveDns D4 Client] add passiveDns D4 Client

.gitignore

@@ -46,6 +46,7 @@ configs/core.cfg.backup
 configs/update.cfg
 update/current_version
 files
+configs/d4client_passiveDNS_conf/uuid
 
 # Trackers
 bin/trackers/yara/custom-rules/*

bin/DomClassifier.py

@@ -9,13 +9,18 @@ The DomClassifier modules extract and classify Internet domains/hostnames/IP add
 the out output of the Global module.
 
 """
+import os
+import sys
 import time
-from packages import Paste
 from pubsublogger import publisher
 
 import DomainClassifier.domainclassifier
 from Helper import Process
 
+sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
+import d4
+import item_basic
+
 
 def main():
     publisher.port = 6380
@@ -35,35 +40,42 @@ def main():
 
     while True:
         try:
-            message = p.get_from_set()
+            item_id = p.get_from_set()
 
-            if message is not None:
+            if item_id is None:
-                PST = Paste.Paste(message)
-            else:
                 publisher.debug("Script DomClassifier is idling 1s")
                 time.sleep(1)
                 continue
-            paste = PST.get_p_content()
-            mimetype = PST._get_p_encoding()
 
-            if mimetype == "text/plain":
+            item_content = item_basic.get_item_content(item_id)
-                c.text(rawtext=paste)
+            mimetype = item_basic.get_item_mimetype(item_id)
+            item_basename = item_basic.get_basename(item_id)
+            item_source = item_basic.get_source(item_id)
+            item_date = item_basic.get_item_date(item_id)
+
+            if mimetype.split('/')[0] == "text":
+                c.text(rawtext=item_content)
                 c.potentialdomain()
-                c.validdomain(rtype=['A'], extended=True)
+                c.validdomain(rtype=['A'], passive_dns=True, extended=False)
+                print(c.vdomain)
+
+                if c.vdomain and d4.is_passive_dns_enabled():
+                    for dns_record in c.vdomain:
+                        p.populate_set_out(dns_record)
+
                 localizeddomains = c.include(expression=cc_tld)
                 if localizeddomains:
                     print(localizeddomains)
-                    publisher.warning('DomainC;{};{};{};Checked {} located in {};{}'.format(
+                    publisher.warning(f"DomainC;{item_source};{item_date};{item_basename};Checked {localizeddomains} located in {cc_tld};{item_id}")
-                        PST.p_source, PST.p_date, PST.p_name, localizeddomains, cc_tld, PST.p_rel_path))
                 localizeddomains = c.localizedomain(cc=cc)
+
                 if localizeddomains:
                     print(localizeddomains)
-                    publisher.warning('DomainC;{};{};{};Checked {} located in {};{}'.format(
+                    publisher.warning(f"DomainC;{item_source};{item_date};{item_basename};Checked {localizeddomains} located in {cc};{item_id}")
-                        PST.p_source, PST.p_date, PST.p_name, localizeddomains, cc, PST.p_rel_path))
+
         except IOError:
-            print("CRC Checksum Failed on :", PST.p_rel_path)
+            print("CRC Checksum Failed on :", item_id)
-            publisher.error('Duplicate;{};{};{};CRC Checksum Failed'.format(
+            publisher.error(f"Duplicate;{item_source};{item_date};{item_basename};CRC Checksum Failed")
-                PST.p_source, PST.p_date, PST.p_name))
 
 if __name__ == "__main__":
     main()

bin/LAUNCH.sh

@@ -152,6 +152,8 @@ function launching_scripts {
     sleep 0.1
     screen -S "Script_AIL" -X screen -t "Crawler_manager" bash -c "cd ${AIL_BIN}/core; ${ENV_PY} ./Crawler_manager.py; read x"
     sleep 0.1
+    screen -S "Script_AIL" -X screen -t "D4_client" bash -c "cd ${AIL_BIN}/core; ${ENV_PY} ./D4_client.py; read x"
+    sleep 0.1
 
 
     screen -S "Script_AIL" -X screen -t "ModuleInformation" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./ModulesInformationV2.py -k 0 -c 1; read x"

bin/core/D4_client.py

@@ -0,0 +1,52 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+"""
+The D4_Client Module
+============================
+
+The D4_Client modules send all DNS records to a D4 Server.
+Data produced by D4 sensors are ingested into
+a Passive DNS server which can be queried later to search for the Passive DNS records.
+"""
+
+import os
+import sys
+import time
+from pubsublogger import publisher
+sys.path.append(os.environ['AIL_BIN'])
+from Helper import Process
+
+sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
+import ConfigLoader
+import d4
+
+# # TODO: lauch me in core screen
+# # TODO: check if already launched in core screen
+
+if __name__ == '__main__':
+    publisher.port = 6380
+    publisher.channel = "Script"
+
+    config_section = 'D4_client'
+    p = Process(config_section)
+    publisher.info("""D4_client is Running""")
+
+    last_refresh = time.time()
+    d4_client = d4.create_d4_client()
+
+    while True:
+        if last_refresh < d4.get_config_last_update_time():
+            d4_client = d4.create_d4_client()
+            last_refresh = time.time()
+            print('D4 Client: config updated')
+
+        dns_record = p.get_from_set()
+        if dns_record is None:
+            publisher.debug("Script D4_client is idling 1s")
+            time.sleep(1)
+            continue
+
+        if d4_client:
+            # Send DNS Record to D4Server
+            d4_client.send_manual_data(dns_record)

bin/lib/d4.py

@@ -0,0 +1,74 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+import os
+import sys
+import time
+import redis
+import d4_pyclient
+
+sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
+import ConfigLoader
+
+config_loader = ConfigLoader.ConfigLoader()
+r_serv_db = config_loader.get_redis_conn("ARDB_DB")
+r_cache = config_loader.get_redis_conn("Redis_Cache")
+config_loader = None
+
+def get_ail_uuid():
+    return r_serv_db.get('ail:uuid')
+
+def get_d4_client_config_dir():
+    return os.path.join(os.environ['AIL_HOME'], 'configs', 'd4client_passiveDNS_conf')
+
+def create_d4_config_file(filename, content):
+    if not os.path.isfile(filename):
+        with open(filename, 'a') as f:
+            f.write(content)
+
+def get_d4_client_config():
+    d4_client_config = get_d4_client_config_dir()
+    filename = os.path.join(d4_client_config, 'uuid')
+    if not os.path.isfile(filename):
+        create_d4_config_file(filename, get_ail_uuid())
+    return d4_client_config
+
+def is_passive_dns_enabled(cache=True):
+    if cache:
+        res = r_cache.get('d4:passivedns:enabled')
+        if res is None:
+            res = r_serv_db.hget('d4:passivedns', 'enabled') == 'True'
+            r_cache.set('d4:passivedns:enabled', res)
+            return res
+        else:
+            return res == 'True'
+    else:
+        return r_serv_db.hget('d4:passivedns', 'enabled') == 'True'
+
+def change_passive_dns_state(new_state):
+    old_state = is_passive_dns_enabled(cache=False)
+    if old_state != new_state:
+        r_serv_db.hset('d4:passivedns', 'enabled', bool(new_state))
+        r_cache.set('d4:passivedns:enabled', bool(new_state))
+        update_time = time.time()
+        r_serv_db.hset('d4:passivedns', 'update_time', update_time)
+        r_cache.set('d4:passivedns:last_update_time', update_time)
+        return True
+    return False
+
+def get_config_last_update_time():
+    last_update_time = r_cache.get('d4:passivedns:last_update_time')
+    if not last_update_time:
+        last_update_time = r_serv_db.hget('d4:passivedns', 'update_time')
+        if not last_update_time:
+            last_update_time = 0
+        last_update_time = float(last_update_time)
+        r_cache.set('d4:passivedns:last_update_time', last_update_time)
+    return float(last_update_time)
+
+def create_d4_client():
+    if is_passive_dns_enabled():
+        d4_client = d4_pyclient.D4Client(get_d4_client_config(), False)
+        return d4_client
+    else:
+        return None

bin/lib/item_basic.py

@@ -5,6 +5,8 @@ import os
 import sys
 import gzip
 
+import magic
+
 sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
 import ConfigLoader
 
@@ -35,6 +37,9 @@ def get_item_date(item_id, add_separator=False):
     else:
         return '{}{}{}'.format(l_directory[-4], l_directory[-3], l_directory[-2])
 
+def get_basename(item_id):
+    return os.path.basename(item_id)
+
 def get_source(item_id):
     return item_id.split('/')[-5]
 
@@ -63,6 +68,9 @@ def get_item_content(item_id):
             item_content = ''
     return str(item_content)
 
+def get_item_mimetype(item_id):
+    return magic.from_buffer(get_item_content(item_id), mime=True)
+
 #### TREE CHILD/FATHER ####
 def is_father(item_id):
     return r_serv_metadata.exists('paste_children:{}'.format(item_id))

bin/packages/modules.cfg

@@ -21,6 +21,10 @@ subscribe = Redis_Global
 
 [DomClassifier]
 subscribe = Redis_Global
+publish = Redis_D4_client
+
+[D4_client]
+subscribe = Redis_D4_client
 
 [TermTrackerMod]
 subscribe = Redis_Global

configs/d4client_passiveDNS_conf/destination

@@ -0,0 +1 @@
+d4pdns.circl.lu:4443

configs/d4client_passiveDNS_conf/key

@@ -0,0 +1 @@
+ail passivedns sensor key

configs/d4client_passiveDNS_conf/snaplen

@@ -0,0 +1 @@
+4096

configs/d4client_passiveDNS_conf/source

@@ -0,0 +1 @@
+stdin

configs/d4client_passiveDNS_conf/type

@@ -0,0 +1 @@
+8

configs/d4client_passiveDNS_conf/version

@@ -0,0 +1 @@
+1

var/www/modules/settings/Flask_settings.py

@@ -14,6 +14,7 @@ import json
 import datetime
 
 import git_status
+import d4
 
 # ============ VARIABLES ============
 import Flask_config
@@ -21,8 +22,6 @@ import Flask_config
 app = Flask_config.app
 baseUrl = Flask_config.baseUrl
 r_serv_db = Flask_config.r_serv_db
-max_preview_char = Flask_config.max_preview_char
-max_preview_modal = Flask_config.max_preview_modal
 REPO_ORIGIN = Flask_config.REPO_ORIGIN
 dict_update_description = Flask_config.dict_update_description
 email_regex = Flask_config.email_regex
@@ -274,5 +273,20 @@ def get_background_update_stats_json():
     else:
         return jsonify({})
 
+@settings.route("/settings/passivedns", methods=['GET'])
+@login_required
+@login_read_only
+def passive_dns():
+    passivedns_enabled = d4.is_passive_dns_enabled()
+    return render_template("passive_dns.html", passivedns_enabled=passivedns_enabled)
+
+@settings.route("/settings/passivedns/change_state", methods=['GET'])
+@login_required
+@login_admin
+def passive_dns_change_state():
+    new_state = request.args.get('state') == 'enable'
+    passivedns_enabled = d4.change_passive_dns_state(new_state)
+    return redirect(url_for('settings.passive_dns'))
+
 # ========= REGISTRATION =========
 app.register_blueprint(settings, url_prefix=baseUrl)

var/www/modules/settings/templates/passive_dns.html

@@ -0,0 +1,98 @@
+<!DOCTYPE html>
+<html>
+
+<head>
+  <title>Passive DNS - AIL</title>
+	<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
+
+  <!-- Core CSS -->
+	<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/dataTables.bootstrap4.min.css') }}" rel="stylesheet">
+
+  <!-- JS -->
+	<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
+
+</head>
+<body>
+
+  {% include 'nav_bar.html' %}
+
+	<div class="container-fluid">
+		<div class="row">
+
+			{% include 'settings/menu_sidebar.html' %}
+
+			<div class="col-12 col-lg-10" id="core_content">
+
+        <div class="d-flex justify-content-center my-4">
+          <a href="https://d4-project.org/">
+            <img src="{{ url_for('static', filename='image/d4-logo.png')}}" alt="D4 project">
+          </a>
+        </div>
+
+        <p class="lead px-4">
+         Passive DNS or pDNS is a service which records domain name system server (DNS) answers to DNS client requests.<br>
+         In order to see the evolution of records over time, a history is recorded.<br>
+         Various sources can be used to build a large sensor network.<br>
+         <br>
+         Enabling the D4 passive DNS sensor in AIL will contribute resolved domains and host to the global Passive DNS community operated by
+         <a href="https://www.circl.lu/">
+            CIRCL.lu
+         </a>
+         <br>
+         <br>
+         (if you want to have access to the global Passive DNS community
+         <a href="https://www.circl.lu/services/passive-dns/">
+            https://www.circl.lu/services/passive-dns
+         </a>
+         )
+        </p>
+
+
+        {% if passivedns_enabled %}
+          <a href="{{ url_for('settings.passive_dns_change_state') }}?state=disable">
+            <button class="btn btn-danger mx-4 my-2">
+              Disable D4 Client
+            </button>
+          </a>
+        {% else %}
+          <a href="{{ url_for('settings.passive_dns_change_state') }}?state=enable">
+            <button class="btn btn-primary mx-4 my-2">
+              Enable D4 Client
+            </button>
+          </a>
+        {% endif %}
+    	</div>
+		</div>
+	</div>
+
+</body>
+
+<script>
+$(document).ready(function(){
+  $("#nav_settings").addClass("active");
+	$("#passive_dns").removeClass("text-muted");
+} );
+
+function toggle_sidebar(){
+	if($('#nav_menu').is(':visible')){
+		$('#nav_menu').hide();
+		$('#side_menu').removeClass('border-right')
+		$('#side_menu').removeClass('col-lg-2')
+		$('#core_content').removeClass('col-lg-10')
+	}else{
+		$('#nav_menu').show();
+		$('#side_menu').addClass('border-right')
+		$('#side_menu').addClass('col-lg-2')
+		$('#core_content').addClass('col-lg-10')
+	}
+}
+
+</script>
+
+</html>

var/www/templates/settings/menu_sidebar.html

@@ -17,6 +17,17 @@
       	      </a>
       	    </li>
           </ul>
+          <h5 class="d-flex text-muted w-100" id="nav_settings">
+      		  <span>Settings</span>
+      		</h5>
+          <ul class="nav flex-md-column flex-row navbar-nav justify-content-between w-100"> <!--nav-pills-->
+      	    <li class="nav-item">
+      	      <a class="nav-link" href="{{url_for('settings.passive_dns')}}" id="passive_dns">
+      	        <img src="{{ url_for('static', filename='image/d4-logo.png')}}" alt="D4 project" style="width:25px;">
+      	        <span>Passive DNS</span>
+      	      </a>
+      	    </li>
+          </ul>
           <h5 class="d-flex text-muted w-100 py-2" id="nav_my_profile">
             <span>My Profile</span>
           </h5>