CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

chg: [passiveDns D4 Client] add passiveDns D4 Client

pull/559/head
Terrtia 2021-03-31 11:25:09 +02:00
parent b37d05842b
commit 9974823464
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
17 changed files with 301 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -46,6 +46,7 @@ configs/core.cfg.backup
configs/update.cfg
update/current_version
files
configs/d4client_passiveDNS_conf/uuid
# Trackers
bin/trackers/yara/custom-rules/*

46
bin/DomClassifier.py
View File

@ -9,13 +9,18 @@ The DomClassifier modules extract and classify Internet domains/hostnames/IP add
the out output of the Global module.
"""
import os
import sys
import time
from packages import Paste
from pubsublogger import publisher
import DomainClassifier.domainclassifier
from Helper import Process
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
import d4
import item_basic
def main():
publisher.port = 6380
@ -35,35 +40,42 @@ def main():
while True:
try:
message = p.get_from_set()
item_id = p.get_from_set()
if message is not None:
PST = Paste.Paste(message)
else:
if item_id is None:
publisher.debug("Script DomClassifier is idling 1s")
time.sleep(1)
continue
paste = PST.get_p_content()
mimetype = PST._get_p_encoding()
if mimetype == "text/plain":
c.text(rawtext=paste)
item_content = item_basic.get_item_content(item_id)
mimetype = item_basic.get_item_mimetype(item_id)
item_basename = item_basic.get_basename(item_id)
item_source = item_basic.get_source(item_id)
item_date = item_basic.get_item_date(item_id)
if mimetype.split('/')[0] == "text":
c.text(rawtext=item_content)
c.potentialdomain()
c.validdomain(rtype=['A'], extended=True)
c.validdomain(rtype=['A'], passive_dns=True, extended=False)
print(c.vdomain)
if c.vdomain and d4.is_passive_dns_enabled():
for dns_record in c.vdomain:
p.populate_set_out(dns_record)
localizeddomains = c.include(expression=cc_tld)
if localizeddomains:
print(localizeddomains)
publisher.warning('DomainC;{};{};{};Checked {} located in {};{}'.format(
PST.p_source, PST.p_date, PST.p_name, localizeddomains, cc_tld, PST.p_rel_path))
publisher.warning(f"DomainC;{item_source};{item_date};{item_basename};Checked {localizeddomains} located in {cc_tld};{item_id}")
localizeddomains = c.localizedomain(cc=cc)
if localizeddomains:
print(localizeddomains)
publisher.warning('DomainC;{};{};{};Checked {} located in {};{}'.format(
PST.p_source, PST.p_date, PST.p_name, localizeddomains, cc, PST.p_rel_path))
publisher.warning(f"DomainC;{item_source};{item_date};{item_basename};Checked {localizeddomains} located in {cc};{item_id}")
except IOError:
print("CRC Checksum Failed on :", PST.p_rel_path)
publisher.error('Duplicate;{};{};{};CRC Checksum Failed'.format(
PST.p_source, PST.p_date, PST.p_name))
print("CRC Checksum Failed on :", item_id)
publisher.error(f"Duplicate;{item_source};{item_date};{item_basename};CRC Checksum Failed")
if __name__ == "__main__":
main()

2
bin/LAUNCH.sh
View File

@ -152,6 +152,8 @@ function launching_scripts {
sleep 0.1
screen -S "Script_AIL" -X screen -t "Crawler_manager" bash -c "cd ${AIL_BIN}/core; ${ENV_PY} ./Crawler_manager.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "D4_client" bash -c "cd ${AIL_BIN}/core; ${ENV_PY} ./D4_client.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "ModuleInformation" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./ModulesInformationV2.py -k 0 -c 1; read x"

52
bin/core/D4_client.py Executable file
View File

@ -0,0 +1,52 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
"""
The D4_Client Module
============================
The D4_Client modules send all DNS records to a D4 Server.
Data produced by D4 sensors are ingested into
a Passive DNS server which can be queried later to search for the Passive DNS records.
"""
import os
import sys
import time
from pubsublogger import publisher
sys.path.append(os.environ['AIL_BIN'])
from Helper import Process
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
import ConfigLoader
import d4
# # TODO: lauch me in core screen
# # TODO: check if already launched in core screen
if __name__ == '__main__':
publisher.port = 6380
publisher.channel = "Script"
config_section = 'D4_client'
p = Process(config_section)
publisher.info("""D4_client is Running""")
last_refresh = time.time()
d4_client = d4.create_d4_client()
while True:
if last_refresh < d4.get_config_last_update_time():
d4_client = d4.create_d4_client()
last_refresh = time.time()
print('D4 Client: config updated')
dns_record = p.get_from_set()
if dns_record is None:
publisher.debug("Script D4_client is idling 1s")
time.sleep(1)
continue
if d4_client:
# Send DNS Record to D4Server
d4_client.send_manual_data(dns_record)

74
bin/lib/d4.py Executable file
View File

@ -0,0 +1,74 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
import os
import sys
import time
import redis
import d4_pyclient
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
import ConfigLoader
config_loader = ConfigLoader.ConfigLoader()
r_serv_db = config_loader.get_redis_conn("ARDB_DB")
r_cache = config_loader.get_redis_conn("Redis_Cache")
config_loader = None
def get_ail_uuid():
return r_serv_db.get('ail:uuid')
def get_d4_client_config_dir():
return os.path.join(os.environ['AIL_HOME'], 'configs', 'd4client_passiveDNS_conf')
def create_d4_config_file(filename, content):
if not os.path.isfile(filename):
with open(filename, 'a') as f:
f.write(content)
def get_d4_client_config():
d4_client_config = get_d4_client_config_dir()
filename = os.path.join(d4_client_config, 'uuid')
if not os.path.isfile(filename):
create_d4_config_file(filename, get_ail_uuid())
return d4_client_config
def is_passive_dns_enabled(cache=True):
if cache:
res = r_cache.get('d4:passivedns:enabled')
if res is None:
res = r_serv_db.hget('d4:passivedns', 'enabled') == 'True'
r_cache.set('d4:passivedns:enabled', res)
return res
else:
return res == 'True'
else:
return r_serv_db.hget('d4:passivedns', 'enabled') == 'True'
def change_passive_dns_state(new_state):
old_state = is_passive_dns_enabled(cache=False)
if old_state != new_state:
r_serv_db.hset('d4:passivedns', 'enabled', bool(new_state))
r_cache.set('d4:passivedns:enabled', bool(new_state))
update_time = time.time()
r_serv_db.hset('d4:passivedns', 'update_time', update_time)
r_cache.set('d4:passivedns:last_update_time', update_time)
return True
return False
def get_config_last_update_time():
last_update_time = r_cache.get('d4:passivedns:last_update_time')
if not last_update_time:
last_update_time = r_serv_db.hget('d4:passivedns', 'update_time')
if not last_update_time:
last_update_time = 0
last_update_time = float(last_update_time)
r_cache.set('d4:passivedns:last_update_time', last_update_time)
return float(last_update_time)
def create_d4_client():
if is_passive_dns_enabled():
d4_client = d4_pyclient.D4Client(get_d4_client_config(), False)
return d4_client
else:
return None

8
bin/lib/item_basic.py
View File

@ -5,6 +5,8 @@ import os
import sys
import gzip
import magic
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader
@ -35,6 +37,9 @@ def get_item_date(item_id, add_separator=False):
else:
return '{}{}{}'.format(l_directory[-4], l_directory[-3], l_directory[-2])
def get_basename(item_id):
return os.path.basename(item_id)
def get_source(item_id):
return item_id.split('/')[-5]
@ -63,6 +68,9 @@ def get_item_content(item_id):
item_content = ''
return str(item_content)
def get_item_mimetype(item_id):
return magic.from_buffer(get_item_content(item_id), mime=True)
#### TREE CHILD/FATHER ####
def is_father(item_id):
return r_serv_metadata.exists('paste_children:{}'.format(item_id))

4
bin/packages/modules.cfg
View File

@ -21,6 +21,10 @@ subscribe = Redis_Global
[DomClassifier]
subscribe = Redis_Global
publish = Redis_D4_client
[D4_client]
subscribe = Redis_D4_client
[TermTrackerMod]
subscribe = Redis_Global

1
configs/d4client_passiveDNS_conf/destination Normal file
View File

@ -0,0 +1 @@
d4pdns.circl.lu:4443

1
configs/d4client_passiveDNS_conf/key Normal file
View File

@ -0,0 +1 @@
ail passivedns sensor key

1
configs/d4client_passiveDNS_conf/snaplen Normal file
View File

@ -0,0 +1 @@
4096

1
configs/d4client_passiveDNS_conf/source Normal file
View File

@ -0,0 +1 @@
stdin

1
configs/d4client_passiveDNS_conf/type Normal file
View File

@ -0,0 +1 @@
8

1
configs/d4client_passiveDNS_conf/version Normal file
View File

@ -0,0 +1 @@
1

18
var/www/modules/settings/Flask_settings.py
View File

@ -14,6 +14,7 @@ import json
import datetime
import git_status
import d4
# ============ VARIABLES ============
import Flask_config
@ -21,8 +22,6 @@ import Flask_config
app = Flask_config.app
baseUrl = Flask_config.baseUrl
r_serv_db = Flask_config.r_serv_db
max_preview_char = Flask_config.max_preview_char
max_preview_modal = Flask_config.max_preview_modal
REPO_ORIGIN = Flask_config.REPO_ORIGIN
dict_update_description = Flask_config.dict_update_description
email_regex = Flask_config.email_regex
@ -274,5 +273,20 @@ def get_background_update_stats_json():
else:
return jsonify({})
@settings.route("/settings/passivedns", methods=['GET'])
@login_required
@login_read_only
def passive_dns():
passivedns_enabled = d4.is_passive_dns_enabled()
return render_template("passive_dns.html", passivedns_enabled=passivedns_enabled)
@settings.route("/settings/passivedns/change_state", methods=['GET'])
@login_required
@login_admin
def passive_dns_change_state():
new_state = request.args.get('state') == 'enable'
passivedns_enabled = d4.change_passive_dns_state(new_state)
return redirect(url_for('settings.passive_dns'))
# ========= REGISTRATION =========
app.register_blueprint(settings, url_prefix=baseUrl)

98
var/www/modules/settings/templates/passive_dns.html Normal file
View File

@ -0,0 +1,98 @@
<!DOCTYPE html>
<html>
<head>
<title>Passive DNS - AIL</title>
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dataTables.bootstrap4.min.css') }}" rel="stylesheet">
<!-- JS -->
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
</head>
<body>
{% include 'nav_bar.html' %}
<div class="container-fluid">
<div class="row">
{% include 'settings/menu_sidebar.html' %}
<div class="col-12 col-lg-10" id="core_content">
<div class="d-flex justify-content-center my-4">
<a href="https://d4-project.org/">
<img src="{{ url_for('static', filename='image/d4-logo.png')}}" alt="D4 project">
</a>
</div>
<p class="lead px-4">
Passive DNS or pDNS is a service which records domain name system server (DNS) answers to DNS client requests.<br>
In order to see the evolution of records over time, a history is recorded.<br>
Various sources can be used to build a large sensor network.<br>
<br>
Enabling the D4 passive DNS sensor in AIL will contribute resolved domains and host to the global Passive DNS community operated by
<a href="https://www.circl.lu/">
CIRCL.lu
</a>
<br>
<br>
(if you want to have access to the global Passive DNS community
<a href="https://www.circl.lu/services/passive-dns/">
https://www.circl.lu/services/passive-dns
</a>
)
</p>
{% if passivedns_enabled %}
<a href="{{ url_for('settings.passive_dns_change_state') }}?state=disable">
<button class="btn btn-danger mx-4 my-2">
Disable D4 Client
</button>
</a>
{% else %}
<a href="{{ url_for('settings.passive_dns_change_state') }}?state=enable">
<button class="btn btn-primary mx-4 my-2">
Enable D4 Client
</button>
</a>
{% endif %}
</div>
</div>
</div>
</body>
<script>
$(document).ready(function(){
$("#nav_settings").addClass("active");
$("#passive_dns").removeClass("text-muted");
} );
function toggle_sidebar(){
if($('#nav_menu').is(':visible')){
$('#nav_menu').hide();
$('#side_menu').removeClass('border-right')
$('#side_menu').removeClass('col-lg-2')
$('#core_content').removeClass('col-lg-10')
}else{
$('#nav_menu').show();
$('#side_menu').addClass('border-right')
$('#side_menu').addClass('col-lg-2')
$('#core_content').addClass('col-lg-10')
}
}
</script>
</html>

BIN
var/www/static/image/d4-logo.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 17 KiB

11
var/www/templates/settings/menu_sidebar.html
View File

@ -17,6 +17,17 @@
</a>
</li>
</ul>
<h5 class="d-flex text-muted w-100" id="nav_settings">
<span>Settings</span>
</h5>
<ul class="nav flex-md-column flex-row navbar-nav justify-content-between w-100"> <!--nav-pills-->
<li class="nav-item">
<a class="nav-link" href="{{url_for('settings.passive_dns')}}" id="passive_dns">
<img src="{{ url_for('static', filename='image/d4-logo.png')}}" alt="D4 project" style="width:25px;">
<span>Passive DNS</span>
</a>
</li>
</ul>
<h5 class="d-flex text-muted w-100 py-2" id="nav_my_profile">
<span>My Profile</span>
</h5>