analyzer-d4-passivedns/bin/pdns-cof-server.py

183 lines
17 KiB
Python
Raw Normal View History

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# A Passive DNS COF compliant passive DNS server for the analyzer-d4-passivedns
#
# The output format is compliant with Passive DNS - Common Output Format
#
# https://tools.ietf.org/html/draft-dulaunoy-dnsop-passive-dns-cof
#
# This software is part of the D4 project.
#
# The software is released under the GNU Affero General Public version 3.
#
# Copyright (c) 2013-2019 Alexandre Dulaunoy - a@foo.be
# Copyright (c) 2019 Computer Incident Response Center Luxembourg (CIRCL)
from datetime import date
import tornado.escape
import tornado.ioloop
import tornado.web
import iptools
import redis
import json
2019-06-11 22:41:32 +02:00
import os
rrset = [{"Reference": "[RFC1035]", "Type": "A", "Value": "1", "Meaning": "a host address", "Template": "", "Registration Date": ""}, {"Reference": "[RFC1035]", "Type": "NS", "Value": "2", "Meaning": "an authoritative name server", "Template": "", "Registration Date": ""}, {"Reference": "[RFC1035]", "Type": "MD", "Value": "3", "Meaning": "a mail destination (OBSOLETE - use MX)", "Template": "", "Registration Date": ""}, {"Reference": "[RFC1035]", "Type": "MF", "Value": "4", "Meaning": "a mail forwarder (OBSOLETE - use MX)", "Template": "", "Registration Date": ""}, {"Reference": "[RFC1035]", "Type": "CNAME", "Value": "5", "Meaning": "the canonical name for an alias", "Template": "", "Registration Date": ""}, {"Reference": "[RFC1035]", "Type": "SOA", "Value": "6", "Meaning": "marks the start of a zone of authority", "Template": "", "Registration Date": ""}, {"Reference": "[RFC1035]", "Type": "MB", "Value": "7", "Meaning": "a mailbox domain name (EXPERIMENTAL)", "Template": "", "Registration Date": ""}, {"Reference": "[RFC1035]", "Type": "MG", "Value": "8", "Meaning": "a mail group member (EXPERIMENTAL)", "Template": "", "Registration Date": ""}, {"Reference": "[RFC1035]", "Type": "MR", "Value": "9", "Meaning": "a mail rename domain name (EXPERIMENTAL)", "Template": "", "Registration Date": ""}, {"Reference": "[RFC1035]", "Type": "NULL", "Value": "10", "Meaning": "a null RR (EXPERIMENTAL)", "Template": "", "Registration Date": ""}, {"Reference": "[RFC1035]", "Type": "WKS", "Value": "11", "Meaning": "a well known service description", "Template": "", "Registration Date": ""}, {"Reference": "[RFC1035]", "Type": "PTR", "Value": "12", "Meaning": "a domain name pointer", "Template": "", "Registration Date": ""}, {"Reference": "[RFC1035]", "Type": "HINFO", "Value": "13", "Meaning": "host information", "Template": "", "Registration Date": ""}, {"Reference": "[RFC1035]", "Type": "MINFO", "Value": "14", "Meaning": "mailbox or mail list information", "Template": "", "Registration Date": ""}, {"Reference": "[RFC1035]", "Type": "MX", "Value": "15", "Meaning": "mail exchange", "Template": "", "Registration Date": ""}, {"Reference": "[RFC1035]", "Type": "TXT", "Value": "16", "Meaning": "text strings", "Template": "", "Registration Date": ""}, {"Reference": "[RFC1183]", "Type": "RP", "Value": "17", "Meaning": "for Responsible Person", "Template": "", "Registration Date": ""}, {"Reference": "[RFC1183][RFC5864]", "Type": "AFSDB", "Value": "18", "Meaning": "for AFS Data Base location", "Template": "", "Registration Date": ""}, {"Reference": "[RFC1183]", "Type": "X25", "Value": "19", "Meaning": "for X.25 PSDN address", "Template": "", "Registration Date": ""}, {"Reference": "[RFC1183]", "Type": "ISDN", "Value": "20", "Meaning": "for ISDN address", "Template": "", "Registration Date": ""}, {"Reference": "[RFC1183]", "Type": "RT", "Value": "21", "Meaning": "for Route Through", "Template": "", "Registration Date": ""}, {"Reference": "[RFC1706]", "Type": "NSAP", "Value": "22", "Meaning": "for NSAP address, NSAP style A record", "Template": "", "Registration Date": ""}, {"Reference": "[RFC1348][RFC1637][RFC1706]", "Type": "NSAP-PTR", "Value": "23", "Meaning": "for domain name pointer, NSAP style", "Template": "", "Registration Date": ""}, {"Reference": "[RFC4034][RFC3755][RFC2535][RFC2536][RFC2537][RFC2931][RFC3110][RFC3008]", "Type": "SIG", "Value": "24", "Meaning": "for security signature", "Template": "", "Registration Date": ""}, {"Reference": "[RFC4034][RFC3755][RFC2535][RFC2536][RFC2537][RFC2539][RFC3008][RFC3110]", "Type": "KEY", "Value": "25", "Meaning": "for security key", "Template": "", "Registration Date": ""}, {"Reference": "[RFC2163]", "Type": "PX", "Value": "26", "Meaning": "X.400 mail mapping information", "Template": "", "Registration Date": ""}, {"Reference": "[RFC1712]", "Type": "GPOS", "Value": "27", "Meaning": "Geographical Position", "Template": "", "Registration Date": ""}, {"Reference": "[RFC3596]", "Type": "AAAA", "Value": "28", "Meaning": "IP6 Address", "Template": "", "Registration Date": ""}, {"Reference": "[RFC1876]"
2019-06-11 22:41:32 +02:00
analyzer_redis_host = os.getenv('D4_ANALYZER_REDIS_HOST', '127.0.0.1')
analyzer_redis_port = int(os.getenv('D4_ANALYZER_REDIS_PORT', 6400))
r = redis.StrictRedis(host=analyzer_redis_host, port=analyzer_redis_port, db=0)
rrset_supported = ['1','2','5','15','16','28','33','46']
expiring_type = ['16']
origin = "origin not configured"
def getFirstSeen(t1 = None, t2 = None):
if t1 is None or t2 is None:
return False
2019-02-08 21:56:04 +01:00
rec = "s:{}:{}".format(t1.lower(),t2.lower())
for rr in rrset:
if (rr['Value']) is not None and rr['Value'] in rrset_supported:
qrec = rec+":{}".format(rr['Value'])
recget = r.get(qrec)
if recget is not None:
return int(recget.decode(encoding='UTF-8'))
def getLastSeen(t1 = None, t2 = None):
if t1 is None or t2 is None:
return False
2019-02-08 21:56:04 +01:00
rec = "l:{}:{}".format(t1.lower(),t2.lower())
for rr in rrset:
if (rr['Value']) is not None and rr['Value'] in rrset_supported:
qrec = rec+":{}".format(rr['Value'])
recget = r.get(qrec)
if recget is not None:
return int(recget.decode(encoding='UTF-8'))
def getCount(t1 = None, t2 = None):
if t1 is None or t2 is None:
return False
2019-02-08 21:56:04 +01:00
rec = "o:{}:{}".format(t1.lower(),t2.lower())
for rr in rrset:
if (rr['Value']) is not None and rr['Value'] in rrset_supported:
qrec = rec+":{}".format(rr['Value'])
recget = r.get(qrec)
if recget is not None:
return int(recget.decode(encoding='UTF-8'))
def getRecord(t = None):
if t is None:
return False
rrfound = []
for rr in rrset:
if (rr['Value']) is not None and rr['Value'] in rrset_supported:
2019-02-08 21:56:04 +01:00
rec = "r:{}:{}".format(t,rr['Value'])
setsize = r.scard(rec)
if setsize < 200:
rs = r.smembers(rec)
else:
#rs = r.srandmember(rec, number=300)
rs = False
if rs:
for v in rs:
rrval = {}
rdata = v.decode(encoding='UTF-8').strip()
rrval['time_first'] = getFirstSeen(t1=t, t2=rdata)
rrval['time_last'] = getLastSeen(t1=t, t2=rdata)
if rrval['time_first'] is None:
break
rrval['count'] = getCount(t1=t, t2=rdata)
rrval['rrtype'] = rr['Type']
rrval['rrname'] = t
rrval['rdata'] = rdata
if origin:
rrval['origin'] = origin
rrfound.append(rrval)
return rrfound
def getAssociatedRecords(rdata = None):
if rdata is None:
return False
rec = "v:"+rdata.lower()
records = []
for rr in rrset:
if (rr['Value']) is not None and rr['Value'] in rrset_supported:
qrec = rec+":{}".format(rr['Value'])
if r.smembers(qrec):
for v in r.smembers(qrec):
records.append(v.decode(encoding='UTF-8'))
return records
def RemDuplicate(d = None):
if d is None:
return False
outd = [dict(t) for t in set([tuple(o.items()) for o in d])]
return outd
def JsonQOF(rrfound = None, RemoveDuplicate=True):
if rrfound is None:
return False
rrqof = ""
if RemoveDuplicate:
rrfound=RemDuplicate(d=rrfound)
for rr in rrfound:
rrqof = rrqof + json.dumps(rr) + "\n"
return rrqof
class InfoHandler(tornado.web.RequestHandler):
def get(self):
response = { 'version': 'git',
'software': 'analyzer-d4-passivedns' }
self.write(response)
class QueryHandler(tornado.web.RequestHandler):
def get(self, q):
2019-02-02 20:13:01 +01:00
print ("query: {}".format(q))
if iptools.ipv4.validate_ip(q) or iptools.ipv6.validate_ip(q):
for x in getAssociatedRecords(q):
self.write(JsonQOF(getRecord(x)))
else:
self.write(JsonQOF(getRecord(t = q.strip())))
class FullQueryHandler(tornado.web.RequestHandler):
def get(self, q):
print ("fquery: "+q)
if iptools.ipv4.validate_ip(q) or iptools.ipv6.validate_ip(q):
for x in getAssociatedRecords(q):
self.write(JsonQOF(getRecord(x)))
else:
for x in getAssociatedRecords(q):
self.write(JsonQOF(getRecord(t = x.strip())))
application = tornado.web.Application([
(r"/query/(.*)",QueryHandler),
(r"/fquery/(.*)",FullQueryHandler),
(r"/info", InfoHandler)
])
if __name__ == "test":
qq = ["foo.be", "8.8.8.8"]
for q in qq:
if iptools.ipv4.validate_ip(q) or iptools.ipv6.validate_ip(q):
for x in getAssociatedRecords(q):
print (JsonQOF(getRecord(x)))
else:
print (JsonQOF(getRecord(t = q)))
else:
application.listen(8400)
tornado.ioloop.IOLoop.instance().start()