add: [pdns-import]# pdns-import is a simple import from Passive DNS cof format (in an array) and import these back into a Passive DNS back-end

pull/3/head
Alexandre Dulaunoy 2019-04-05 09:29:34 +02:00
parent 853e1144b9
commit 389b034893
No known key found for this signature in database
GPG Key ID: 09E2CD4944E6CBCD
1 changed files with 121 additions and 0 deletions

121
bin/pdns-import.py Normal file
View File

@ -0,0 +1,121 @@
#!/usr/bin/env python3
#
# pdns-import is a simple import from Passive DNS cof format (in an array)
# and import these back into a Passive DNS backend
#
# This software is part of the D4 project.
#
# The software is released under the GNU Affero General Public version 3.
#
# Copyright (c) 2019 Alexandre Dulaunoy - a@foo.be
# Copyright (c) Computer Incident Response Center Luxembourg (CIRCL)
import re
import redis
import fileinput
import json
import configparser
import time
import logging
import sys
import argparse
parser = argparse.ArgumentParser(description='Import array of standard Passive DNS cof format into your Passive DNS server')
parser.add_argument('--file', dest='filetoimport', help='JSON file to import')
args = parser.parse_args()
config = configparser.RawConfigParser()
config.read('../etc/analyzer.conf')
expirations = config.items('expiration')
excludesubstrings = config.get('exclude', 'substring').split(',')
myuuid = config.get('global', 'my-uuid')
myqueue = "analyzer:8:{}".format(myuuid)
mylogginglevel = config.get('global', 'logging-level')
logger = logging.getLogger('pdns ingestor')
ch = logging.StreamHandler()
if mylogginglevel == 'DEBUG':
logger.setLevel(logging.DEBUG)
ch.setLevel(logging.DEBUG)
elif mylogginglevel == 'INFO':
logger.setLevel(logging.INFO)
ch.setLevel(logging.INFO)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
logger.addHandler(ch)
logger.info("Starting and using FIFO {} from D4 server".format(myqueue))
d4_server = config.get('global', 'd4-server')
r = redis.Redis(host="127.0.0.1",port=6400)
r_d4 = redis.Redis(host=d4_server.split(':')[0], port=d4_server.split(':')[1], db=2)
with open('../etc/records-type.json') as rtypefile:
rtype = json.load(rtypefile)
dnstype = {}
stats = True
for v in rtype:
dnstype[(v['type'])] = v['value']
while (True):
expiration = None
if not (args.filetoimport):
parser.print_help()
sys.exit(0)
with open(args.filetoimport) as dnsimport:
records = json.load(dnsimport)
print (records)
if records is False:
logger.debug('Parsing of passive DNS line failed: {}'.format(l.strip()))
continue
for rdns in records:
logger.debug("parsed record: {}".format(r))
if 'rrname' not in rdns:
logger.debug('Parsing of passive DNS line is incomplete: {}'.format(l.strip()))
continue
if rdns['rrname'] and rdns['rrtype']:
rdns['type'] = dnstype[rdns['rrtype']]
rdns['v'] = rdns['rdata']
excludeflag = False
for exclude in excludesubstrings:
if exclude in rdns['rrname']:
excludeflag = True
if excludeflag:
logger.debug('Excluded {}'.format(rdns['rrname']))
continue
if rdns['type'] == '16':
rdns['v'] = rdns['v'].replace("\"", "", 1)
query = "r:{}:{}".format(rdns['rrname'],rdns['type'])
logger.debug('redis sadd: {} -> {}'.format(query,rdns['v']))
r.sadd(query, rdns['v'])
res = "v:{}:{}".format(rdns['v'], rdns['type'])
logger.debug('redis sadd: {} -> {}'.format(res,rdns['rrname']))
r.sadd(res, rdns['q'])
firstseen = "s:{}:{}:{}".format(rdns['rrname'], rdns['v'], rdns['type'])
if not r.exists(firstseen):
r.set(firstseen, rdns['time_first'])
logger.debug('redis set: {} -> {}'.format(firstseen, rdns['time_first']))
lastseen = "l:{}:{}:{}".format(rdns['rrname'], rdns['v'], rdns['type'])
last = r.get(lastseen)
if last is None or int(last) < int(rdns['timestamp']):
r.set(lastseen, rdns['time_last'])
logger.debug('redis set: {} -> {}'.format(lastseen, rdns['time_last']))
occ = "o:{}:{}:{}".format(rdns['rrname'], rdns['v'], rdns['type'])
r.set(occ, rdns['count'])
if stats:
r.incrby('stats:processed', amount=1)
if not r:
logger.info('empty passive dns record')
continue