D4
/
analyzer-d4-passivedns
mirror of https://github.com/D4-project/analyzer-d4-passivedns
2
0
Fork 0
Code Issues Releases Wiki Activity

chg: [pdns-import] clean-up

pull/3/head v0.1
Alexandre Dulaunoy 2019-04-05 09:35:20 +02:00
parent 389b034893
commit 771718b7f7
No known key found for this signature in database
GPG Key ID: 09E2CD4944E6CBCD
1 changed files with 44 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

92
bin/pdns-import.py
View File

@ -62,60 +62,56 @@ stats = True
for v in rtype: for v in rtype:
dnstype[(v['type'])] = v['value'] dnstype[(v['type'])] = v['value']
while (True): expiration = None
expiration = None if not (args.filetoimport):
if not (args.filetoimport): parser.print_help()
parser.print_help() sys.exit(0)
sys.exit(0) with open(args.filetoimport) as dnsimport:
with open(args.filetoimport) as dnsimport: records = json.load(dnsimport)
records = json.load(dnsimport)
print (records) print (records)
if records is False: for rdns in records:
logger.debug('Parsing of passive DNS line failed: {}'.format(l.strip())) logger.debug("parsed record: {}".format(r))
if 'rrname' not in rdns:
logger.debug('Parsing of passive DNS line is incomplete: {}'.format(l.strip()))
continue continue
for rdns in records: if rdns['rrname'] and rdns['rrtype']:
logger.debug("parsed record: {}".format(r)) rdns['type'] = dnstype[rdns['rrtype']]
if 'rrname' not in rdns: rdns['v'] = rdns['rdata']
logger.debug('Parsing of passive DNS line is incomplete: {}'.format(l.strip())) excludeflag = False
for exclude in excludesubstrings:
if exclude in rdns['rrname']:
excludeflag = True
if excludeflag:
logger.debug('Excluded {}'.format(rdns['rrname']))
continue continue
if rdns['rrname'] and rdns['rrtype']: if rdns['type'] == '16':
rdns['type'] = dnstype[rdns['rrtype']] rdns['v'] = rdns['v'].replace("\"", "", 1)
rdns['v'] = rdns['rdata'] query = "r:{}:{}".format(rdns['rrname'],rdns['type'])
excludeflag = False logger.debug('redis sadd: {} -> {}'.format(query,rdns['v']))
for exclude in excludesubstrings: r.sadd(query, rdns['v'])
if exclude in rdns['rrname']: res = "v:{}:{}".format(rdns['v'], rdns['type'])
excludeflag = True logger.debug('redis sadd: {} -> {}'.format(res,rdns['rrname']))
if excludeflag: r.sadd(res, rdns['rrname'])
logger.debug('Excluded {}'.format(rdns['rrname']))
continue
if rdns['type'] == '16':
rdns['v'] = rdns['v'].replace("\"", "", 1)
query = "r:{}:{}".format(rdns['rrname'],rdns['type'])
logger.debug('redis sadd: {} -> {}'.format(query,rdns['v']))
r.sadd(query, rdns['v'])
res = "v:{}:{}".format(rdns['v'], rdns['type'])
logger.debug('redis sadd: {} -> {}'.format(res,rdns['rrname']))
r.sadd(res, rdns['q'])
firstseen = "s:{}:{}:{}".format(rdns['rrname'], rdns['v'], rdns['type']) firstseen = "s:{}:{}:{}".format(rdns['rrname'], rdns['v'], rdns['type'])
if not r.exists(firstseen): if not r.exists(firstseen):
r.set(firstseen, rdns['time_first']) r.set(firstseen, rdns['time_first'])
logger.debug('redis set: {} -> {}'.format(firstseen, rdns['time_first'])) logger.debug('redis set: {} -> {}'.format(firstseen, rdns['time_first']))
lastseen = "l:{}:{}:{}".format(rdns['rrname'], rdns['v'], rdns['type']) lastseen = "l:{}:{}:{}".format(rdns['rrname'], rdns['v'], rdns['type'])
last = r.get(lastseen) last = r.get(lastseen)
if last is None or int(last) < int(rdns['timestamp']): if last is None or int(last) < int(rdns['time_last']):
r.set(lastseen, rdns['time_last']) r.set(lastseen, rdns['time_last'])
logger.debug('redis set: {} -> {}'.format(lastseen, rdns['time_last'])) logger.debug('redis set: {} -> {}'.format(lastseen, rdns['time_last']))
occ = "o:{}:{}:{}".format(rdns['rrname'], rdns['v'], rdns['type']) occ = "o:{}:{}:{}".format(rdns['rrname'], rdns['v'], rdns['type'])
r.set(occ, rdns['count']) r.set(occ, rdns['count'])
if stats: if stats:
r.incrby('stats:processed', amount=1) r.incrby('stats:processed', amount=1)
if not r: if not r:
logger.info('empty passive dns record') logger.info('empty passive dns record')
continue continue