From c852241b52539aeeab478bc4dd01fe50a0c4a387 Mon Sep 17 00:00:00 2001 From: ljaqueme Date: Tue, 11 Jun 2019 14:41:32 -0600 Subject: [PATCH 1/4] support env for docker --- bin/pdns-cof-server.py | 6 +++++- bin/pdns-import.py | 6 +++++- bin/pdns-ingestion.py | 6 +++++- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/bin/pdns-cof-server.py b/bin/pdns-cof-server.py index ea4f31a..a0711d2 100644 --- a/bin/pdns-cof-server.py +++ b/bin/pdns-cof-server.py @@ -24,10 +24,14 @@ import tornado.web import iptools import redis import json +import os rrset = [{"Reference": "[RFC1035]", "Type": "A", "Value": "1", "Meaning": "a host address", "Template": "", "Registration Date": ""}, {"Reference": "[RFC1035]", "Type": "NS", "Value": "2", "Meaning": "an authoritative name server", "Template": "", "Registration Date": ""}, {"Reference": "[RFC1035]", "Type": "MD", "Value": "3", "Meaning": "a mail destination (OBSOLETE - use MX)", "Template": "", "Registration Date": ""}, {"Reference": "[RFC1035]", "Type": "MF", "Value": "4", "Meaning": "a mail forwarder (OBSOLETE - use MX)", "Template": "", "Registration Date": ""}, {"Reference": "[RFC1035]", "Type": "CNAME", "Value": "5", "Meaning": "the canonical name for an alias", "Template": "", "Registration Date": ""}, {"Reference": "[RFC1035]", "Type": "SOA", "Value": "6", "Meaning": "marks the start of a zone of authority", "Template": "", "Registration Date": ""}, {"Reference": "[RFC1035]", "Type": "MB", "Value": "7", "Meaning": "a mailbox domain name (EXPERIMENTAL)", "Template": "", "Registration Date": ""}, {"Reference": "[RFC1035]", "Type": "MG", "Value": "8", "Meaning": "a mail group member (EXPERIMENTAL)", "Template": "", "Registration Date": ""}, {"Reference": "[RFC1035]", "Type": "MR", "Value": "9", "Meaning": "a mail rename domain name (EXPERIMENTAL)", "Template": "", "Registration Date": ""}, {"Reference": "[RFC1035]", "Type": "NULL", "Value": "10", "Meaning": "a null RR (EXPERIMENTAL)", "Template": "", "Registration Date": ""}, {"Reference": "[RFC1035]", "Type": "WKS", "Value": "11", "Meaning": "a well known service description", "Template": "", "Registration Date": ""}, {"Reference": "[RFC1035]", "Type": "PTR", "Value": "12", "Meaning": "a domain name pointer", "Template": "", "Registration Date": ""}, {"Reference": "[RFC1035]", "Type": "HINFO", "Value": "13", "Meaning": "host information", "Template": "", "Registration Date": ""}, {"Reference": "[RFC1035]", "Type": "MINFO", "Value": "14", "Meaning": "mailbox or mail list information", "Template": "", "Registration Date": ""}, {"Reference": "[RFC1035]", "Type": "MX", "Value": "15", "Meaning": "mail exchange", "Template": "", "Registration Date": ""}, {"Reference": "[RFC1035]", "Type": "TXT", "Value": "16", "Meaning": "text strings", "Template": "", "Registration Date": ""}, {"Reference": "[RFC1183]", "Type": "RP", "Value": "17", "Meaning": "for Responsible Person", "Template": "", "Registration Date": ""}, {"Reference": "[RFC1183][RFC5864]", "Type": "AFSDB", "Value": "18", "Meaning": "for AFS Data Base location", "Template": "", "Registration Date": ""}, {"Reference": "[RFC1183]", "Type": "X25", "Value": "19", "Meaning": "for X.25 PSDN address", "Template": "", "Registration Date": ""}, {"Reference": "[RFC1183]", "Type": "ISDN", "Value": "20", "Meaning": "for ISDN address", "Template": "", "Registration Date": ""}, {"Reference": "[RFC1183]", "Type": "RT", "Value": "21", "Meaning": "for Route Through", "Template": "", "Registration Date": ""}, {"Reference": "[RFC1706]", "Type": "NSAP", "Value": "22", "Meaning": "for NSAP address, NSAP style A record", "Template": "", "Registration Date": ""}, {"Reference": "[RFC1348][RFC1637][RFC1706]", "Type": "NSAP-PTR", "Value": "23", "Meaning": "for domain name pointer, NSAP style", "Template": "", "Registration Date": ""}, {"Reference": "[RFC4034][RFC3755][RFC2535][RFC2536][RFC2537][RFC2931][RFC3110][RFC3008]", "Type": "SIG", "Value": "24", "Meaning": "for security signature", "Template": "", "Registration Date": ""}, {"Reference": "[RFC4034][RFC3755][RFC2535][RFC2536][RFC2537][RFC2539][RFC3008][RFC3110]", "Type": "KEY", "Value": "25", "Meaning": "for security key", "Template": "", "Registration Date": ""}, {"Reference": "[RFC2163]", "Type": "PX", "Value": "26", "Meaning": "X.400 mail mapping information", "Template": "", "Registration Date": ""}, {"Reference": "[RFC1712]", "Type": "GPOS", "Value": "27", "Meaning": "Geographical Position", "Template": "", "Registration Date": ""}, {"Reference": "[RFC3596]", "Type": "AAAA", "Value": "28", "Meaning": "IP6 Address", "Template": "", "Registration Date": ""}, {"Reference": "[RFC1876]", "Type": "LOC", "Value": "29", "Meaning": "Location Information", "Template": "", "Registration Date": ""}, {"Reference": "[RFC3755][RFC2535]", "Type": "NXT", "Value": "30", "Meaning": "Next Domain (OBSOLETE)", "Template": "", "Registration Date": ""}, {"Reference": "[Michael_Patton][http://ana-3.lcs.mit.edu/~jnc/nimrod/dns.txt]", "Type": "EID", "Value": "31", "Meaning": "Endpoint Identifier", "Template": "", "Registration Date": "1995-06"}, {"Reference": "[1][Michael_Patton][http://ana-3.lcs.mit.edu/~jnc/nimrod/dns.txt]", "Type": "NIMLOC", "Value": "32", "Meaning": "Nimrod Locator", "Template": "", "Registration Date": "1995-06"}, {"Reference": "[1][RFC2782]", "Type": "SRV", "Value": "33", "Meaning": "Server Selection", "Template": "", "Registration Date": ""}, {"Reference": "[\n ATM Forum Technical Committee, \"ATM Name System, V2.0\", Doc ID: AF-DANS-0152.000, July 2000. Available from and held in escrow by IANA.]", "Type": "ATMA", "Value": "34", "Meaning": "ATM Address", "Template": "", "Registration Date": ""}, {"Reference": "[RFC2915][RFC2168][RFC3403]", "Type": "NAPTR", "Value": "35", "Meaning": "Naming Authority Pointer", "Template": "", "Registration Date": ""}, {"Reference": "[RFC2230]", "Type": "KX", "Value": "36", "Meaning": "Key Exchanger", "Template": "", "Registration Date": ""}, {"Reference": "[RFC4398]", "Type": "CERT", "Value": "37", "Meaning": "CERT", "Template": "", "Registration Date": ""}, {"Reference": "[RFC3226][RFC2874][RFC6563]", "Type": "A6", "Value": "38", "Meaning": "A6 (OBSOLETE - use AAAA)", "Template": "", "Registration Date": ""}, {"Reference": "[RFC6672]", "Type": "DNAME", "Value": "39", "Meaning": "DNAME", "Template": "", "Registration Date": ""}, {"Reference": "[Donald_E_Eastlake][http://tools.ietf.org/html/draft-eastlake-kitchen-sink]", "Type": "SINK", "Value": "40", "Meaning": "SINK", "Template": "", "Registration Date": "1997-11"}, {"Reference": "[RFC6891][RFC3225]", "Type": "OPT", "Value": "41", "Meaning": "OPT", "Template": "", "Registration Date": ""}, {"Reference": "[RFC3123]", "Type": "APL", "Value": "42", "Meaning": "APL", "Template": "", "Registration Date": ""}, {"Reference": "[RFC4034][RFC3658]", "Type": "DS", "Value": "43", "Meaning": "Delegation Signer", "Template": "", "Registration Date": ""}, {"Reference": "[RFC4255]", "Type": "SSHFP", "Value": "44", "Meaning": "SSH Key Fingerprint", "Template": "", "Registration Date": ""}, {"Reference": "[RFC4025]", "Type": "IPSECKEY", "Value": "45", "Meaning": "IPSECKEY", "Template": "", "Registration Date": ""}, {"Reference": "[RFC4034][RFC3755]", "Type": "RRSIG", "Value": "46", "Meaning": "RRSIG", "Template": "", "Registration Date": ""}, {"Reference": "[RFC4034][RFC3755]", "Type": "NSEC", "Value": "47", "Meaning": "NSEC", "Template": "", "Registration Date": ""}, {"Reference": "[RFC4034][RFC3755]", "Type": "DNSKEY", "Value": "48", "Meaning": "DNSKEY", "Template": "", "Registration Date": ""}, {"Reference": "[RFC4701]", "Type": "DHCID", "Value": "49", "Meaning": "DHCID", "Template": "", "Registration Date": ""}, {"Reference": "[RFC5155]", "Type": "NSEC3", "Value": "50", "Meaning": "NSEC3", "Template": "", "Registration Date": ""}, {"Reference": "[RFC5155]", "Type": "NSEC3PARAM", "Value": "51", "Meaning": "NSEC3PARAM", "Template": "", "Registration Date": ""}, {"Reference": "[RFC6698]", "Type": "TLSA", "Value": "52", "Meaning": "TLSA", "Template": "", "Registration Date": ""}, {"Reference": "[RFC5205]", "Type": "HIP", "Value": "55", "Meaning": "Host Identity Protocol", "Template": "", "Registration Date": ""}, {"Reference": "[Jim_Reid]", "Type": "NINFO", "Value": "56", "Meaning": "NINFO", "Template": "NINFO/ninfo-completed-template", "Registration Date": "2008-01-21"}, {"Reference": "[Jim_Reid]", "Type": "RKEY", "Value": "57", "Meaning": "RKEY", "Template": "RKEY/rkey-completed-template", "Registration Date": "2008-01-21"}, {"Reference": "[Wouter_Wijngaards]", "Type": "TALINK", "Value": "58", "Meaning": "Trust Anchor LINK", "Template": "TALINK/talink-completed-template", "Registration Date": "2010-02-17"}, {"Reference": "[George_Barwood]", "Type": "CDS", "Value": "59", "Meaning": "Child DS", "Template": "CDS/cds-completed-template", "Registration Date": "2011-06-06"}, {"Reference": "[RFC4408]", "Type": "SPF", "Value": "99", "Meaning": "", "Template": "", "Registration Date": ""}, {"Reference": "[IANA-Reserved]", "Type": "UINFO", "Value": "100", "Meaning": "", "Template": "", "Registration Date": ""}, {"Reference": "[IANA-Reserved]", "Type": "UID", "Value": "101", "Meaning": "", "Template": "", "Registration Date": ""}, {"Reference": "[IANA-Reserved]", "Type": "GID", "Value": "102", "Meaning": "", "Template": "", "Registration Date": ""}, {"Reference": "[IANA-Reserved]", "Type": "UNSPEC", "Value": "103", "Meaning": "", "Template": "", "Registration Date": ""}, {"Reference": "[RFC6742]", "Type": "NID", "Value": "104", "Meaning": "", "Template": "ILNP/nid-completed-template", "Registration Date": ""}, {"Reference": "[RFC6742]", "Type": "L32", "Value": "105", "Meaning": "", "Template": "ILNP/l32-completed-template", "Registration Date": ""}, {"Reference": "[RFC6742]", "Type": "L64", "Value": "106", "Meaning": "", "Template": "ILNP/l64-completed-template", "Registration Date": ""}, {"Reference": "[RFC6742]", "Type": "LP", "Value": "107", "Meaning": "", "Template": "ILNP/lp-completed-template", "Registration Date": ""}, {"Reference": "[RFC7043]", "Type": "EUI48", "Value": "108", "Meaning": "an EUI-48 address", "Template": "EUI48/eui48-completed-template", "Registration Date": "2013-03-27"}, {"Reference": "[RFC7043]", "Type": "EUI64", "Value": "109", "Meaning": "an EUI-64 address", "Template": "EUI64/eui64-completed-template", "Registration Date": "2013-03-27"}, {"Reference": "[RFC2930]", "Type": "TKEY", "Value": "249", "Meaning": "Transaction Key", "Template": "", "Registration Date": ""}, {"Reference": "[RFC2845]", "Type": "TSIG", "Value": "250", "Meaning": "Transaction Signature", "Template": "", "Registration Date": ""}, {"Reference": "[RFC1995]", "Type": "IXFR", "Value": "251", "Meaning": "incremental transfer", "Template": "", "Registration Date": ""}, {"Reference": "[RFC1035][RFC5936]", "Type": "AXFR", "Value": "252", "Meaning": "transfer of an entire zone", "Template": "", "Registration Date": ""}, {"Reference": "[RFC1035]", "Type": "MAILB", "Value": "253", "Meaning": "mailbox-related RRs (MB, MG or MR)", "Template": "", "Registration Date": ""}, {"Reference": "[RFC1035]", "Type": "MAILA", "Value": "254", "Meaning": "mail agent RRs (OBSOLETE - see MX)", "Template": "", "Registration Date": ""}, {"Reference": "[RFC1035][RFC6895]", "Type": "*", "Value": "255", "Meaning": "A request for all records the server/cache has available", "Template": "", "Registration Date": ""}, {"Reference": "[Patrik_Faltstrom]", "Type": "URI", "Value": "256", "Meaning": "URI", "Template": "URI/uri-completed-template", "Registration Date": "2011-02-22"}, {"Reference": "[RFC6844]", "Type": "CAA", "Value": "257", "Meaning": "Certification Authority Restriction", "Template": "CAA/caa-completed-template", "Registration Date": "2011-04-07"}, {"Reference": "[Sam_Weiler][http://cameo.library.cmu.edu/][\n Deploying DNSSEC Without a Signed Root. Technical Report 1999-19,\nInformation Networking Institute, Carnegie Mellon University, April 2004.]", "Type": "TA", "Value": "32768", "Meaning": "DNSSEC Trust Authorities", "Template": "", "Registration Date": "2005-12-13"}, {"Reference": "[RFC4431]", "Type": "DLV", "Value": "32769", "Meaning": "DNSSEC Lookaside Validation", "Template": "", "Registration Date": ""}, {"Reference": "", "Type": "Reserved", "Value": "65535", "Meaning": "", "Template": "", "Registration Date": ""}] -r=redis.StrictRedis(host='localhost', port=6400, db=0) +analyzer_redis_host = os.getenv('D4_ANALYZER_REDIS_HOST', '127.0.0.1') +analyzer_redis_port = int(os.getenv('D4_ANALYZER_REDIS_PORT', 6400)) + +r = redis.StrictRedis(host=analyzer_redis_host, port=analyzer_redis_port, db=0) rrset_supported = ['1','2','5','15','16','28','33','46'] expiring_type = ['16'] diff --git a/bin/pdns-import.py b/bin/pdns-import.py index eb40216..4f89f33 100644 --- a/bin/pdns-import.py +++ b/bin/pdns-import.py @@ -20,6 +20,7 @@ import time import logging import sys import argparse +import os parser = argparse.ArgumentParser(description='Import array of standard Passive DNS cof format into your Passive DNS server') parser.add_argument('--file', dest='filetoimport', help='JSON file to import') @@ -47,8 +48,11 @@ logger.addHandler(ch) logger.info("Starting and using FIFO {} from D4 server".format(myqueue)) +analyzer_redis_host = os.getenv('D4_ANALYZER_REDIS_HOST', '127.0.0.1') +analyzer_redis_port = int(os.getenv('D4_ANALYZER_REDIS_PORT', 6400)) + d4_server = config.get('global', 'd4-server') -r = redis.Redis(host="127.0.0.1",port=6400) +r = redis.Redis(host=analyzer_redis_host, port=analyzer_redis_port) r_d4 = redis.Redis(host=d4_server.split(':')[0], port=d4_server.split(':')[1], db=2) diff --git a/bin/pdns-ingestion.py b/bin/pdns-ingestion.py index 810f139..4b36a62 100644 --- a/bin/pdns-ingestion.py +++ b/bin/pdns-ingestion.py @@ -22,6 +22,7 @@ import configparser import time import logging import sys +import os config = configparser.RawConfigParser() config.read('../etc/analyzer.conf') @@ -45,8 +46,11 @@ logger.addHandler(ch) logger.info("Starting and using FIFO {} from D4 server".format(myqueue)) +analyzer_redis_host = os.getenv('D4_ANALYZER_REDIS_HOST', '127.0.0.1') +analyzer_redis_port = int(os.getenv('D4_ANALYZER_REDIS_PORT', 6400)) + d4_server = config.get('global', 'd4-server') -r = redis.Redis(host="127.0.0.1",port=6400) +r = redis.Redis(host=analyzer_redis_host, port=analyzer_redis_port) r_d4 = redis.Redis(host=d4_server.split(':')[0], port=d4_server.split(':')[1], db=2) From 54022f6a560ac99cb2540b36aa4a547158418ed6 Mon Sep 17 00:00:00 2001 From: ljaqueme Date: Tue, 11 Jun 2019 14:46:46 -0600 Subject: [PATCH 2/4] superseed config with ENV if available --- bin/pdns-import.py | 8 +++++--- bin/pdns-ingestion.py | 7 +++++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/bin/pdns-import.py b/bin/pdns-import.py index 4f89f33..33647c5 100644 --- a/bin/pdns-import.py +++ b/bin/pdns-import.py @@ -51,10 +51,12 @@ logger.info("Starting and using FIFO {} from D4 server".format(myqueue)) analyzer_redis_host = os.getenv('D4_ANALYZER_REDIS_HOST', '127.0.0.1') analyzer_redis_port = int(os.getenv('D4_ANALYZER_REDIS_PORT', 6400)) -d4_server = config.get('global', 'd4-server') -r = redis.Redis(host=analyzer_redis_host, port=analyzer_redis_port) -r_d4 = redis.Redis(host=d4_server.split(':')[0], port=d4_server.split(':')[1], db=2) +d4_server, d4_port = config.get('global', 'd4-server').split(':') +host_redis_metadata = os.getenv('D4_REDIS_METADATA_HOST', d4_server) +port_redis_metadata = int(os.getenv('D4_REDIS_METADATA_PORT', d4_port)) +r = redis.Redis(host=analyzer_redis_host, port=analyzer_redis_port) +r_d4 = redis.Redis(host=host_redis_metadata, port=port_redis_metadata, db=2) with open('../etc/records-type.json') as rtypefile: rtype = json.load(rtypefile) diff --git a/bin/pdns-ingestion.py b/bin/pdns-ingestion.py index 4b36a62..aafb56a 100644 --- a/bin/pdns-ingestion.py +++ b/bin/pdns-ingestion.py @@ -49,9 +49,12 @@ logger.info("Starting and using FIFO {} from D4 server".format(myqueue)) analyzer_redis_host = os.getenv('D4_ANALYZER_REDIS_HOST', '127.0.0.1') analyzer_redis_port = int(os.getenv('D4_ANALYZER_REDIS_PORT', 6400)) -d4_server = config.get('global', 'd4-server') +d4_server, d4_port = config.get('global', 'd4-server').split(':') +host_redis_metadata = os.getenv('D4_REDIS_METADATA_HOST', d4_server) +port_redis_metadata = int(os.getenv('D4_REDIS_METADATA_PORT', d4_port)) + r = redis.Redis(host=analyzer_redis_host, port=analyzer_redis_port) -r_d4 = redis.Redis(host=d4_server.split(':')[0], port=d4_server.split(':')[1], db=2) +r_d4 = redis.Redis(host=host_redis_metadata, port=port_redis_metadata, db=2) with open('../etc/records-type.json') as rtypefile: From 505860831d2f3570b3211a46aca91d92688ad2da Mon Sep 17 00:00:00 2001 From: ljaqueme Date: Tue, 11 Jun 2019 16:11:47 -0600 Subject: [PATCH 3/4] let be simple --- etc/analyzer.conf.sample | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/analyzer.conf.sample b/etc/analyzer.conf.sample index 1b7c1bd..c82622f 100644 --- a/etc/analyzer.conf.sample +++ b/etc/analyzer.conf.sample @@ -2,7 +2,7 @@ my-uuid = 6a2461ce-c29d-44fc-b4fa-947d68826639 d4-server = 127.0.0.1:6380 # INFO|DEBUG -logging-level = INFO +logging-level = DEBUG [expiration] 16 = 24000 99 = 26000 From a3f1a89506c8bec7dd62734ce9a6adb07a8de5c0 Mon Sep 17 00:00:00 2001 From: ljaqueme Date: Wed, 12 Jun 2019 07:04:59 -0600 Subject: [PATCH 4/4] back to INFO --- etc/analyzer.conf.sample | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/analyzer.conf.sample b/etc/analyzer.conf.sample index c82622f..1b7c1bd 100644 --- a/etc/analyzer.conf.sample +++ b/etc/analyzer.conf.sample @@ -2,7 +2,7 @@ my-uuid = 6a2461ce-c29d-44fc-b4fa-947d68826639 d4-server = 127.0.0.1:6380 # INFO|DEBUG -logging-level = DEBUG +logging-level = INFO [expiration] 16 = 24000 99 = 26000