From e7389306bb2281393c701f5dc700d7b566755b15 Mon Sep 17 00:00:00 2001 From: Hannah Ward Date: Thu, 21 Feb 2019 13:50:06 +0000 Subject: [PATCH] new: Add resolution for multiple POLL uris --- README.md | 27 ++++++++ scripts/run-taxii-poll.py | 129 +++++++++++++++++++++++--------------- 2 files changed, 106 insertions(+), 50 deletions(-) diff --git a/README.md b/README.md index 8f4d5b3..d2f5172 100644 --- a/README.md +++ b/README.md @@ -198,3 +198,30 @@ MariaDB [taxiiauth]> select * from accounts; | 1 | ltaxii | pbkdf2:sha256:50000$99999999$1111111111111111111111111111111111111111111111111111111111111111 | +----+----------+-----------------------------------------------------------------------------------------------+ ``` + +### Ambigious Polling Service + +In the case that the server you want to poll has multiple `POLL` services, +run + +```bash +taxii-discovery \ + --host + --port + --discovery +``` + +It'll show you the services available on the server. You'll *probably* +see two POLL services, for different version of TAXII (message binding) + +Find the one relevent to you, copy its `Service Address`, +and modify `~/.misptaxii/remote-servers.yml` to resemble + +```yaml +- name: "my server" + taxii_version: "1.1" + ... + uri: +``` + +now try polling again diff --git a/scripts/run-taxii-poll.py b/scripts/run-taxii-poll.py index aa9099e..2b7b3cc 100644 --- a/scripts/run-taxii-poll.py +++ b/scripts/run-taxii-poll.py @@ -12,60 +12,75 @@ from datetime import datetime # Create an argument parser for our program # Will just take in a config file and logging options parser = argparse.ArgumentParser(description='Run MISP taxii pull.') -parser.add_argument('-c', "--configdir", default="~/.misptaxii", help='Config directory') -parser.add_argument("-v", "--verbose", action="store_true", help="More verbose logging") -parser.add_argument("-s", "--stdout", action="store_true", help="Log to STDOUT") -parser.add_argument("--start", help="Date to poll from (YYYY-MM-DDTHH:MM:SS), Exclusive") -parser.add_argument("--end", help="Date to poll to (YYYY-MM-DDTHH:MM:SS), Inclusive") -parser.add_argument("--subscription_id", help="The ID of the subscription", default=None) -parser.add_argument("--tz", help="Your timezone, e.g Europe/London. Default utc", +parser.add_argument('-c', "--configdir", default="~/.misptaxii", + help='Config directory') +parser.add_argument("-v", "--verbose", action="store_true", + help="More verbose logging") +parser.add_argument("-s", "--stdout", action="store_true", + help="Log to STDOUT") +parser.add_argument("--start", + help="Date to poll from (YYYY-MM-DDTHH:MM:SS), Exclusive") +parser.add_argument("--end", + help="Date to poll to (YYYY-MM-DDTHH:MM:SS), Inclusive") +parser.add_argument("--subscription_id", help="The ID of the subscription", + default=None) +parser.add_argument("--tz", + help="Your timezone, e.g Europe/London. Default utc", default="utc") - + args = parser.parse_args() # Set up a logger for logging's sake log = logging.getLogger(__name__) -logging.basicConfig(filename="poll.log", format="%(asctime)s - %(name)s - %(levelname)s - %(message)s") +logging.basicConfig( + filename="poll.log", + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s") log.setLevel(logging.DEBUG if args.verbose else logging.INFO) # If we want, print the output to stdout if args.stdout: - formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") - ch = logging.StreamHandler(sys.stdout) + formatter = logging.Formatter( + "%(asctime)s - %(name)s - %(levelname)s - %(message)s") + ch = logging.StreamHandler(sys.stdout) ch.setFormatter(formatter) log.addHandler(ch) # Read in the remote server configurations -configFile = "{}/remote-servers.yml".format(os.path.expanduser(args.configdir)) -log.debug("Opening config file %s", configFile) -with open(configFile, "r") as f: +config_file = "{}/remote-servers.yml".format( + os.path.expanduser(args.configdir)) + +log.debug("Opening config file %s", config_file) +with open(config_file, "r") as f: config = yaml.load(f.read()) log.debug("Config read %s", config) # Read in the local server configuration -localConfig = "{}/local-server.yml".format(os.path.expanduser(args.configdir)) +local_config = "{}/local-server.yml".format(os.path.expanduser(args.configdir)) log.debug("Reading local server config") -with open(localConfig, "r") as f: - localConfig = yaml.load(f.read()) +with open(local_config, "r") as f: + local_config = yaml.load(f.read()) # Attempt to make contact with the local server log.info("Connecting to local server...") -localClient = create_client(host = localConfig["host"], - port = localConfig["port"], - discovery_path = localConfig["discovery_path"], - use_https = localConfig["use_https"], - version = localConfig["taxii_version"], - headers = localConfig["headers"]) -localClient.username = localConfig["auth"]["username"] -localClient.password = localConfig["auth"]["password"] +local_client = create_client(host=local_config["host"], + port=local_config["port"], + discovery_path=local_config["discovery_path"], + use_https=local_config["use_https"], + version=local_config["taxii_version"], + headers=local_config["headers"]) -localInbox = "{}://{}:{}{}".format("https" if localConfig["use_https"] else "http", - localConfig["host"], localConfig["port"], - localConfig["inbox_path"]) +local_client.username = local_config["auth"]["username"] +local_client.password = local_config["auth"]["password"] + + +local_inbox = "{}://{}:{}{}".format( + "https" if local_config["use_https"] else "http", + local_config["host"], local_config["port"], + local_config["inbox_path"]) # Check that we're all good and authenticated try: - list(localClient.discover_services()) + list(local_client.discover_services()) except Exception as ex: log.fatal("Could not connect to local server") log.fatal(ex) @@ -102,26 +117,31 @@ for server in config: log.debug("Creating client") log.debug("HOST:PORT : %s:%s", server["host"], server["port"]) log.debug("DISCPATH: %s", server["discovery_path"]) - cli = create_client(host = server["host"], - port = server["port"], - discovery_path = server["discovery_path"], - use_https = server["use_https"], - version = server["taxii_version"], - headers = server["headers"]) + + # Standard autodiscovery + client_args = { + "host": server["host"], + "port": server["port"], + "discovery_path": server["discovery_path"], + "use_https": server["use_https"], + "version": server["taxii_version"], + "headers": server["headers"] + } + + cli = create_client(**client_args) log.debug("Setting client log level") cli.log.setLevel(logging.DEBUG if args.verbose else logging.INFO) - log.debug("Setting authentication...") - cli.set_auth(username = server["auth"]["username"], - password = server["auth"]["password"], - ca_cert = server["auth"].get("ca_cert"), - cert_file= server["auth"].get("cert_file"), - key_file = server["auth"].get("key_file"), - key_password = server["auth"].get("key_password"), - jwt_auth_url = server["auth"].get("jwt_auth_url"), - verify_ssl = server["auth"].get("verify_ssl")) + cli.set_auth(username=server["auth"]["username"], + password=server["auth"]["password"], + ca_cert=server["auth"].get("ca_cert"), + cert_file=server["auth"].get("cert_file"), + key_file=server["auth"].get("key_file"), + key_password=server["auth"].get("key_password"), + jwt_auth_url=server["auth"].get("jwt_auth_url"), + verify_ssl=server["auth"].get("verify_ssl")) log.debug("Discovering services...") services = cli.discover_services() @@ -130,18 +150,27 @@ for server in config: log.debug("Auth set.") for collection in server["collections"]: log.debug("Polling %s", collection) - log.debug("Within date range %s - %s", poll_from or "Beginning of time", poll_to) + server_uri_override = server.get("uri", None) + if not server_uri_override.startswith("http"): + server_uri_override = None + if server_uri_override: + log.debug("Poll URL override set to %s", server_uri_override) + + log.debug("Within date range %s - %s", + poll_from or "Beginning of time", poll_to) try: for content_block in cli.poll(collection_name=collection, subscription_id=subscription_id, begin_date=poll_from, - end_date=poll_to): + end_date=poll_to, + uri=server.get("uri", None)): try: log.debug("Pushing block %s", content_block) - localClient.push(content_block.content.decode("utf-8"), - collection_names=localConfig["collections"], - content_binding=content_block.binding, - uri=localInbox) + local_client.push( + content_block.content.decode("utf-8"), + collection_names=local_config["collections"], + content_binding=content_block.binding, + uri=local_inbox) except Exception as ex: log.error("FAILED TO PUSH BLOCK!") log.error("%s", content_block)