new: Add resolution for multiple POLL uris

README.md

@@ -198,3 +198,30 @@ MariaDB [taxiiauth]> select * from accounts;
 |  1 | ltaxii   | pbkdf2:sha256:50000$99999999$1111111111111111111111111111111111111111111111111111111111111111 |
 +----+----------+-----------------------------------------------------------------------------------------------+
 ```
+
+### Ambigious Polling Service
+
+In the case that the server you want to poll has multiple `POLL` services,
+run
+
+```bash
+taxii-discovery \
+  --host <HOST TO POLL>
+  --port <POLLING PORT>
+  --discovery <DISCOVERY PATH, sometimes /taxii-discovery-service, may vary>
+```
+
+It'll show you the services available on the server. You'll *probably*
+see two POLL services, for different version of TAXII (message binding)
+
+Find the one relevent to you, copy its `Service Address`,
+and modify `~/.misptaxii/remote-servers.yml` to resemble
+
+```yaml
+- name: "my server"
+  taxii_version: "1.1"
+  ...
+  uri: <SERVICE ADDRESS>
+```
+
+now try polling again

scripts/run-taxii-poll.py

@@ -12,60 +12,75 @@ from datetime import datetime
 # Create an argument parser for our program
 # Will just take in a config file and logging options
 parser = argparse.ArgumentParser(description='Run MISP taxii pull.')
-parser.add_argument('-c', "--configdir", default="~/.misptaxii", help='Config directory')
+parser.add_argument('-c', "--configdir", default="~/.misptaxii",
-parser.add_argument("-v", "--verbose", action="store_true", help="More verbose logging")
+                    help='Config directory')
-parser.add_argument("-s", "--stdout", action="store_true", help="Log to STDOUT")
+parser.add_argument("-v", "--verbose", action="store_true",
-parser.add_argument("--start", help="Date to poll from (YYYY-MM-DDTHH:MM:SS), Exclusive")
+                    help="More verbose logging")
-parser.add_argument("--end", help="Date to poll to (YYYY-MM-DDTHH:MM:SS), Inclusive")
+parser.add_argument("-s", "--stdout", action="store_true",
-parser.add_argument("--subscription_id", help="The ID of the subscription", default=None)
+                    help="Log to STDOUT")
-parser.add_argument("--tz", help="Your timezone, e.g Europe/London. Default utc", 
+parser.add_argument("--start",
+                    help="Date to poll from (YYYY-MM-DDTHH:MM:SS), Exclusive")
+parser.add_argument("--end",
+                    help="Date to poll to (YYYY-MM-DDTHH:MM:SS), Inclusive")
+parser.add_argument("--subscription_id", help="The ID of the subscription",
+                    default=None)
+parser.add_argument("--tz",
+                    help="Your timezone, e.g Europe/London. Default utc",
                     default="utc")
-  
+
 args = parser.parse_args()
 
 # Set up a logger for logging's sake
 log = logging.getLogger(__name__)
-logging.basicConfig(filename="poll.log", format="%(asctime)s - %(name)s - %(levelname)s - %(message)s")
+logging.basicConfig(
+    filename="poll.log",
+    format="%(asctime)s - %(name)s - %(levelname)s - %(message)s")
 log.setLevel(logging.DEBUG if args.verbose else logging.INFO)
 
 # If we want, print the output to stdout
 if args.stdout:
-    formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
+    formatter = logging.Formatter(
-    ch = logging.StreamHandler(sys.stdout)    
+        "%(asctime)s - %(name)s - %(levelname)s - %(message)s")
+    ch = logging.StreamHandler(sys.stdout)
     ch.setFormatter(formatter)
     log.addHandler(ch)
 
 # Read in the remote server configurations
-configFile = "{}/remote-servers.yml".format(os.path.expanduser(args.configdir))
+config_file = "{}/remote-servers.yml".format(
-log.debug("Opening config file %s", configFile)
+    os.path.expanduser(args.configdir))
-with open(configFile, "r") as f:
+
+log.debug("Opening config file %s", config_file)
+with open(config_file, "r") as f:
     config = yaml.load(f.read())
 log.debug("Config read %s", config)
 
 # Read in the local server configuration
-localConfig = "{}/local-server.yml".format(os.path.expanduser(args.configdir))
+local_config = "{}/local-server.yml".format(os.path.expanduser(args.configdir))
 log.debug("Reading local server config")
-with open(localConfig, "r") as f:
+with open(local_config, "r") as f:
-    localConfig = yaml.load(f.read())
+    local_config = yaml.load(f.read())
 
 # Attempt to make contact with the local server
 log.info("Connecting to local server...")
-localClient = create_client(host = localConfig["host"],
+local_client = create_client(host=local_config["host"],
-                            port = localConfig["port"],
+                             port=local_config["port"],
-                            discovery_path = localConfig["discovery_path"],
+                             discovery_path=local_config["discovery_path"],
-                            use_https = localConfig["use_https"],
+                             use_https=local_config["use_https"],
-                            version = localConfig["taxii_version"],
+                             version=local_config["taxii_version"],
-                            headers = localConfig["headers"])
+                             headers=local_config["headers"])
-localClient.username = localConfig["auth"]["username"]
-localClient.password = localConfig["auth"]["password"]
 
-localInbox = "{}://{}:{}{}".format("https" if localConfig["use_https"] else "http",
+local_client.username = local_config["auth"]["username"]
-                                   localConfig["host"], localConfig["port"],
+local_client.password = local_config["auth"]["password"]
-                                   localConfig["inbox_path"])
+
+
+local_inbox = "{}://{}:{}{}".format(
+    "https" if local_config["use_https"] else "http",
+    local_config["host"], local_config["port"],
+    local_config["inbox_path"])
 
 # Check that we're all good and authenticated
 try:
-    list(localClient.discover_services())
+    list(local_client.discover_services())
 except Exception as ex:
     log.fatal("Could not connect to local server")
     log.fatal(ex)
@@ -102,26 +117,31 @@ for server in config:
     log.debug("Creating client")
     log.debug("HOST:PORT : %s:%s", server["host"], server["port"])
     log.debug("DISCPATH: %s", server["discovery_path"])
-    cli = create_client(host = server["host"],
+
-                        port = server["port"],
+    # Standard autodiscovery
-                        discovery_path = server["discovery_path"],
+    client_args = {
-                        use_https = server["use_https"],
+        "host": server["host"],
-                        version = server["taxii_version"],
+        "port": server["port"],
-                        headers = server["headers"])
+        "discovery_path": server["discovery_path"],
+        "use_https": server["use_https"],
+        "version": server["taxii_version"],
+        "headers": server["headers"]
+    }
+
+    cli = create_client(**client_args)
 
     log.debug("Setting client log level")
     cli.log.setLevel(logging.DEBUG if args.verbose else logging.INFO)
-        
 
     log.debug("Setting authentication...")
-    cli.set_auth(username = server["auth"]["username"],
+    cli.set_auth(username=server["auth"]["username"],
-                 password = server["auth"]["password"],
+                 password=server["auth"]["password"],
-                 ca_cert  = server["auth"].get("ca_cert"),
+                 ca_cert=server["auth"].get("ca_cert"),
-                 cert_file= server["auth"].get("cert_file"),
+                 cert_file=server["auth"].get("cert_file"),
-                 key_file = server["auth"].get("key_file"),
+                 key_file=server["auth"].get("key_file"),
-                 key_password = server["auth"].get("key_password"),
+                 key_password=server["auth"].get("key_password"),
-                 jwt_auth_url = server["auth"].get("jwt_auth_url"),
+                 jwt_auth_url=server["auth"].get("jwt_auth_url"),
-                 verify_ssl = server["auth"].get("verify_ssl"))
+                 verify_ssl=server["auth"].get("verify_ssl"))
 
     log.debug("Discovering services...")
     services = cli.discover_services()
@@ -130,18 +150,27 @@ for server in config:
     log.debug("Auth set.")
     for collection in server["collections"]:
         log.debug("Polling %s", collection)
-        log.debug("Within date range %s - %s", poll_from or "Beginning of time", poll_to)
+        server_uri_override = server.get("uri", None)
+        if not server_uri_override.startswith("http"):
+            server_uri_override = None
+        if server_uri_override:
+            log.debug("Poll URL override set to %s", server_uri_override)
+
+        log.debug("Within date range %s - %s",
+                  poll_from or "Beginning of time", poll_to)
         try:
             for content_block in cli.poll(collection_name=collection,
                                           subscription_id=subscription_id,
                                           begin_date=poll_from,
-                                          end_date=poll_to):
+                                          end_date=poll_to,
+                                          uri=server.get("uri", None)):
                 try:
                     log.debug("Pushing block %s", content_block)
-                    localClient.push(content_block.content.decode("utf-8"), 
+                    local_client.push(
-                                     collection_names=localConfig["collections"],
+                        content_block.content.decode("utf-8"),
-                                     content_binding=content_block.binding,
+                        collection_names=local_config["collections"],
-                                     uri=localInbox)
+                        content_binding=content_block.binding,
+                        uri=local_inbox)
                 except Exception as ex:
                     log.error("FAILED TO PUSH BLOCK!")
                     log.error("%s", content_block)