new: Add resolution for multiple POLL uris
parent
2e2081ec48
commit
e7389306bb
27
README.md
27
README.md
|
@ -198,3 +198,30 @@ MariaDB [taxiiauth]> select * from accounts;
|
|||
| 1 | ltaxii | pbkdf2:sha256:50000$99999999$1111111111111111111111111111111111111111111111111111111111111111 |
|
||||
+----+----------+-----------------------------------------------------------------------------------------------+
|
||||
```
|
||||
|
||||
### Ambigious Polling Service
|
||||
|
||||
In the case that the server you want to poll has multiple `POLL` services,
|
||||
run
|
||||
|
||||
```bash
|
||||
taxii-discovery \
|
||||
--host <HOST TO POLL>
|
||||
--port <POLLING PORT>
|
||||
--discovery <DISCOVERY PATH, sometimes /taxii-discovery-service, may vary>
|
||||
```
|
||||
|
||||
It'll show you the services available on the server. You'll *probably*
|
||||
see two POLL services, for different version of TAXII (message binding)
|
||||
|
||||
Find the one relevent to you, copy its `Service Address`,
|
||||
and modify `~/.misptaxii/remote-servers.yml` to resemble
|
||||
|
||||
```yaml
|
||||
- name: "my server"
|
||||
taxii_version: "1.1"
|
||||
...
|
||||
uri: <SERVICE ADDRESS>
|
||||
```
|
||||
|
||||
now try polling again
|
||||
|
|
|
@ -12,60 +12,75 @@ from datetime import datetime
|
|||
# Create an argument parser for our program
|
||||
# Will just take in a config file and logging options
|
||||
parser = argparse.ArgumentParser(description='Run MISP taxii pull.')
|
||||
parser.add_argument('-c', "--configdir", default="~/.misptaxii", help='Config directory')
|
||||
parser.add_argument("-v", "--verbose", action="store_true", help="More verbose logging")
|
||||
parser.add_argument("-s", "--stdout", action="store_true", help="Log to STDOUT")
|
||||
parser.add_argument("--start", help="Date to poll from (YYYY-MM-DDTHH:MM:SS), Exclusive")
|
||||
parser.add_argument("--end", help="Date to poll to (YYYY-MM-DDTHH:MM:SS), Inclusive")
|
||||
parser.add_argument("--subscription_id", help="The ID of the subscription", default=None)
|
||||
parser.add_argument("--tz", help="Your timezone, e.g Europe/London. Default utc",
|
||||
parser.add_argument('-c', "--configdir", default="~/.misptaxii",
|
||||
help='Config directory')
|
||||
parser.add_argument("-v", "--verbose", action="store_true",
|
||||
help="More verbose logging")
|
||||
parser.add_argument("-s", "--stdout", action="store_true",
|
||||
help="Log to STDOUT")
|
||||
parser.add_argument("--start",
|
||||
help="Date to poll from (YYYY-MM-DDTHH:MM:SS), Exclusive")
|
||||
parser.add_argument("--end",
|
||||
help="Date to poll to (YYYY-MM-DDTHH:MM:SS), Inclusive")
|
||||
parser.add_argument("--subscription_id", help="The ID of the subscription",
|
||||
default=None)
|
||||
parser.add_argument("--tz",
|
||||
help="Your timezone, e.g Europe/London. Default utc",
|
||||
default="utc")
|
||||
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# Set up a logger for logging's sake
|
||||
log = logging.getLogger(__name__)
|
||||
logging.basicConfig(filename="poll.log", format="%(asctime)s - %(name)s - %(levelname)s - %(message)s")
|
||||
logging.basicConfig(
|
||||
filename="poll.log",
|
||||
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s")
|
||||
log.setLevel(logging.DEBUG if args.verbose else logging.INFO)
|
||||
|
||||
# If we want, print the output to stdout
|
||||
if args.stdout:
|
||||
formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
|
||||
ch = logging.StreamHandler(sys.stdout)
|
||||
formatter = logging.Formatter(
|
||||
"%(asctime)s - %(name)s - %(levelname)s - %(message)s")
|
||||
ch = logging.StreamHandler(sys.stdout)
|
||||
ch.setFormatter(formatter)
|
||||
log.addHandler(ch)
|
||||
|
||||
# Read in the remote server configurations
|
||||
configFile = "{}/remote-servers.yml".format(os.path.expanduser(args.configdir))
|
||||
log.debug("Opening config file %s", configFile)
|
||||
with open(configFile, "r") as f:
|
||||
config_file = "{}/remote-servers.yml".format(
|
||||
os.path.expanduser(args.configdir))
|
||||
|
||||
log.debug("Opening config file %s", config_file)
|
||||
with open(config_file, "r") as f:
|
||||
config = yaml.load(f.read())
|
||||
log.debug("Config read %s", config)
|
||||
|
||||
# Read in the local server configuration
|
||||
localConfig = "{}/local-server.yml".format(os.path.expanduser(args.configdir))
|
||||
local_config = "{}/local-server.yml".format(os.path.expanduser(args.configdir))
|
||||
log.debug("Reading local server config")
|
||||
with open(localConfig, "r") as f:
|
||||
localConfig = yaml.load(f.read())
|
||||
with open(local_config, "r") as f:
|
||||
local_config = yaml.load(f.read())
|
||||
|
||||
# Attempt to make contact with the local server
|
||||
log.info("Connecting to local server...")
|
||||
localClient = create_client(host = localConfig["host"],
|
||||
port = localConfig["port"],
|
||||
discovery_path = localConfig["discovery_path"],
|
||||
use_https = localConfig["use_https"],
|
||||
version = localConfig["taxii_version"],
|
||||
headers = localConfig["headers"])
|
||||
localClient.username = localConfig["auth"]["username"]
|
||||
localClient.password = localConfig["auth"]["password"]
|
||||
local_client = create_client(host=local_config["host"],
|
||||
port=local_config["port"],
|
||||
discovery_path=local_config["discovery_path"],
|
||||
use_https=local_config["use_https"],
|
||||
version=local_config["taxii_version"],
|
||||
headers=local_config["headers"])
|
||||
|
||||
localInbox = "{}://{}:{}{}".format("https" if localConfig["use_https"] else "http",
|
||||
localConfig["host"], localConfig["port"],
|
||||
localConfig["inbox_path"])
|
||||
local_client.username = local_config["auth"]["username"]
|
||||
local_client.password = local_config["auth"]["password"]
|
||||
|
||||
|
||||
local_inbox = "{}://{}:{}{}".format(
|
||||
"https" if local_config["use_https"] else "http",
|
||||
local_config["host"], local_config["port"],
|
||||
local_config["inbox_path"])
|
||||
|
||||
# Check that we're all good and authenticated
|
||||
try:
|
||||
list(localClient.discover_services())
|
||||
list(local_client.discover_services())
|
||||
except Exception as ex:
|
||||
log.fatal("Could not connect to local server")
|
||||
log.fatal(ex)
|
||||
|
@ -102,26 +117,31 @@ for server in config:
|
|||
log.debug("Creating client")
|
||||
log.debug("HOST:PORT : %s:%s", server["host"], server["port"])
|
||||
log.debug("DISCPATH: %s", server["discovery_path"])
|
||||
cli = create_client(host = server["host"],
|
||||
port = server["port"],
|
||||
discovery_path = server["discovery_path"],
|
||||
use_https = server["use_https"],
|
||||
version = server["taxii_version"],
|
||||
headers = server["headers"])
|
||||
|
||||
# Standard autodiscovery
|
||||
client_args = {
|
||||
"host": server["host"],
|
||||
"port": server["port"],
|
||||
"discovery_path": server["discovery_path"],
|
||||
"use_https": server["use_https"],
|
||||
"version": server["taxii_version"],
|
||||
"headers": server["headers"]
|
||||
}
|
||||
|
||||
cli = create_client(**client_args)
|
||||
|
||||
log.debug("Setting client log level")
|
||||
cli.log.setLevel(logging.DEBUG if args.verbose else logging.INFO)
|
||||
|
||||
|
||||
log.debug("Setting authentication...")
|
||||
cli.set_auth(username = server["auth"]["username"],
|
||||
password = server["auth"]["password"],
|
||||
ca_cert = server["auth"].get("ca_cert"),
|
||||
cert_file= server["auth"].get("cert_file"),
|
||||
key_file = server["auth"].get("key_file"),
|
||||
key_password = server["auth"].get("key_password"),
|
||||
jwt_auth_url = server["auth"].get("jwt_auth_url"),
|
||||
verify_ssl = server["auth"].get("verify_ssl"))
|
||||
cli.set_auth(username=server["auth"]["username"],
|
||||
password=server["auth"]["password"],
|
||||
ca_cert=server["auth"].get("ca_cert"),
|
||||
cert_file=server["auth"].get("cert_file"),
|
||||
key_file=server["auth"].get("key_file"),
|
||||
key_password=server["auth"].get("key_password"),
|
||||
jwt_auth_url=server["auth"].get("jwt_auth_url"),
|
||||
verify_ssl=server["auth"].get("verify_ssl"))
|
||||
|
||||
log.debug("Discovering services...")
|
||||
services = cli.discover_services()
|
||||
|
@ -130,18 +150,27 @@ for server in config:
|
|||
log.debug("Auth set.")
|
||||
for collection in server["collections"]:
|
||||
log.debug("Polling %s", collection)
|
||||
log.debug("Within date range %s - %s", poll_from or "Beginning of time", poll_to)
|
||||
server_uri_override = server.get("uri", None)
|
||||
if not server_uri_override.startswith("http"):
|
||||
server_uri_override = None
|
||||
if server_uri_override:
|
||||
log.debug("Poll URL override set to %s", server_uri_override)
|
||||
|
||||
log.debug("Within date range %s - %s",
|
||||
poll_from or "Beginning of time", poll_to)
|
||||
try:
|
||||
for content_block in cli.poll(collection_name=collection,
|
||||
subscription_id=subscription_id,
|
||||
begin_date=poll_from,
|
||||
end_date=poll_to):
|
||||
end_date=poll_to,
|
||||
uri=server.get("uri", None)):
|
||||
try:
|
||||
log.debug("Pushing block %s", content_block)
|
||||
localClient.push(content_block.content.decode("utf-8"),
|
||||
collection_names=localConfig["collections"],
|
||||
content_binding=content_block.binding,
|
||||
uri=localInbox)
|
||||
local_client.push(
|
||||
content_block.content.decode("utf-8"),
|
||||
collection_names=local_config["collections"],
|
||||
content_binding=content_block.binding,
|
||||
uri=local_inbox)
|
||||
except Exception as ex:
|
||||
log.error("FAILED TO PUSH BLOCK!")
|
||||
log.error("%s", content_block)
|
||||
|
|
Loading…
Reference in New Issue