diff --git a/.gitignore b/.gitignore index 55a48d0..4429bc1 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,5 @@ config.yaml __pycache__ build dist +src +vscode/ diff --git a/config/config.default.yaml b/config/config.default.yaml index d7f3469..ce9947e 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -29,6 +29,11 @@ zmq: misp: url: "http://localhost" api: KEY + dedup: true + collections: + - my_collection + - my_collection2 + publish: false taxii: auth: diff --git a/config/local-server.default.yaml b/config/local-server.default.yaml new file mode 100644 index 0000000..2d90b5f --- /dev/null +++ b/config/local-server.default.yaml @@ -0,0 +1,12 @@ +host: localhost +port: 9000 +discovery_path: /services/discovery +inbox_path: /services/inbox +use_https: False +taxii_version: '1.1' +headers: +auth: + username: test + password: test +collections: + - collection \ No newline at end of file diff --git a/config/remote-servers.default.yaml b/config/remote-servers.default.yaml new file mode 100644 index 0000000..be7a211 --- /dev/null +++ b/config/remote-servers.default.yaml @@ -0,0 +1,18 @@ +- name: 'default' + host: localhost + port: 9000 + discovery_path: + use_https: False + taxii_version: '1.1' + headers: + auth: + username: + password: + cacert_path: + cert_file: + key_file: + key_password: + jwt_auth_url: + verify_ssl: True + collections: + - collection \ No newline at end of file diff --git a/misp_taxii_hooks/hooks.py b/misp_taxii_hooks/hooks.py index 4119797..7d2e1d2 100644 --- a/misp_taxii_hooks/hooks.py +++ b/misp_taxii_hooks/hooks.py @@ -9,43 +9,84 @@ import pymisp import tempfile import logging from pyaml import yaml +from yaml import Loader from io import StringIO +from requests.exceptions import ConnectionError +from pymisp.exceptions import PyMISPError +import warnings +warnings.filterwarnings("ignore") -log = logging.getLogger("__main__") +TOTAL_ATTRIBUTES_SENT = 0 +TOTAL_ATTRIBUTES_ANALYZED = 0 + +logging_level = logging.INFO + +log = logging.getLogger("misp_taxii_server") +log.setLevel(logging_level) +handler = logging.StreamHandler() +handler.setLevel(logging_level) +formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') +handler.setFormatter(formatter) +if log.handlers: + log.handlers = [] +log.addHandler(handler) from opentaxii.signals import ( CONTENT_BLOCK_CREATED, INBOX_MESSAGE_CREATED ) +def env_config_helper(env_name): + if env_name in os.environ: + if env_name == "MISP_COLLECTIONS": + name = os.environ[env_name] + return name.split(',') + return os.environ[env_name] + else: + log.error("Missing env setting {0}. Set OPENTAXII_CONFIG or {0}.".format(env_name)) + return "UNKNOWN" + +def yaml_config_helper(config_name, CONFIG): + if config_name in CONFIG["misp"]: + if not CONFIG["misp"][config_name] and CONFIG["misp"][config_name] != False: + CONFIG["misp"][config_name] = "UNKNOWN" + else: + CONFIG["misp"][config_name] = "UNKNOWN" + return CONFIG + ## CONFIG if "OPENTAXII_CONFIG" in os.environ: - print("Using config from {}".format(os.environ["OPENTAXII_CONFIG"])) - CONFIG = yaml.load(open(os.environ["OPENTAXII_CONFIG"], "r")) + log.debug("Using config from {}".format(os.environ["OPENTAXII_CONFIG"])) + CONFIG = yaml.load(open(os.environ["OPENTAXII_CONFIG"], "r"), Loader=Loader) + # validate dedup and collections and publish + CONFIG = yaml_config_helper("dedup", CONFIG) + CONFIG = yaml_config_helper("collections", CONFIG) + CONFIG = yaml_config_helper("publish", CONFIG) + else: - print("Trying to use env variables...") - if "MISP_URL" in os.environ: - misp_url = os.environ["MISP_URL"] - else: - print("Unkown misp URL. Set OPENTAXII_CONFIG or MISP_URL.") - misp_url = "UNKNOWN" - if "MISP_API" in os.environ: - misp_api = os.environ["MISP_API"] - else: - print("Unknown misp API key. Set OPENTAXII_CONFIG or MISP_API.") - misp_api = "UNKNOWN" + log.debug("Trying to use env variables...") + misp_url = env_config_helper("MISP_URL") + misp_api = env_config_helper("MISP_API") + misp_dedup = env_config_helper("MISP_DEDUP") + misp_collections = env_config_helper("MISP_COLLECTIONS") + misp_publish = env_config_helper("MISP_PUBLISH") CONFIG = { "misp" : { "url" : misp_url, - "api" : misp_api + "api" : misp_api, + "dedup" : misp_dedup, + "collections": misp_collections } } - -MISP = pymisp.PyMISP( +MISP = '' +try: + MISP = pymisp.PyMISP( CONFIG["misp"]["url"], CONFIG["misp"]["api"], ssl = CONFIG["misp"].get("verifySSL", True) ) +except PyMISPError: + log.error("Cannot connect to MISP; please ensure that MISP is up and running at {}. Skipping MISP upload.".format(CONFIG['misp']['url'])) def post_stix(manager, content_block, collection_ids, service_id): ''' @@ -53,34 +94,95 @@ def post_stix(manager, content_block, collection_ids, service_id): Will convert it to a MISPEvent and push to the server ''' + # make sure collections, if specified are supposed to be sent to + if CONFIG["misp"]["collections"] != "UNKNOWN" or CONFIG["misp"]["collections"] == False: + log.debug("Using collections") + should_send_to_misp = False + collection_names = [collection.name for collection in manager.get_collections(service_id) if collection.id in collection_ids] + for collection in CONFIG["misp"]["collections"]: + if collection in collection_names or collection in collection_ids: + log.debug("Collection specified matches push collection: {}".format(collection)) + should_send_to_misp = True + break + if should_send_to_misp == False: + log.debug('''No collections match misp.collections; aborting MISP extraction.''') + log.debug("Collection ids whitelisted: {}".format(CONFIG["misp"]["collections"])) + log.debug("Collection ids sent to: {}".format(collection_ids)) + log.debug('''Collection names sent to: {}'''.format(collection_names)) + return None + # Load the package - log.info("Posting STIX...") + log.debug("Posting STIX...") block = content_block.content if isinstance(block, bytes): block = block.decode() - - package = pymisp.tools.stix.load_stix(StringIO(block)) - log.info("STIX loaded succesfully.") + + try: + package = pymisp.tools.stix.load_stix(StringIO(block)) + except Exception: + log.error('Could not load stix into MISP format; exiting.') + return 0 + log.debug("STIX loaded succesfully.") values = [x.value for x in package.attributes] - log.info("Extracted %s", values) - for attrib in values: - log.info("Checking for existence of %s", attrib) - search = MISP.search("attributes", values=str(attrib)) - if search["response"]["Attribute"] != []: - # This means we have it! - log.info("%s is a duplicate, we'll ignore it.", attrib) - package.attributes.pop([x.value for x in package.attributes].index(attrib)) - else: - log.info("%s is unique, we'll keep it", attrib) + log.debug("Extracted %s", values) + TOTAL_ATTRIBUTES_ANALYZED = len(values) + + # if deduping is enabled, start deduping + if ( + CONFIG["misp"]["dedup"] or + CONFIG["misp"]["dedup"] == "True" or + CONFIG["misp"]["dedup"] == "UNKNOWN" + ): + for attrib in values: + log.debug("Checking for existence of %s", attrib) + search = '' + if MISP: + search = MISP.search("attributes", values=str(attrib)) + else: + return 0 + if 'response' in search: + if search["response"]["Attribute"] != []: + # This means we have it! + log.debug("%s is a duplicate, we'll ignore it.", attrib) + package.attributes.pop([x.value for x in package.attributes].index(attrib)) + else: + log.debug("%s is unique, we'll keep it", attrib) + elif 'Attribute' in search: + if search["Attribute"] != []: + # This means we have it! + log.debug("%s is a duplicate, we'll ignore it.", attrib) + package.attributes.pop([x.value for x in package.attributes].index(attrib)) + else: + log.debug("%s is unique, we'll keep it", attrib) + else: + log.error("Something went wrong with search, and it doesn't have an 'attribute' or a 'response' key: {}".format(search.keys())) + else: + log.debug("Skipping deduplication") # Push the event to MISP # TODO: There's probably a proper method to do this rather than json_full # But I don't wanna read docs if (len(package.attributes) > 0): - log.info("Uploading event to MISP with attributes %s", [x.value for x in package.attributes]) - MISP.add_event(package) + log.debug("Uploading event to MISP with attributes %s", [x.value for x in package.attributes]) + event = '' + try: + if MISP: + event = MISP.add_event(package) + TOTAL_ATTRIBUTES_SENT = len(package.attributes) + except ConnectionError: + log.error("Cannot push to MISP; please ensure that MISP is up and running at {}. Skipping MISP upload.".format(CONFIG['misp']['url'])) + if ( + CONFIG["misp"]["publish"] == True or + CONFIG["misp"]["publish"] == "True" + ): + log.info("Publishing event to MISP with ID {}".format(event.get('uuid'))) + if MISP: + MISP.publish(event) + else: + log.debug("Skipping MISP event publishing") else: log.info("No attributes, not bothering.") +log.debug("total_attributes_analyzed={}, total_attributes_sent={}".format(TOTAL_ATTRIBUTES_ANALYZED, TOTAL_ATTRIBUTES_SENT)) # Make TAXII call our push function whenever it gets new data CONTENT_BLOCK_CREATED.connect(post_stix) diff --git a/scripts/push_published_to_taxii.py b/scripts/push_published_to_taxii.py index f8ca786..f465fe9 100644 --- a/scripts/push_published_to_taxii.py +++ b/scripts/push_published_to_taxii.py @@ -4,6 +4,7 @@ import sys import json import pymisp from pyaml import yaml +from yaml import Loader from cabby import create_client from misp_stix_converter.converters import lint_roller import logging @@ -20,7 +21,7 @@ log.setLevel(logging.DEBUG) log.info("Starting...") # Try to load in config if "OPENTAXII_CONFIG" in os.environ: - config = yaml.load(open(os.environ["OPENTAXII_CONFIG"], "r")) + config = yaml.load(open(os.environ["OPENTAXII_CONFIG"], "r"), Loader=Loader) else: print("OPENTAXII CONFIG NOT EXPORTED") sys.exit() diff --git a/scripts/run-taxii-poll.py b/scripts/run-taxii-poll.py index f7e9a21..5be085c 100644 --- a/scripts/run-taxii-poll.py +++ b/scripts/run-taxii-poll.py @@ -2,6 +2,7 @@ from cabby import create_client from pyaml import yaml +from yaml import Loader import pytz import argparse import os @@ -51,14 +52,14 @@ config_file = "{}/remote-servers.yml".format( log.debug("Opening config file %s", config_file) with open(config_file, "r") as f: - config = yaml.load(f.read()) + config = yaml.load(f.read(), Loader=Loader) log.debug("Config read %s", config) # Read in the local server configuration local_config = "{}/local-server.yml".format(os.path.expanduser(args.configdir)) log.debug("Reading local server config") with open(local_config, "r") as f: - local_config = yaml.load(f.read()) + local_config = yaml.load(f.read(), Loader=Loader) # Attempt to make contact with the local server log.info("Connecting to local server...")