From c86e4940afc4f88bfd7fc8e5f179bff98d7d9445 Mon Sep 17 00:00:00 2001 From: arcsector Date: Wed, 6 May 2020 16:58:21 -0700 Subject: [PATCH 01/24] Including pyaml's Loader to suppress warnings See these docs for info: https://github.com/yaml/pyyaml/wiki/PyYAML-yaml.load(input)-Deprecation --- scripts/run-taxii-poll.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/run-taxii-poll.py b/scripts/run-taxii-poll.py index f7e9a21..5be085c 100644 --- a/scripts/run-taxii-poll.py +++ b/scripts/run-taxii-poll.py @@ -2,6 +2,7 @@ from cabby import create_client from pyaml import yaml +from yaml import Loader import pytz import argparse import os @@ -51,14 +52,14 @@ config_file = "{}/remote-servers.yml".format( log.debug("Opening config file %s", config_file) with open(config_file, "r") as f: - config = yaml.load(f.read()) + config = yaml.load(f.read(), Loader=Loader) log.debug("Config read %s", config) # Read in the local server configuration local_config = "{}/local-server.yml".format(os.path.expanduser(args.configdir)) log.debug("Reading local server config") with open(local_config, "r") as f: - local_config = yaml.load(f.read()) + local_config = yaml.load(f.read(), Loader=Loader) # Attempt to make contact with the local server log.info("Connecting to local server...") From 53309e3477234111039eb4559b758aab69daae19 Mon Sep 17 00:00:00 2001 From: arcsector Date: Wed, 6 May 2020 17:00:04 -0700 Subject: [PATCH 02/24] Include Loader in hooks.py --- misp_taxii_hooks/hooks.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/misp_taxii_hooks/hooks.py b/misp_taxii_hooks/hooks.py index 4119797..85d7566 100644 --- a/misp_taxii_hooks/hooks.py +++ b/misp_taxii_hooks/hooks.py @@ -9,6 +9,7 @@ import pymisp import tempfile import logging from pyaml import yaml +from yaml import Loader from io import StringIO log = logging.getLogger("__main__") @@ -20,7 +21,7 @@ from opentaxii.signals import ( ## CONFIG if "OPENTAXII_CONFIG" in os.environ: print("Using config from {}".format(os.environ["OPENTAXII_CONFIG"])) - CONFIG = yaml.load(open(os.environ["OPENTAXII_CONFIG"], "r")) + CONFIG = yaml.load(open(os.environ["OPENTAXII_CONFIG"], "r"), Loader=Loader) else: print("Trying to use env variables...") if "MISP_URL" in os.environ: From b50a60485fe185ddcf5c587a284a3fa228bae673 Mon Sep 17 00:00:00 2001 From: haraksin Date: Wed, 6 May 2020 17:28:04 -0700 Subject: [PATCH 03/24] adding Loader to push_published_to_taxii adding local-server default yaml and remote-servers default yaml --- config/local-server.default.yaml | 12 ++++++++++++ config/remote-servers.default.yaml | 18 ++++++++++++++++++ scripts/push_published_to_taxii.py | 3 ++- 3 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 config/local-server.default.yaml create mode 100644 config/remote-servers.default.yaml diff --git a/config/local-server.default.yaml b/config/local-server.default.yaml new file mode 100644 index 0000000..2d90b5f --- /dev/null +++ b/config/local-server.default.yaml @@ -0,0 +1,12 @@ +host: localhost +port: 9000 +discovery_path: /services/discovery +inbox_path: /services/inbox +use_https: False +taxii_version: '1.1' +headers: +auth: + username: test + password: test +collections: + - collection \ No newline at end of file diff --git a/config/remote-servers.default.yaml b/config/remote-servers.default.yaml new file mode 100644 index 0000000..be7a211 --- /dev/null +++ b/config/remote-servers.default.yaml @@ -0,0 +1,18 @@ +- name: 'default' + host: localhost + port: 9000 + discovery_path: + use_https: False + taxii_version: '1.1' + headers: + auth: + username: + password: + cacert_path: + cert_file: + key_file: + key_password: + jwt_auth_url: + verify_ssl: True + collections: + - collection \ No newline at end of file diff --git a/scripts/push_published_to_taxii.py b/scripts/push_published_to_taxii.py index f8ca786..f465fe9 100644 --- a/scripts/push_published_to_taxii.py +++ b/scripts/push_published_to_taxii.py @@ -4,6 +4,7 @@ import sys import json import pymisp from pyaml import yaml +from yaml import Loader from cabby import create_client from misp_stix_converter.converters import lint_roller import logging @@ -20,7 +21,7 @@ log.setLevel(logging.DEBUG) log.info("Starting...") # Try to load in config if "OPENTAXII_CONFIG" in os.environ: - config = yaml.load(open(os.environ["OPENTAXII_CONFIG"], "r")) + config = yaml.load(open(os.environ["OPENTAXII_CONFIG"], "r"), Loader=Loader) else: print("OPENTAXII CONFIG NOT EXPORTED") sys.exit() From 277dfcdb97ed9de75a08e0d10676538d76ffee78 Mon Sep 17 00:00:00 2001 From: haraksin Date: Wed, 6 May 2020 21:05:36 -0700 Subject: [PATCH 04/24] Dealing with key issue in search Line 70 --- .gitignore | 1 + misp_taxii_hooks/hooks.py | 20 +++++++++++++++----- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index 55a48d0..b09e2b4 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ config.yaml __pycache__ build dist +hooks_2.py \ No newline at end of file diff --git a/misp_taxii_hooks/hooks.py b/misp_taxii_hooks/hooks.py index 85d7566..8a7ae0d 100644 --- a/misp_taxii_hooks/hooks.py +++ b/misp_taxii_hooks/hooks.py @@ -67,12 +67,22 @@ def post_stix(manager, content_block, collection_ids, service_id): for attrib in values: log.info("Checking for existence of %s", attrib) search = MISP.search("attributes", values=str(attrib)) - if search["response"]["Attribute"] != []: - # This means we have it! - log.info("%s is a duplicate, we'll ignore it.", attrib) - package.attributes.pop([x.value for x in package.attributes].index(attrib)) + if 'response' in search: + if search["response"]["Attribute"] != []: + # This means we have it! + log.info("%s is a duplicate, we'll ignore it.", attrib) + package.attributes.pop([x.value for x in package.attributes].index(attrib)) + else: + log.info("%s is unique, we'll keep it", attrib) + elif 'Attribute' in search: + if search["Attribute"] != []: + # This means we have it! + log.info("%s is a duplicate, we'll ignore it.", attrib) + package.attributes.pop([x.value for x in package.attributes].index(attrib)) + else: + log.info("%s is unique, we'll keep it", attrib) else: - log.info("%s is unique, we'll keep it", attrib) + log.error("Something went wrong with search, and it doesn't have an 'attribute' or a 'response' key: {}".format(search.keys())) # Push the event to MISP # TODO: There's probably a proper method to do this rather than json_full From c55f9994c6696a97254f91b204f145658f8800b3 Mon Sep 17 00:00:00 2001 From: haraksin Date: Wed, 13 May 2020 13:41:12 -0700 Subject: [PATCH 05/24] Adding deduplication as an optional item --- misp_taxii_hooks/hooks.py | 45 ++++++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/misp_taxii_hooks/hooks.py b/misp_taxii_hooks/hooks.py index 8a7ae0d..e7be3c9 100644 --- a/misp_taxii_hooks/hooks.py +++ b/misp_taxii_hooks/hooks.py @@ -34,11 +34,17 @@ else: else: print("Unknown misp API key. Set OPENTAXII_CONFIG or MISP_API.") misp_api = "UNKNOWN" + if "MISP_DEDUP" in os.environ: + misp_dedup = os.environ["MISP_DEDUP"] + else: + print("Unknown misp deduplication setting. Set OPENTAXII_CONFIG or MISP_DEDUP.") + misp_dedup = "UNKNOWN" CONFIG = { "misp" : { "url" : misp_url, - "api" : misp_api + "api" : misp_api, + "dedup" : misp_dedup } } @@ -64,25 +70,26 @@ def post_stix(manager, content_block, collection_ids, service_id): log.info("STIX loaded succesfully.") values = [x.value for x in package.attributes] log.info("Extracted %s", values) - for attrib in values: - log.info("Checking for existence of %s", attrib) - search = MISP.search("attributes", values=str(attrib)) - if 'response' in search: - if search["response"]["Attribute"] != []: - # This means we have it! - log.info("%s is a duplicate, we'll ignore it.", attrib) - package.attributes.pop([x.value for x in package.attributes].index(attrib)) + if CONFIG['MISP_DEDUP'] == "true" or CONFIG['MISP_DEDUP'] == "True" or CONFIG['MISP_DEDUP'] == "TRUE" or CONFIG['MISP_DEDUP'] == "UNKNOWN": + for attrib in values: + log.info("Checking for existence of %s", attrib) + search = MISP.search("attributes", values=str(attrib)) + if 'response' in search: + if search["response"]["Attribute"] != []: + # This means we have it! + log.info("%s is a duplicate, we'll ignore it.", attrib) + package.attributes.pop([x.value for x in package.attributes].index(attrib)) + else: + log.info("%s is unique, we'll keep it", attrib) + elif 'Attribute' in search: + if search["Attribute"] != []: + # This means we have it! + log.info("%s is a duplicate, we'll ignore it.", attrib) + package.attributes.pop([x.value for x in package.attributes].index(attrib)) + else: + log.info("%s is unique, we'll keep it", attrib) else: - log.info("%s is unique, we'll keep it", attrib) - elif 'Attribute' in search: - if search["Attribute"] != []: - # This means we have it! - log.info("%s is a duplicate, we'll ignore it.", attrib) - package.attributes.pop([x.value for x in package.attributes].index(attrib)) - else: - log.info("%s is unique, we'll keep it", attrib) - else: - log.error("Something went wrong with search, and it doesn't have an 'attribute' or a 'response' key: {}".format(search.keys())) + log.error("Something went wrong with search, and it doesn't have an 'attribute' or a 'response' key: {}".format(search.keys())) # Push the event to MISP # TODO: There's probably a proper method to do this rather than json_full From 823822bbf14d3e7a0b1436d332ce7864abbc7191 Mon Sep 17 00:00:00 2001 From: haraksin Date: Wed, 13 May 2020 13:49:34 -0700 Subject: [PATCH 06/24] Making dedup optional --- misp_taxii_hooks/hooks.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/misp_taxii_hooks/hooks.py b/misp_taxii_hooks/hooks.py index e7be3c9..25a3964 100644 --- a/misp_taxii_hooks/hooks.py +++ b/misp_taxii_hooks/hooks.py @@ -22,6 +22,8 @@ from opentaxii.signals import ( if "OPENTAXII_CONFIG" in os.environ: print("Using config from {}".format(os.environ["OPENTAXII_CONFIG"])) CONFIG = yaml.load(open(os.environ["OPENTAXII_CONFIG"], "r"), Loader=Loader) + if "dedup" not in CONFIG["misp"]: + CONFIG["misp"]["dedup"] = "UNKNOWN" else: print("Trying to use env variables...") if "MISP_URL" in os.environ: @@ -70,7 +72,11 @@ def post_stix(manager, content_block, collection_ids, service_id): log.info("STIX loaded succesfully.") values = [x.value for x in package.attributes] log.info("Extracted %s", values) - if CONFIG['MISP_DEDUP'] == "true" or CONFIG['MISP_DEDUP'] == "True" or CONFIG['MISP_DEDUP'] == "TRUE" or CONFIG['MISP_DEDUP'] == "UNKNOWN": + if ( + CONFIG["misp"]["dedup"] == True or + CONFIG["misp"]["dedup"] == "True" or + CONFIG["misp"]["dedup"] == "UNKNOWN" + ): for attrib in values: log.info("Checking for existence of %s", attrib) search = MISP.search("attributes", values=str(attrib)) From 8a204588988aa919ddf70c8b6ff561094e556982 Mon Sep 17 00:00:00 2001 From: haraksin Date: Wed, 13 May 2020 13:59:56 -0700 Subject: [PATCH 07/24] removing gitignore changes --- .gitignore | 3 +-- config/config.default.yaml | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index b09e2b4..477a63e 100644 --- a/.gitignore +++ b/.gitignore @@ -5,5 +5,4 @@ config.yaml *.egg-info __pycache__ build -dist -hooks_2.py \ No newline at end of file +dist \ No newline at end of file diff --git a/config/config.default.yaml b/config/config.default.yaml index d7f3469..9d05064 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -29,6 +29,7 @@ zmq: misp: url: "http://localhost" api: KEY + dedup: true taxii: auth: From f3c583f0f4dff7b08c8fe270c9b7b00acd02b07d Mon Sep 17 00:00:00 2001 From: George Haraksin Date: Wed, 13 May 2020 15:28:11 -0700 Subject: [PATCH 08/24] adding src/ to gitignure --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 477a63e..8056eda 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,5 @@ config.yaml *.egg-info __pycache__ build -dist \ No newline at end of file +dist +src From 00e70113ad82ea2f8988a696acb58cb70a3245b2 Mon Sep 17 00:00:00 2001 From: haraksin Date: Wed, 13 May 2020 15:29:20 -0700 Subject: [PATCH 09/24] Add src/ to gitignore --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 477a63e..7bf4e3a 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,5 @@ config.yaml *.egg-info __pycache__ build -dist \ No newline at end of file +dist +src \ No newline at end of file From 934bcf1b6ab9bf31413dfea436160aed24c83c19 Mon Sep 17 00:00:00 2001 From: haraksin Date: Wed, 13 May 2020 19:21:52 -0700 Subject: [PATCH 10/24] Adding collections and publish args and features. --- config/config.default.yaml | 4 ++ misp_taxii_hooks/hooks.py | 75 ++++++++++++++++++++++++++++---------- 2 files changed, 59 insertions(+), 20 deletions(-) diff --git a/config/config.default.yaml b/config/config.default.yaml index 9d05064..ce9947e 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -30,6 +30,10 @@ misp: url: "http://localhost" api: KEY dedup: true + collections: + - my_collection + - my_collection2 + publish: false taxii: auth: diff --git a/misp_taxii_hooks/hooks.py b/misp_taxii_hooks/hooks.py index 25a3964..70b5a36 100644 --- a/misp_taxii_hooks/hooks.py +++ b/misp_taxii_hooks/hooks.py @@ -18,35 +18,47 @@ from opentaxii.signals import ( CONTENT_BLOCK_CREATED, INBOX_MESSAGE_CREATED ) +def env_config_helper(env_name): + if env_name in os.environ: + if env_name == "MISP_COLLECTIONS": + name = os.environ[env_name] + return name.split(',') + return os.environ[env_name] + else: + print("Missing env setting {0}. Set OPENTAXII_CONFIG or {0}.".format(env_name)) + return "UNKNOWN" + +def yaml_config_helper(config_name, CONFIG): + if config_name in CONFIG["misp"]: + if not CONFIG["misp"][config_name]: + CONFIG["misp"][config_name] = "UNKNOWN" + else: + CONFIG["misp"][config_name] = "UNKNOWN" + return CONFIG + ## CONFIG if "OPENTAXII_CONFIG" in os.environ: print("Using config from {}".format(os.environ["OPENTAXII_CONFIG"])) CONFIG = yaml.load(open(os.environ["OPENTAXII_CONFIG"], "r"), Loader=Loader) - if "dedup" not in CONFIG["misp"]: - CONFIG["misp"]["dedup"] = "UNKNOWN" + # validate dedup and collections and publish + CONFIG = yaml_config_helper("dedup", CONFIG) + CONFIG = yaml_config_helper("collections", CONFIG) + CONFIG = yaml_config_helper("publish", CONFIG) + else: print("Trying to use env variables...") - if "MISP_URL" in os.environ: - misp_url = os.environ["MISP_URL"] - else: - print("Unkown misp URL. Set OPENTAXII_CONFIG or MISP_URL.") - misp_url = "UNKNOWN" - if "MISP_API" in os.environ: - misp_api = os.environ["MISP_API"] - else: - print("Unknown misp API key. Set OPENTAXII_CONFIG or MISP_API.") - misp_api = "UNKNOWN" - if "MISP_DEDUP" in os.environ: - misp_dedup = os.environ["MISP_DEDUP"] - else: - print("Unknown misp deduplication setting. Set OPENTAXII_CONFIG or MISP_DEDUP.") - misp_dedup = "UNKNOWN" + misp_url = env_config_helper("MISP_URL") + misp_api = env_config_helper("MISP_API") + misp_dedup = env_config_helper("MISP_DEDUP") + misp_collections = env_config_helper("MISP_COLLECTIONS") + misp_publish = env_config_helper("MISP_PUBLISH") CONFIG = { "misp" : { "url" : misp_url, "api" : misp_api, - "dedup" : misp_dedup + "dedup" : misp_dedup, + "collections": misp_collections } } @@ -62,6 +74,21 @@ def post_stix(manager, content_block, collection_ids, service_id): Will convert it to a MISPEvent and push to the server ''' + # make sure collections, if specified are supposed to be sent to + if CONFIG["misp"]["collections"] != "UNKNOWN" or CONFIG["misp"]["collections"] == False: + should_send_to_misp = False + for collection in CONFIG["misp"]["collections"]: + if collection in collection_ids: + should_send_to_misp = True + if should_send_to_misp == False: + log.info('''No collections match misp.collections; aborting MISP extraction. + Collection ids whitelisted: {} + Collection ids sent to: {}'''.format( + CONFIG["misp"]["collections"], + collection_ids + )) + return None + # Load the package log.info("Posting STIX...") block = content_block.content @@ -72,8 +99,10 @@ def post_stix(manager, content_block, collection_ids, service_id): log.info("STIX loaded succesfully.") values = [x.value for x in package.attributes] log.info("Extracted %s", values) + + # if deduping is enabled, start deduping if ( - CONFIG["misp"]["dedup"] == True or + CONFIG["misp"]["dedup"] or CONFIG["misp"]["dedup"] == "True" or CONFIG["misp"]["dedup"] == "UNKNOWN" ): @@ -102,7 +131,13 @@ def post_stix(manager, content_block, collection_ids, service_id): # But I don't wanna read docs if (len(package.attributes) > 0): log.info("Uploading event to MISP with attributes %s", [x.value for x in package.attributes]) - MISP.add_event(package) + event = MISP.add_event(package) + if ( + CONFIG["misp"]["publish"] or + CONFIG["misp"]["publish"] == "True" + ): + log.info("Publishing event to MISP with ID {}".format(event['id'])) + MISP.publish(event) else: log.info("No attributes, not bothering.") From 4a41e963df702656a8cd823029887bee693ccca6 Mon Sep 17 00:00:00 2001 From: haraksin Date: Thu, 14 May 2020 16:15:23 -0700 Subject: [PATCH 11/24] getting collection names for name lookup --- misp_taxii_hooks/hooks.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/misp_taxii_hooks/hooks.py b/misp_taxii_hooks/hooks.py index 70b5a36..4d863ec 100644 --- a/misp_taxii_hooks/hooks.py +++ b/misp_taxii_hooks/hooks.py @@ -77,15 +77,18 @@ def post_stix(manager, content_block, collection_ids, service_id): # make sure collections, if specified are supposed to be sent to if CONFIG["misp"]["collections"] != "UNKNOWN" or CONFIG["misp"]["collections"] == False: should_send_to_misp = False + collection_names = [collection.name for collection in manager.get_collections(service_id)] for collection in CONFIG["misp"]["collections"]: - if collection in collection_ids: + if collection in collection_names or collection in collection_ids: should_send_to_misp = True if should_send_to_misp == False: log.info('''No collections match misp.collections; aborting MISP extraction. Collection ids whitelisted: {} - Collection ids sent to: {}'''.format( + Collection ids sent to: {} + Collection names sent to: {}'''.format( CONFIG["misp"]["collections"], - collection_ids + collection_ids, + collection_names )) return None From 9b8ed9ddf9f88b1b5a55638b2a2635dab2e3b44b Mon Sep 17 00:00:00 2001 From: haraksin Date: Thu, 14 May 2020 16:18:53 -0700 Subject: [PATCH 12/24] Adding better logging --- misp_taxii_hooks/hooks.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/misp_taxii_hooks/hooks.py b/misp_taxii_hooks/hooks.py index 4d863ec..4b76251 100644 --- a/misp_taxii_hooks/hooks.py +++ b/misp_taxii_hooks/hooks.py @@ -76,11 +76,14 @@ def post_stix(manager, content_block, collection_ids, service_id): # make sure collections, if specified are supposed to be sent to if CONFIG["misp"]["collections"] != "UNKNOWN" or CONFIG["misp"]["collections"] == False: + log.info("Using collections") should_send_to_misp = False collection_names = [collection.name for collection in manager.get_collections(service_id)] for collection in CONFIG["misp"]["collections"]: if collection in collection_names or collection in collection_ids: + log.info("Collection specified matches push collection: {}".format(collection)) should_send_to_misp = True + break if should_send_to_misp == False: log.info('''No collections match misp.collections; aborting MISP extraction. Collection ids whitelisted: {} From 310d8145d5923c01891f299a25db8ec6895debb5 Mon Sep 17 00:00:00 2001 From: haraksin Date: Thu, 14 May 2020 16:26:51 -0700 Subject: [PATCH 13/24] Making sure config can't be switched to unknown if it's false --- misp_taxii_hooks/hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_taxii_hooks/hooks.py b/misp_taxii_hooks/hooks.py index 4b76251..f17300c 100644 --- a/misp_taxii_hooks/hooks.py +++ b/misp_taxii_hooks/hooks.py @@ -30,7 +30,7 @@ def env_config_helper(env_name): def yaml_config_helper(config_name, CONFIG): if config_name in CONFIG["misp"]: - if not CONFIG["misp"][config_name]: + if not CONFIG["misp"][config_name] and CONFIG["misp"][config_name] != False: CONFIG["misp"][config_name] = "UNKNOWN" else: CONFIG["misp"][config_name] = "UNKNOWN" From 4d9f63522b3dfa1bc76676b1bce69e0bd49eb1e5 Mon Sep 17 00:00:00 2001 From: haraksin Date: Thu, 14 May 2020 16:33:15 -0700 Subject: [PATCH 14/24] Using UUID instead of ID for publish event log --- misp_taxii_hooks/hooks.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/misp_taxii_hooks/hooks.py b/misp_taxii_hooks/hooks.py index f17300c..89bf7da 100644 --- a/misp_taxii_hooks/hooks.py +++ b/misp_taxii_hooks/hooks.py @@ -131,6 +131,8 @@ def post_stix(manager, content_block, collection_ids, service_id): log.info("%s is unique, we'll keep it", attrib) else: log.error("Something went wrong with search, and it doesn't have an 'attribute' or a 'response' key: {}".format(search.keys())) + else: + log.info("Skipping deduplication") # Push the event to MISP # TODO: There's probably a proper method to do this rather than json_full @@ -139,10 +141,10 @@ def post_stix(manager, content_block, collection_ids, service_id): log.info("Uploading event to MISP with attributes %s", [x.value for x in package.attributes]) event = MISP.add_event(package) if ( - CONFIG["misp"]["publish"] or + CONFIG["misp"]["publish"] == True or CONFIG["misp"]["publish"] == "True" ): - log.info("Publishing event to MISP with ID {}".format(event['id'])) + log.info("Publishing event to MISP with ID {}".format(event.get('uuid'))) MISP.publish(event) else: log.info("No attributes, not bothering.") From 5cc5f99c45f626409ad9993632ec970111dbbd9e Mon Sep 17 00:00:00 2001 From: haraksin Date: Thu, 14 May 2020 16:38:28 -0700 Subject: [PATCH 15/24] Making sure collections returned from manager instance are only the ones we want --- misp_taxii_hooks/hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_taxii_hooks/hooks.py b/misp_taxii_hooks/hooks.py index 89bf7da..9eccc7e 100644 --- a/misp_taxii_hooks/hooks.py +++ b/misp_taxii_hooks/hooks.py @@ -78,7 +78,7 @@ def post_stix(manager, content_block, collection_ids, service_id): if CONFIG["misp"]["collections"] != "UNKNOWN" or CONFIG["misp"]["collections"] == False: log.info("Using collections") should_send_to_misp = False - collection_names = [collection.name for collection in manager.get_collections(service_id)] + collection_names = [collection.name for collection in manager.get_collections(service_id) if collection.id in collection_ids] for collection in CONFIG["misp"]["collections"]: if collection in collection_names or collection in collection_ids: log.info("Collection specified matches push collection: {}".format(collection)) From 461452f4200d7b40200508b2b36414ed437f3ee8 Mon Sep 17 00:00:00 2001 From: haraksin Date: Thu, 14 May 2020 16:43:01 -0700 Subject: [PATCH 16/24] Adding log about MISP publish skip --- misp_taxii_hooks/hooks.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/misp_taxii_hooks/hooks.py b/misp_taxii_hooks/hooks.py index 9eccc7e..1fce618 100644 --- a/misp_taxii_hooks/hooks.py +++ b/misp_taxii_hooks/hooks.py @@ -146,6 +146,8 @@ def post_stix(manager, content_block, collection_ids, service_id): ): log.info("Publishing event to MISP with ID {}".format(event.get('uuid'))) MISP.publish(event) + else: + log.info("Skipping MISP event publishing") else: log.info("No attributes, not bothering.") From 6056501ff42eedc50fb5531ae03efad80da0cce4 Mon Sep 17 00:00:00 2001 From: haraksin Date: Mon, 15 Jun 2020 18:08:33 -0700 Subject: [PATCH 17/24] Adding checking if connection to MISP is valid --- .gitignore | 3 ++- misp_taxii_hooks/hooks.py | 18 ++++++++++++++---- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index 7bf4e3a..18aa560 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,5 @@ config.yaml __pycache__ build dist -src \ No newline at end of file +src +vscode/ \ No newline at end of file diff --git a/misp_taxii_hooks/hooks.py b/misp_taxii_hooks/hooks.py index 1fce618..6325a81 100644 --- a/misp_taxii_hooks/hooks.py +++ b/misp_taxii_hooks/hooks.py @@ -11,8 +11,15 @@ import logging from pyaml import yaml from yaml import Loader from io import StringIO +from requests.exceptions import ConnectionError +logging_level = logging.INFO log = logging.getLogger("__main__") +log.setLevel(logging_level) +handler.setLevel(logging_level) +formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') +handler.setFormatter(formatter) + from opentaxii.signals import ( CONTENT_BLOCK_CREATED, INBOX_MESSAGE_CREATED @@ -25,7 +32,7 @@ def env_config_helper(env_name): return name.split(',') return os.environ[env_name] else: - print("Missing env setting {0}. Set OPENTAXII_CONFIG or {0}.".format(env_name)) + log.error("Missing env setting {0}. Set OPENTAXII_CONFIG or {0}.".format(env_name)) return "UNKNOWN" def yaml_config_helper(config_name, CONFIG): @@ -38,7 +45,7 @@ def yaml_config_helper(config_name, CONFIG): ## CONFIG if "OPENTAXII_CONFIG" in os.environ: - print("Using config from {}".format(os.environ["OPENTAXII_CONFIG"])) + log.info("Using config from {}".format(os.environ["OPENTAXII_CONFIG"])) CONFIG = yaml.load(open(os.environ["OPENTAXII_CONFIG"], "r"), Loader=Loader) # validate dedup and collections and publish CONFIG = yaml_config_helper("dedup", CONFIG) @@ -46,7 +53,7 @@ if "OPENTAXII_CONFIG" in os.environ: CONFIG = yaml_config_helper("publish", CONFIG) else: - print("Trying to use env variables...") + log.debug("Trying to use env variables...") misp_url = env_config_helper("MISP_URL") misp_api = env_config_helper("MISP_API") misp_dedup = env_config_helper("MISP_DEDUP") @@ -139,7 +146,10 @@ def post_stix(manager, content_block, collection_ids, service_id): # But I don't wanna read docs if (len(package.attributes) > 0): log.info("Uploading event to MISP with attributes %s", [x.value for x in package.attributes]) - event = MISP.add_event(package) + try: + event = MISP.add_event(package) + except ConnectionError: + log.error("MISP-Taxii-Server - Cannot connect to MISP; please ensure that MISP is up and running at {}. Skipping MISP upload.".format(CONFIG['misp']['url'])) if ( CONFIG["misp"]["publish"] == True or CONFIG["misp"]["publish"] == "True" From e4ab93e954b3ee9148bd57f7b74cc5cb5da25d80 Mon Sep 17 00:00:00 2001 From: haraksin Date: Mon, 15 Jun 2020 18:12:24 -0700 Subject: [PATCH 18/24] fixing logging changes --- misp_taxii_hooks/hooks.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/misp_taxii_hooks/hooks.py b/misp_taxii_hooks/hooks.py index 6325a81..c64218d 100644 --- a/misp_taxii_hooks/hooks.py +++ b/misp_taxii_hooks/hooks.py @@ -15,10 +15,12 @@ from requests.exceptions import ConnectionError logging_level = logging.INFO log = logging.getLogger("__main__") +handler = logging.StreamHandler() log.setLevel(logging_level) handler.setLevel(logging_level) formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') handler.setFormatter(formatter) +log.addHandler(handler) from opentaxii.signals import ( From 67f9c52deff425e9db415f83bf794fbe4bdc9202 Mon Sep 17 00:00:00 2001 From: haraksin Date: Mon, 15 Jun 2020 19:00:47 -0700 Subject: [PATCH 19/24] adding try/except for pymisp error --- misp_taxii_hooks/hooks.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/misp_taxii_hooks/hooks.py b/misp_taxii_hooks/hooks.py index c64218d..f0fe453 100644 --- a/misp_taxii_hooks/hooks.py +++ b/misp_taxii_hooks/hooks.py @@ -12,6 +12,7 @@ from pyaml import yaml from yaml import Loader from io import StringIO from requests.exceptions import ConnectionError +from pymisp.exceptions import PyMISPError logging_level = logging.INFO log = logging.getLogger("__main__") @@ -70,12 +71,14 @@ else: "collections": misp_collections } } - -MISP = pymisp.PyMISP( +try: + MISP = pymisp.PyMISP( CONFIG["misp"]["url"], CONFIG["misp"]["api"], ssl = CONFIG["misp"].get("verifySSL", True) ) +except PyMISPError: + log.error("Cannot connect to MISP; please ensure that MISP is up and running at {}. Skipping MISP upload.".format(CONFIG['misp']['url'])) def post_stix(manager, content_block, collection_ids, service_id): ''' @@ -151,7 +154,7 @@ def post_stix(manager, content_block, collection_ids, service_id): try: event = MISP.add_event(package) except ConnectionError: - log.error("MISP-Taxii-Server - Cannot connect to MISP; please ensure that MISP is up and running at {}. Skipping MISP upload.".format(CONFIG['misp']['url'])) + log.error("Cannot push to MISP; please ensure that MISP is up and running at {}. Skipping MISP upload.".format(CONFIG['misp']['url'])) if ( CONFIG["misp"]["publish"] == True or CONFIG["misp"]["publish"] == "True" From 0aaed665403ce003690819d0de5d7972c2ae5390 Mon Sep 17 00:00:00 2001 From: haraksin Date: Mon, 15 Jun 2020 19:24:18 -0700 Subject: [PATCH 20/24] Checking for MISP existence --- misp_taxii_hooks/hooks.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/misp_taxii_hooks/hooks.py b/misp_taxii_hooks/hooks.py index f0fe453..fa6178f 100644 --- a/misp_taxii_hooks/hooks.py +++ b/misp_taxii_hooks/hooks.py @@ -71,6 +71,7 @@ else: "collections": misp_collections } } +MISP = '' try: MISP = pymisp.PyMISP( CONFIG["misp"]["url"], @@ -126,7 +127,9 @@ def post_stix(manager, content_block, collection_ids, service_id): ): for attrib in values: log.info("Checking for existence of %s", attrib) - search = MISP.search("attributes", values=str(attrib)) + search = '' + if MISP: + search = MISP.search("attributes", values=str(attrib)) if 'response' in search: if search["response"]["Attribute"] != []: # This means we have it! @@ -151,16 +154,19 @@ def post_stix(manager, content_block, collection_ids, service_id): # But I don't wanna read docs if (len(package.attributes) > 0): log.info("Uploading event to MISP with attributes %s", [x.value for x in package.attributes]) + event = '' try: - event = MISP.add_event(package) - except ConnectionError: + if MISP: + event = MISP.add_event(package) + except ConnectionError, NameError: log.error("Cannot push to MISP; please ensure that MISP is up and running at {}. Skipping MISP upload.".format(CONFIG['misp']['url'])) if ( CONFIG["misp"]["publish"] == True or CONFIG["misp"]["publish"] == "True" ): log.info("Publishing event to MISP with ID {}".format(event.get('uuid'))) - MISP.publish(event) + if MISP: + MISP.publish(event) else: log.info("Skipping MISP event publishing") else: From e020452a3d60a8573b21c2c493a9823c9c86634e Mon Sep 17 00:00:00 2001 From: haraksin Date: Mon, 15 Jun 2020 19:26:07 -0700 Subject: [PATCH 21/24] invalid syntaxi in except clause --- misp_taxii_hooks/hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_taxii_hooks/hooks.py b/misp_taxii_hooks/hooks.py index fa6178f..1c79021 100644 --- a/misp_taxii_hooks/hooks.py +++ b/misp_taxii_hooks/hooks.py @@ -158,7 +158,7 @@ def post_stix(manager, content_block, collection_ids, service_id): try: if MISP: event = MISP.add_event(package) - except ConnectionError, NameError: + except ConnectionError: log.error("Cannot push to MISP; please ensure that MISP is up and running at {}. Skipping MISP upload.".format(CONFIG['misp']['url'])) if ( CONFIG["misp"]["publish"] == True or From 67d08c443fcf32b86c79e20332583e238bd64d56 Mon Sep 17 00:00:00 2001 From: haraksin Date: Tue, 16 Jun 2020 01:12:23 -0700 Subject: [PATCH 22/24] try/except on pymisp loading stix failing --- misp_taxii_hooks/hooks.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/misp_taxii_hooks/hooks.py b/misp_taxii_hooks/hooks.py index 1c79021..a6f94fb 100644 --- a/misp_taxii_hooks/hooks.py +++ b/misp_taxii_hooks/hooks.py @@ -113,8 +113,12 @@ def post_stix(manager, content_block, collection_ids, service_id): block = content_block.content if isinstance(block, bytes): block = block.decode() - - package = pymisp.tools.stix.load_stix(StringIO(block)) + + try: + package = pymisp.tools.stix.load_stix(StringIO(block)) + except: + log.error('Could not load stix into MISP format; exiting.') + return 0 log.info("STIX loaded succesfully.") values = [x.value for x in package.attributes] log.info("Extracted %s", values) @@ -130,6 +134,8 @@ def post_stix(manager, content_block, collection_ids, service_id): search = '' if MISP: search = MISP.search("attributes", values=str(attrib)) + else: + return 0 if 'response' in search: if search["response"]["Attribute"] != []: # This means we have it! From 43a27c83b1403d1aaa9931bfbc61dab86fe1c2ed Mon Sep 17 00:00:00 2001 From: George Haraksin Date: Tue, 14 Jun 2022 00:01:09 -0700 Subject: [PATCH 23/24] even better logging --- misp_taxii_hooks/hooks.py | 56 ++++++++++++++++++++++----------------- 1 file changed, 31 insertions(+), 25 deletions(-) diff --git a/misp_taxii_hooks/hooks.py b/misp_taxii_hooks/hooks.py index a6f94fb..803c370 100644 --- a/misp_taxii_hooks/hooks.py +++ b/misp_taxii_hooks/hooks.py @@ -13,17 +13,24 @@ from yaml import Loader from io import StringIO from requests.exceptions import ConnectionError from pymisp.exceptions import PyMISPError +import warnings +warnings.filterwarnings("ignore") + +TOTAL_ATTRIBUTES_SENT = 0 +TOTAL_ATTRIBUTES_ANALYZED = 0 logging_level = logging.INFO -log = logging.getLogger("__main__") -handler = logging.StreamHandler() + +log = logging.getLogger("misp_taxii_server") log.setLevel(logging_level) +handler = logging.StreamHandler() handler.setLevel(logging_level) formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') handler.setFormatter(formatter) +if log.handlers: + log.handlers = [] log.addHandler(handler) - from opentaxii.signals import ( CONTENT_BLOCK_CREATED, INBOX_MESSAGE_CREATED ) @@ -48,7 +55,7 @@ def yaml_config_helper(config_name, CONFIG): ## CONFIG if "OPENTAXII_CONFIG" in os.environ: - log.info("Using config from {}".format(os.environ["OPENTAXII_CONFIG"])) + log.debug("Using config from {}".format(os.environ["OPENTAXII_CONFIG"])) CONFIG = yaml.load(open(os.environ["OPENTAXII_CONFIG"], "r"), Loader=Loader) # validate dedup and collections and publish CONFIG = yaml_config_helper("dedup", CONFIG) @@ -89,27 +96,23 @@ def post_stix(manager, content_block, collection_ids, service_id): # make sure collections, if specified are supposed to be sent to if CONFIG["misp"]["collections"] != "UNKNOWN" or CONFIG["misp"]["collections"] == False: - log.info("Using collections") + log.debug("Using collections") should_send_to_misp = False collection_names = [collection.name for collection in manager.get_collections(service_id) if collection.id in collection_ids] for collection in CONFIG["misp"]["collections"]: if collection in collection_names or collection in collection_ids: - log.info("Collection specified matches push collection: {}".format(collection)) + log.debug("Collection specified matches push collection: {}".format(collection)) should_send_to_misp = True break if should_send_to_misp == False: - log.info('''No collections match misp.collections; aborting MISP extraction. - Collection ids whitelisted: {} - Collection ids sent to: {} - Collection names sent to: {}'''.format( - CONFIG["misp"]["collections"], - collection_ids, - collection_names - )) + log.debug('''No collections match misp.collections; aborting MISP extraction.''') + log.debug("Collection ids whitelisted: {}".format(CONFIG["misp"]["collections"])) + log.debug("Collection ids sent to: {}".format(collection_ids)) + log.debug('''Collection names sent to: {}'''.format(collection_names)) return None # Load the package - log.info("Posting STIX...") + log.debug("Posting STIX...") block = content_block.content if isinstance(block, bytes): block = block.decode() @@ -119,9 +122,10 @@ def post_stix(manager, content_block, collection_ids, service_id): except: log.error('Could not load stix into MISP format; exiting.') return 0 - log.info("STIX loaded succesfully.") + log.debug("STIX loaded succesfully.") values = [x.value for x in package.attributes] - log.info("Extracted %s", values) + log.debug("Extracted %s", values) + TOTAL_ATTRIBUTES_ANALYZED = len(values) # if deduping is enabled, start deduping if ( @@ -130,7 +134,7 @@ def post_stix(manager, content_block, collection_ids, service_id): CONFIG["misp"]["dedup"] == "UNKNOWN" ): for attrib in values: - log.info("Checking for existence of %s", attrib) + log.debug("Checking for existence of %s", attrib) search = '' if MISP: search = MISP.search("attributes", values=str(attrib)) @@ -139,31 +143,32 @@ def post_stix(manager, content_block, collection_ids, service_id): if 'response' in search: if search["response"]["Attribute"] != []: # This means we have it! - log.info("%s is a duplicate, we'll ignore it.", attrib) + log.debug("%s is a duplicate, we'll ignore it.", attrib) package.attributes.pop([x.value for x in package.attributes].index(attrib)) else: - log.info("%s is unique, we'll keep it", attrib) + log.debug("%s is unique, we'll keep it", attrib) elif 'Attribute' in search: if search["Attribute"] != []: # This means we have it! - log.info("%s is a duplicate, we'll ignore it.", attrib) + log.debug("%s is a duplicate, we'll ignore it.", attrib) package.attributes.pop([x.value for x in package.attributes].index(attrib)) else: - log.info("%s is unique, we'll keep it", attrib) + log.debug("%s is unique, we'll keep it", attrib) else: log.error("Something went wrong with search, and it doesn't have an 'attribute' or a 'response' key: {}".format(search.keys())) else: - log.info("Skipping deduplication") + log.debug("Skipping deduplication") # Push the event to MISP # TODO: There's probably a proper method to do this rather than json_full # But I don't wanna read docs if (len(package.attributes) > 0): - log.info("Uploading event to MISP with attributes %s", [x.value for x in package.attributes]) + log.debug("Uploading event to MISP with attributes %s", [x.value for x in package.attributes]) event = '' try: if MISP: event = MISP.add_event(package) + TOTAL_ATTRIBUTES_SENT = len(package.attributes) except ConnectionError: log.error("Cannot push to MISP; please ensure that MISP is up and running at {}. Skipping MISP upload.".format(CONFIG['misp']['url'])) if ( @@ -174,9 +179,10 @@ def post_stix(manager, content_block, collection_ids, service_id): if MISP: MISP.publish(event) else: - log.info("Skipping MISP event publishing") + log.debug("Skipping MISP event publishing") else: log.info("No attributes, not bothering.") +log.debug("total_attributes_analyzed={}, total_attributes_sent={}".format(TOTAL_ATTRIBUTES_ANALYZED, TOTAL_ATTRIBUTES_SENT)) # Make TAXII call our push function whenever it gets new data CONTENT_BLOCK_CREATED.connect(post_stix) From 2aa3522cae0efc4701112c13d58e4053324faa36 Mon Sep 17 00:00:00 2001 From: George Haraksin Date: Tue, 14 Jun 2022 00:02:15 -0700 Subject: [PATCH 24/24] excepting Exception instead of BaseException --- misp_taxii_hooks/hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_taxii_hooks/hooks.py b/misp_taxii_hooks/hooks.py index 803c370..7d2e1d2 100644 --- a/misp_taxii_hooks/hooks.py +++ b/misp_taxii_hooks/hooks.py @@ -119,7 +119,7 @@ def post_stix(manager, content_block, collection_ids, service_id): try: package = pymisp.tools.stix.load_stix(StringIO(block)) - except: + except Exception: log.error('Could not load stix into MISP format; exiting.') return 0 log.debug("STIX loaded succesfully.")