Merge pull request #82 from arcsector/master

Bringing up-to-date with recent PyMISP and Optional Deduplication

.gitignore

@@ -6,3 +6,5 @@ config.yaml
 __pycache__
 build
 dist
+src
+vscode/

config/config.default.yaml

@@ -29,6 +29,11 @@ zmq:
 misp:
     url: "http://localhost"
     api: KEY
+    dedup: true
+    collections:
+      - my_collection
+      - my_collection2
+    publish: false
 
 taxii:
     auth:

config/local-server.default.yaml

@@ -0,0 +1,12 @@
+host: localhost
+port: 9000
+discovery_path: /services/discovery
+inbox_path: /services/inbox
+use_https: False
+taxii_version: '1.1'
+headers:
+auth:
+  username: test
+  password: test
+collections:
+  - collection

config/remote-servers.default.yaml

@@ -0,0 +1,18 @@
+- name: 'default'
+  host: localhost
+  port: 9000
+  discovery_path:
+  use_https: False
+  taxii_version: '1.1'
+  headers:
+  auth:
+    username:
+    password:
+    cacert_path:
+    cert_file:
+    key_file:
+    key_password:
+    jwt_auth_url:
+    verify_ssl: True
+  collections:
+    - collection

misp_taxii_hooks/hooks.py

@@ -9,43 +9,84 @@ import pymisp
 import tempfile
 import logging
 from pyaml import yaml
+from yaml import Loader
 from io import StringIO
+from requests.exceptions import ConnectionError
+from pymisp.exceptions import PyMISPError
+import warnings
+warnings.filterwarnings("ignore")
 
-log = logging.getLogger("__main__")
+TOTAL_ATTRIBUTES_SENT = 0
+TOTAL_ATTRIBUTES_ANALYZED = 0
+
+logging_level = logging.INFO
+
+log = logging.getLogger("misp_taxii_server")
+log.setLevel(logging_level)
+handler = logging.StreamHandler()
+handler.setLevel(logging_level)
+formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+handler.setFormatter(formatter)
+if log.handlers:
+    log.handlers = []
+log.addHandler(handler)
 
 from opentaxii.signals import (
     CONTENT_BLOCK_CREATED, INBOX_MESSAGE_CREATED
 )
 
+def env_config_helper(env_name):
+    if env_name in os.environ:
+        if env_name == "MISP_COLLECTIONS":
+            name = os.environ[env_name]
+            return name.split(',')
+        return os.environ[env_name]
+    else:
+        log.error("Missing env setting {0}. Set OPENTAXII_CONFIG or {0}.".format(env_name))
+        return "UNKNOWN"
+
+def yaml_config_helper(config_name, CONFIG):
+    if config_name in CONFIG["misp"]:
+        if not CONFIG["misp"][config_name] and CONFIG["misp"][config_name] != False:
+            CONFIG["misp"][config_name] = "UNKNOWN"
+    else:
+        CONFIG["misp"][config_name] = "UNKNOWN"
+    return CONFIG
+
 ## CONFIG
 if "OPENTAXII_CONFIG" in os.environ:
-    print("Using config from {}".format(os.environ["OPENTAXII_CONFIG"]))
+    log.debug("Using config from {}".format(os.environ["OPENTAXII_CONFIG"]))
-    CONFIG =  yaml.load(open(os.environ["OPENTAXII_CONFIG"], "r"))
+    CONFIG =  yaml.load(open(os.environ["OPENTAXII_CONFIG"], "r"), Loader=Loader)
+    # validate dedup and collections and publish
+    CONFIG = yaml_config_helper("dedup", CONFIG)
+    CONFIG = yaml_config_helper("collections", CONFIG)
+    CONFIG = yaml_config_helper("publish", CONFIG)
+
 else:
-    print("Trying to use env variables...")
+    log.debug("Trying to use env variables...")
-    if "MISP_URL" in os.environ:
+    misp_url = env_config_helper("MISP_URL")
-        misp_url = os.environ["MISP_URL"]
+    misp_api = env_config_helper("MISP_API")
-    else:
+    misp_dedup = env_config_helper("MISP_DEDUP")
-        print("Unkown misp URL. Set OPENTAXII_CONFIG or MISP_URL.")
+    misp_collections = env_config_helper("MISP_COLLECTIONS")
-        misp_url = "UNKNOWN"
+    misp_publish = env_config_helper("MISP_PUBLISH")
-    if "MISP_API" in os.environ:
-        misp_api = os.environ["MISP_API"]
-    else:
-        print("Unknown misp API key. Set OPENTAXII_CONFIG or MISP_API.")
-        misp_api = "UNKNOWN"
 
     CONFIG = {
                 "misp" : {
                             "url" : misp_url,
-                            "api" : misp_api
+                            "api" : misp_api,
+                            "dedup" : misp_dedup,
+                            "collections": misp_collections
                         }
             }
-
+MISP = ''
+try:
     MISP = pymisp.PyMISP( 
                         CONFIG["misp"]["url"],
                         CONFIG["misp"]["api"],
                         ssl = CONFIG["misp"].get("verifySSL", True)
                 )
+except PyMISPError:
+    log.error("Cannot connect to MISP; please ensure that MISP is up and running at {}. Skipping MISP upload.".format(CONFIG['misp']['url']))
 
 def post_stix(manager, content_block, collection_ids, service_id):
     '''
@@ -53,34 +94,95 @@ def post_stix(manager, content_block, collection_ids, service_id):
         Will convert it to a MISPEvent and push to the server
     '''
 
+    # make sure collections, if specified are supposed to be sent to 
+    if CONFIG["misp"]["collections"] != "UNKNOWN" or CONFIG["misp"]["collections"] == False:
+        log.debug("Using collections")
+        should_send_to_misp = False
+        collection_names = [collection.name for collection in manager.get_collections(service_id) if collection.id in collection_ids]
+        for collection in CONFIG["misp"]["collections"]:
+            if collection in collection_names or collection in collection_ids:
+                log.debug("Collection specified matches push collection: {}".format(collection))
+                should_send_to_misp = True
+                break
+        if should_send_to_misp == False:
+            log.debug('''No collections match misp.collections; aborting MISP extraction.''')
+            log.debug("Collection ids whitelisted: {}".format(CONFIG["misp"]["collections"]))
+            log.debug("Collection ids sent to: {}".format(collection_ids))
+            log.debug('''Collection names sent to: {}'''.format(collection_names))
+            return None
+
     # Load the package
-    log.info("Posting STIX...")
+    log.debug("Posting STIX...")
     block = content_block.content
     if isinstance(block, bytes):
         block = block.decode()
 
+    try:
         package = pymisp.tools.stix.load_stix(StringIO(block))
-    log.info("STIX loaded succesfully.")
+    except Exception:
+        log.error('Could not load stix into MISP format; exiting.')
+        return 0
+    log.debug("STIX loaded succesfully.")
     values = [x.value for x in package.attributes]
-    log.info("Extracted %s", values)
+    log.debug("Extracted %s", values)
+    TOTAL_ATTRIBUTES_ANALYZED = len(values)
+
+    # if deduping is enabled, start deduping
+    if (
+        CONFIG["misp"]["dedup"] or 
+        CONFIG["misp"]["dedup"] == "True" or 
+        CONFIG["misp"]["dedup"] == "UNKNOWN"
+    ):
         for attrib in values:
-        log.info("Checking for existence of %s", attrib)
+            log.debug("Checking for existence of %s", attrib)
+            search = ''
+            if MISP:
                 search = MISP.search("attributes", values=str(attrib))
+            else:
+                return 0
+            if 'response' in search:
                 if search["response"]["Attribute"] != []:
                     # This means we have it!
-            log.info("%s is a duplicate, we'll ignore it.", attrib)
+                    log.debug("%s is a duplicate, we'll ignore it.", attrib)
                     package.attributes.pop([x.value for x in package.attributes].index(attrib))
                 else:
-            log.info("%s is unique, we'll keep it", attrib)
+                    log.debug("%s is unique, we'll keep it", attrib)
+            elif 'Attribute' in search:
+                if search["Attribute"] != []:
+                    # This means we have it!
+                    log.debug("%s is a duplicate, we'll ignore it.", attrib)
+                    package.attributes.pop([x.value for x in package.attributes].index(attrib))
+                else:
+                    log.debug("%s is unique, we'll keep it", attrib)
+            else:
+                log.error("Something went wrong with search, and it doesn't have an 'attribute' or a 'response' key: {}".format(search.keys()))
+    else:
+        log.debug("Skipping deduplication")
 
     # Push the event to MISP
     # TODO: There's probably a proper method to do this rather than json_full
     # But I don't wanna read docs
     if (len(package.attributes) > 0):
-        log.info("Uploading event to MISP with attributes %s", [x.value for x in package.attributes])
+        log.debug("Uploading event to MISP with attributes %s", [x.value for x in package.attributes])
-        MISP.add_event(package)
+        event = ''
+        try:
+            if MISP:
+                event = MISP.add_event(package)
+                TOTAL_ATTRIBUTES_SENT = len(package.attributes)
+        except ConnectionError:
+            log.error("Cannot push to MISP; please ensure that MISP is up and running at {}. Skipping MISP upload.".format(CONFIG['misp']['url']))
+        if (
+            CONFIG["misp"]["publish"] == True or
+            CONFIG["misp"]["publish"] == "True"
+        ):
+            log.info("Publishing event to MISP with ID {}".format(event.get('uuid')))
+            if MISP:
+                MISP.publish(event)
+        else:
+            log.debug("Skipping MISP event publishing")
     else:
         log.info("No attributes, not bothering.")
 
+log.debug("total_attributes_analyzed={}, total_attributes_sent={}".format(TOTAL_ATTRIBUTES_ANALYZED, TOTAL_ATTRIBUTES_SENT))
 # Make TAXII call our push function whenever it gets new data
 CONTENT_BLOCK_CREATED.connect(post_stix)

scripts/push_published_to_taxii.py

@@ -4,6 +4,7 @@ import sys
 import json
 import pymisp
 from pyaml import yaml
+from yaml import Loader
 from cabby import create_client
 from misp_stix_converter.converters import lint_roller
 import logging
@@ -20,7 +21,7 @@ log.setLevel(logging.DEBUG)
 log.info("Starting...")
 # Try to load in config
 if "OPENTAXII_CONFIG" in os.environ:
-    config = yaml.load(open(os.environ["OPENTAXII_CONFIG"], "r"))
+    config = yaml.load(open(os.environ["OPENTAXII_CONFIG"], "r"), Loader=Loader)
 else:
     print("OPENTAXII CONFIG NOT EXPORTED")
     sys.exit()

scripts/run-taxii-poll.py

@@ -2,6 +2,7 @@
 
 from cabby import create_client
 from pyaml import yaml
+from yaml import Loader
 import pytz
 import argparse
 import os
@@ -51,14 +52,14 @@ config_file = "{}/remote-servers.yml".format(
 
 log.debug("Opening config file %s", config_file)
 with open(config_file, "r") as f:
-    config = yaml.load(f.read())
+    config = yaml.load(f.read(), Loader=Loader)
 log.debug("Config read %s", config)
 
 # Read in the local server configuration
 local_config = "{}/local-server.yml".format(os.path.expanduser(args.configdir))
 log.debug("Reading local server config")
 with open(local_config, "r") as f:
-    local_config = yaml.load(f.read())
+    local_config = yaml.load(f.read(), Loader=Loader)
 
 # Attempt to make contact with the local server
 log.info("Connecting to local server...")