MISP-Taxii-Server/misp_taxii_hooks/hooks.py

#!/usr/bin/env python3

######
# TODO: DETECT DUPLICATE DATA
#####

import os
import pymisp
import tempfile
import logging
from pyaml import yaml
from yaml import Loader
from io import StringIO
from requests.exceptions import ConnectionError
from pymisp.exceptions import PyMISPError
import warnings
warnings.filterwarnings("ignore")

TOTAL_ATTRIBUTES_SENT = 0
TOTAL_ATTRIBUTES_ANALYZED = 0

logging_level = logging.INFO

log = logging.getLogger("misp_taxii_server")
log.setLevel(logging_level)
handler = logging.StreamHandler()
handler.setLevel(logging_level)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)
if log.handlers:
    log.handlers = []
log.addHandler(handler)

from opentaxii.signals import (
    CONTENT_BLOCK_CREATED, INBOX_MESSAGE_CREATED
)

def env_config_helper(env_name):
    if env_name in os.environ:
        if env_name == "MISP_COLLECTIONS":
            name = os.environ[env_name]
            return name.split(',')
        return os.environ[env_name]
    else:
        log.error("Missing env setting {0}. Set OPENTAXII_CONFIG or {0}.".format(env_name))
        return "UNKNOWN"

def yaml_config_helper(config_name, CONFIG):
    if config_name in CONFIG["misp"]:
        if not CONFIG["misp"][config_name] and CONFIG["misp"][config_name] != False:
            CONFIG["misp"][config_name] = "UNKNOWN"
    else:
        CONFIG["misp"][config_name] = "UNKNOWN"
    return CONFIG

## CONFIG
if "OPENTAXII_CONFIG" in os.environ:
    log.debug("Using config from {}".format(os.environ["OPENTAXII_CONFIG"]))
    CONFIG =  yaml.load(open(os.environ["OPENTAXII_CONFIG"], "r"), Loader=Loader)
    # validate dedup and collections and publish
    CONFIG = yaml_config_helper("dedup", CONFIG)
    CONFIG = yaml_config_helper("collections", CONFIG)
    CONFIG = yaml_config_helper("publish", CONFIG)

else:
    log.debug("Trying to use env variables...")
    misp_url = env_config_helper("MISP_URL")
    misp_api = env_config_helper("MISP_API")
    misp_dedup = env_config_helper("MISP_DEDUP")
    misp_collections = env_config_helper("MISP_COLLECTIONS")
    misp_publish = env_config_helper("MISP_PUBLISH")

    CONFIG = {
                "misp" : {
                            "url" : misp_url,
                            "api" : misp_api,
                            "dedup" : misp_dedup,
                            "collections": misp_collections
                        }
            }
MISP = ''
try:
    MISP = pymisp.PyMISP( 
                        CONFIG["misp"]["url"],
                        CONFIG["misp"]["api"],
                        ssl = CONFIG["misp"].get("verifySSL", True)
                )
except PyMISPError:
    log.error("Cannot connect to MISP; please ensure that MISP is up and running at {}. Skipping MISP upload.".format(CONFIG['misp']['url']))

def post_stix(manager, content_block, collection_ids, service_id):
    '''
        Callback function for when our taxii server gets new data
        Will convert it to a MISPEvent and push to the server
    '''

    # make sure collections, if specified are supposed to be sent to 
    if CONFIG["misp"]["collections"] != "UNKNOWN" or CONFIG["misp"]["collections"] == False:
        log.debug("Using collections")
        should_send_to_misp = False
        collection_names = [collection.name for collection in manager.get_collections(service_id) if collection.id in collection_ids]
        for collection in CONFIG["misp"]["collections"]:
            if collection in collection_names or collection in collection_ids:
                log.debug("Collection specified matches push collection: {}".format(collection))
                should_send_to_misp = True
                break
        if should_send_to_misp == False:
            log.debug('''No collections match misp.collections; aborting MISP extraction.''')
            log.debug("Collection ids whitelisted: {}".format(CONFIG["misp"]["collections"]))
            log.debug("Collection ids sent to: {}".format(collection_ids))
            log.debug('''Collection names sent to: {}'''.format(collection_names))
            return None

    # Load the package
    log.debug("Posting STIX...")
    block = content_block.content
    if isinstance(block, bytes):
        block = block.decode()

    try:
        package = pymisp.tools.stix.load_stix(StringIO(block))
    except Exception:
        log.error('Could not load stix into MISP format; exiting.')
        return 0
    log.debug("STIX loaded succesfully.")
    values = [x.value for x in package.attributes]
    log.debug("Extracted %s", values)
    TOTAL_ATTRIBUTES_ANALYZED = len(values)

    # if deduping is enabled, start deduping
    if (
        CONFIG["misp"]["dedup"] or 
        CONFIG["misp"]["dedup"] == "True" or 
        CONFIG["misp"]["dedup"] == "UNKNOWN"
    ):
        for attrib in values:
            log.debug("Checking for existence of %s", attrib)
            search = ''
            if MISP:
                search = MISP.search("attributes", values=str(attrib))
            else:
                return 0
            if 'response' in search:
                if search["response"]["Attribute"] != []:
                    # This means we have it!
                    log.debug("%s is a duplicate, we'll ignore it.", attrib)
                    package.attributes.pop([x.value for x in package.attributes].index(attrib))
                else:
                    log.debug("%s is unique, we'll keep it", attrib)
            elif 'Attribute' in search:
                if search["Attribute"] != []:
                    # This means we have it!
                    log.debug("%s is a duplicate, we'll ignore it.", attrib)
                    package.attributes.pop([x.value for x in package.attributes].index(attrib))
                else:
                    log.debug("%s is unique, we'll keep it", attrib)
            else:
                log.error("Something went wrong with search, and it doesn't have an 'attribute' or a 'response' key: {}".format(search.keys()))
    else:
        log.debug("Skipping deduplication")

    # Push the event to MISP
    # TODO: There's probably a proper method to do this rather than json_full
    # But I don't wanna read docs
    if (len(package.attributes) > 0):
        log.debug("Uploading event to MISP with attributes %s", [x.value for x in package.attributes])
        event = ''
        try:
            if MISP:
                event = MISP.add_event(package)
                TOTAL_ATTRIBUTES_SENT = len(package.attributes)
        except ConnectionError:
            log.error("Cannot push to MISP; please ensure that MISP is up and running at {}. Skipping MISP upload.".format(CONFIG['misp']['url']))
        if (
            CONFIG["misp"]["publish"] == True or
            CONFIG["misp"]["publish"] == "True"
        ):
            log.info("Publishing event to MISP with ID {}".format(event.get('uuid')))
            if MISP:
                MISP.publish(event)
        else:
            log.debug("Skipping MISP event publishing")
    else:
        log.info("No attributes, not bothering.")

log.debug("total_attributes_analyzed={}, total_attributes_sent={}".format(TOTAL_ATTRIBUTES_ANALYZED, TOTAL_ATTRIBUTES_SENT))
# Make TAXII call our push function whenever it gets new data
CONTENT_BLOCK_CREATED.connect(post_stix)