From 29ce499f350a9ead3c2952aa34964a567dfcd3bb Mon Sep 17 00:00:00 2001 From: Davide Baglieri Date: Mon, 26 Feb 2018 19:05:28 +0100 Subject: [PATCH] Hook improved Improved version of the misp taxii hook --- misp_taxii_hooks/hooks.py | 262 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 262 insertions(+) create mode 100644 misp_taxii_hooks/hooks.py diff --git a/misp_taxii_hooks/hooks.py b/misp_taxii_hooks/hooks.py new file mode 100644 index 0000000..719c714 --- /dev/null +++ b/misp_taxii_hooks/hooks.py @@ -0,0 +1,262 @@ +#!/usr/bin/env python3 + +###### +# TODO: DETECT DUPLICATE DATA +##### + +import os +import pymisp +import tempfile +import logging +import datetime +import time +import base64 +import json +from pymisp.abstract import MISPEncode +from misp_stix_converter.converters import convert +from misp_stix_converter.converters import buildMISPAttribute +from pyaml import yaml +from io import StringIO + +log = logging.getLogger("__main__") + +from opentaxii.signals import ( + CONTENT_BLOCK_CREATED, INBOX_MESSAGE_CREATED +) + +## CONFIG +if "OPENTAXII_CONFIG" in os.environ: + log.info("Using config from {}".format(os.environ["OPENTAXII_CONFIG"])) + CONFIG = yaml.load(open(os.environ["OPENTAXII_CONFIG"], "r")) +else: + log.info("Trying to use env variables...") + if "MISP_URL" in os.environ: + misp_url = os.environ["MISP_URL"] + else: + log.info("Unkown misp URL. Set OPENTAXII_CONFIG or MISP_URL.") + misp_url = "UNKNOWN" + if "MISP_API" in os.environ: + misp_api = os.environ["MISP_API"] + else: + log.info("Unknown misp API key. Set OPENTAXII_CONFIG or MISP_API.") + misp_api = "UNKNOWN" + + CONFIG = { + "misp" : { + "url" : misp_url, + "api" : misp_api + } + } + +MISP = pymisp.PyMISP( + CONFIG["misp"]["url"], + CONFIG["misp"]["api"], + ssl = CONFIG["misp"].get("verifySSL", False) + ) + +def mytimestamp(): + return datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d_%H:%M:%S') + +def sanitizer(s): + return s.strip(' \t\n\r') + + +def detect_tlp(package): + + handling = {} + + if hasattr(package, "stix_header"): + if hasattr(package.stix_header, "handling"): + + for markingitem in package.stix_header.handling: + + if hasattr(markingitem, "controlled_structure"): + + handling["marking"] = markingitem.controlled_structure + + for itemcolor in markingitem.marking_structures: + if hasattr(itemcolor, "color"): + handling["color"] = itemcolor.color + else: + log.info("NO Handling found in this package") + else: + log.info("No Header found in this package") + + return handling + + +def detect_source(package): + + source = False + + if hasattr(package, "stix_header"): + if hasattr(package.stix_header, "information_source"): + if hasattr(package.stix_header.information_source.identity, "name"): + source = package.stix_header.information_source.identity.name + + return source + + +def detect_title(package): + ''' + Return a list + - event tile + attachment filename + - 1 for detectable event (and possible event update), 0 for undetectable event + ''' + timestamp = mytimestamp() + + if hasattr(package, "stix_header"): + if hasattr(package.stix_header, "title"): + return package.stix_header.title, 1 + + elif hasattr(package.stix_header, "description"): + return package.stix_header.description, 1 + else: + return "STIX_FILE_NO_TITLE_" + timestamp, 0 + else: + return "STIX_FILE_NO_TITLE_" + timestamp, 0 + +def searchEvent(search): + result = MISP.search("events", values=str(sanitizer(search))) + + if result["response"] == []: + # New event + return 0 + else: + # Maybe an update + # Check if a stix file named with `title`.xml exists + event = result["response"][0]["Event"]["Attribute"][0]["event_id"] + filename = search + ".xml" + attachment = MISP.search("events", values=str(filename), type_attribute="attachment", eventid=event) + + if attachment["response"] == []: + # New event + return 0 + else: + # Event to update + return event + +def post_stix(manager, content_block, collection_ids, service_id): + ''' + Callback function for when our taxii server gets new data + Will convert it to a MISPEvent and push to the server + ''' + + # Load the package + log.info("Posting STIX...") + block = content_block.content + if isinstance(block, bytes): + block = block.decode() + + package = convert.load_stix(StringIO(block)) + # Building event obj + distribution=3 + threat_level_id=2 + analysis=0 + + misp_event = buildMISPAttribute.buildEvent(package, distribution=distribution, threat_level_id=threat_level_id, analysis=analysis) + log.info("STIX loaded succesfully. Let's go!") + + evaluatePackage = detect_title(package) + tlp = detect_tlp(package) + source = detect_source(package) + + title = evaluatePackage[0] + detectable = evaluatePackage[1] + + if detectable == 1: + + search = searchEvent(title) + + if search == 0 : + # New Event! + b64Pkg = base64.b64encode(package.to_xml()).decode("utf-8") + + if misp_event.attributes: + filename = title + ".xml" + misp_event.add_attribute(type="attachment", value=filename, data=b64Pkg) + + if tlp: + misp_event.add_tag("tlp:"+tlp['color']) + misp_event.add_tag("Marking_Controlled_Structure:"+tlp['marking']) + + if source: + misp_event.add_tag("source:"+source) + + response = MISP.add_event(json.dumps(misp_event, cls=MISPEncode)) + + if response.get('errors'): + raise Exception("PACKAGE: {}\nERROR: {}".format(json.dumps(misp_event, cls=MISPEncode),response.get('errors'))) + + else: + MISP.fast_publish(response["Event"]["id"]) + else: + log.info("No attributes detected") + else: + myeventid = search + + # Just the library default! + # Edit if you need + distribution=3 + threat_level_id=2 + analysis=0 + buildattribute = buildMISPAttribute.buildEvent(package, distribution=distribution, threat_level_id=threat_level_id, analysis=analysis) + + items = [x for x in buildattribute.attributes] + + for attrib in items: + + searchatt = MISP.search("attributes", values=str(sanitizer(attrib.value)), type_attribute=str(attrib.type), eventid=myeventid) + + if searchatt["response"] != []: + log.info("%s is a duplicate, we'll ignore it.", attrib.value) + buildattribute.attributes.pop([x.value for x in buildattribute.attributes].index(attrib.value)) + else: + log.info("%s is unique, we'll keep it", attrib.value) + + if (len(buildattribute.attributes) > 0 ): + b64Pkg = base64.b64encode(package.to_xml()).decode("utf-8") + updatetimestamp = mytimestamp() + filename = title + "_" + updatetimestamp + ".xml" + misp_event.add_attribute(type="attachment", value=filename, data=b64Pkg) + + if tlp: + misp_event.add_tag("tlp:"+tlp['color']) + misp_event.add_tag("Marking_Controlled_Structure:"+tlp['marking']) + + if source: + misp_event.add_tag("source:"+source) + + MISP.update_event(myeventid,json.dumps(misp_event, cls=MISPEncode)) + MISP.fast_publish(myeventid) + log.info("Updating event: " +myeventid) + else: + log.info("Nothing to update for event: "+ myeventid) + else: + log.info("Event undetectable. Will be used old style import!") + values = [x.value for x in misp_event.attributes] + log.info("Extracted %s", values) + + for attrib in values: + log.info("Checking for existence of %s", attrib) + search = MISP.search("attributes", values=str(sanitizer(attrib))) + + if search["response"] != []: + # This means we have it! + log.info("%s is a duplicate, we'll ignore it.", attrib) + misp_event.attributes.pop([x.value for x in misp_event.attributes].index(attrib)) + else: + log.info("%s is unique, we'll keep it", attrib) + + # Push the event to MISP + # TODO: There's probably a proper method to do this rather than json_full + # But I don't wanna read docs + if (len(misp_event.attributes) > 0): + log.info("Uploading event to MISP with attributes %s", [x.value for x in misp_event.attributes]) + MISP.add_event(misp_event) + else: + log.info("No attributes, not bothering.") + +# Make TAXII call our push function whenever it gets new data +CONTENT_BLOCK_CREATED.connect(post_stix) +