MISP
/
MISP-Taxii-Server
mirror of https://github.com/MISP/MISP-Taxii-Server
2
0
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #82 from arcsector/master 

Bringing up-to-date with recent PyMISP and Optional Deduplication
master
Alexandre Dulaunoy 2022-06-14 09:44:27 +02:00 committed by GitHub
parent 2300977908 2aa3522cae
commit 28dcfb6f39
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 177 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -6,3 +6,5 @@ config.yaml
__pycache__
build
dist
src
vscode/

5
config/config.default.yaml
View File

@ -29,6 +29,11 @@ zmq:
misp:
url: "http://localhost"
api: KEY
dedup: true
collections:
- my_collection
- my_collection2
publish: false
taxii:
auth:

12
config/local-server.default.yaml Normal file
View File

@ -0,0 +1,12 @@
host: localhost
port: 9000
discovery_path: /services/discovery
inbox_path: /services/inbox
use_https: False
taxii_version: '1.1'
headers:
auth:
username: test
password: test
collections:
- collection

18
config/remote-servers.default.yaml Normal file
View File

@ -0,0 +1,18 @@
- name: 'default'
host: localhost
port: 9000
discovery_path:
use_https: False
taxii_version: '1.1'
headers:
auth:
username:
password:
cacert_path:
cert_file:
key_file:
key_password:
jwt_auth_url:
verify_ssl: True
collections:
- collection

166
misp_taxii_hooks/hooks.py
View File

@ -9,43 +9,84 @@ import pymisp
import tempfile
import logging
from pyaml import yaml
from yaml import Loader
from io import StringIO
from requests.exceptions import ConnectionError
from pymisp.exceptions import PyMISPError
import warnings
warnings.filterwarnings("ignore")
log = logging.getLogger("__main__")
TOTAL_ATTRIBUTES_SENT = 0
TOTAL_ATTRIBUTES_ANALYZED = 0
logging_level = logging.INFO
log = logging.getLogger("misp_taxii_server")
log.setLevel(logging_level)
handler = logging.StreamHandler()
handler.setLevel(logging_level)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)
if log.handlers:
log.handlers = []
log.addHandler(handler)
from opentaxii.signals import (
CONTENT_BLOCK_CREATED, INBOX_MESSAGE_CREATED
)
def env_config_helper(env_name):
if env_name in os.environ:
if env_name == "MISP_COLLECTIONS":
name = os.environ[env_name]
return name.split(',')
return os.environ[env_name]
else:
log.error("Missing env setting {0}. Set OPENTAXII_CONFIG or {0}.".format(env_name))
return "UNKNOWN"
def yaml_config_helper(config_name, CONFIG):
if config_name in CONFIG["misp"]:
if not CONFIG["misp"][config_name] and CONFIG["misp"][config_name] != False:
CONFIG["misp"][config_name] = "UNKNOWN"
else:
CONFIG["misp"][config_name] = "UNKNOWN"
return CONFIG
## CONFIG
if "OPENTAXII_CONFIG" in os.environ:
print("Using config from {}".format(os.environ["OPENTAXII_CONFIG"]))
CONFIG = yaml.load(open(os.environ["OPENTAXII_CONFIG"], "r"))
log.debug("Using config from {}".format(os.environ["OPENTAXII_CONFIG"]))
CONFIG = yaml.load(open(os.environ["OPENTAXII_CONFIG"], "r"), Loader=Loader)
# validate dedup and collections and publish
CONFIG = yaml_config_helper("dedup", CONFIG)
CONFIG = yaml_config_helper("collections", CONFIG)
CONFIG = yaml_config_helper("publish", CONFIG)
else:
print("Trying to use env variables...")
if "MISP_URL" in os.environ:
misp_url = os.environ["MISP_URL"]
else:
print("Unkown misp URL. Set OPENTAXII_CONFIG or MISP_URL.")
misp_url = "UNKNOWN"
if "MISP_API" in os.environ:
misp_api = os.environ["MISP_API"]
else:
print("Unknown misp API key. Set OPENTAXII_CONFIG or MISP_API.")
misp_api = "UNKNOWN"
log.debug("Trying to use env variables...")
misp_url = env_config_helper("MISP_URL")
misp_api = env_config_helper("MISP_API")
misp_dedup = env_config_helper("MISP_DEDUP")
misp_collections = env_config_helper("MISP_COLLECTIONS")
misp_publish = env_config_helper("MISP_PUBLISH")
CONFIG = {
"misp" : {
"url" : misp_url,
"api" : misp_api
"api" : misp_api,
"dedup" : misp_dedup,
"collections": misp_collections
}
}
MISP = pymisp.PyMISP(
MISP = ''
try:
MISP = pymisp.PyMISP(
CONFIG["misp"]["url"],
CONFIG["misp"]["api"],
ssl = CONFIG["misp"].get("verifySSL", True)
)
except PyMISPError:
log.error("Cannot connect to MISP; please ensure that MISP is up and running at {}. Skipping MISP upload.".format(CONFIG['misp']['url']))
def post_stix(manager, content_block, collection_ids, service_id):
'''
@ -53,34 +94,95 @@ def post_stix(manager, content_block, collection_ids, service_id):
Will convert it to a MISPEvent and push to the server
'''
# make sure collections, if specified are supposed to be sent to
if CONFIG["misp"]["collections"] != "UNKNOWN" or CONFIG["misp"]["collections"] == False:
log.debug("Using collections")
should_send_to_misp = False
collection_names = [collection.name for collection in manager.get_collections(service_id) if collection.id in collection_ids]
for collection in CONFIG["misp"]["collections"]:
if collection in collection_names or collection in collection_ids:
log.debug("Collection specified matches push collection: {}".format(collection))
should_send_to_misp = True
break
if should_send_to_misp == False:
log.debug('''No collections match misp.collections; aborting MISP extraction.''')
log.debug("Collection ids whitelisted: {}".format(CONFIG["misp"]["collections"]))
log.debug("Collection ids sent to: {}".format(collection_ids))
log.debug('''Collection names sent to: {}'''.format(collection_names))
return None
# Load the package
log.info("Posting STIX...")
log.debug("Posting STIX...")
block = content_block.content
if isinstance(block, bytes):
block = block.decode()
package = pymisp.tools.stix.load_stix(StringIO(block))
log.info("STIX loaded succesfully.")
try:
package = pymisp.tools.stix.load_stix(StringIO(block))
except Exception:
log.error('Could not load stix into MISP format; exiting.')
return 0
log.debug("STIX loaded succesfully.")
values = [x.value for x in package.attributes]
log.info("Extracted %s", values)
for attrib in values:
log.info("Checking for existence of %s", attrib)
search = MISP.search("attributes", values=str(attrib))
if search["response"]["Attribute"] != []:
# This means we have it!
log.info("%s is a duplicate, we'll ignore it.", attrib)
package.attributes.pop([x.value for x in package.attributes].index(attrib))
else:
log.info("%s is unique, we'll keep it", attrib)
log.debug("Extracted %s", values)
TOTAL_ATTRIBUTES_ANALYZED = len(values)
# if deduping is enabled, start deduping
if (
CONFIG["misp"]["dedup"] or
CONFIG["misp"]["dedup"] == "True" or
CONFIG["misp"]["dedup"] == "UNKNOWN"
):
for attrib in values:
log.debug("Checking for existence of %s", attrib)
search = ''
if MISP:
search = MISP.search("attributes", values=str(attrib))
else:
return 0
if 'response' in search:
if search["response"]["Attribute"] != []:
# This means we have it!
log.debug("%s is a duplicate, we'll ignore it.", attrib)
package.attributes.pop([x.value for x in package.attributes].index(attrib))
else:
log.debug("%s is unique, we'll keep it", attrib)
elif 'Attribute' in search:
if search["Attribute"] != []:
# This means we have it!
log.debug("%s is a duplicate, we'll ignore it.", attrib)
package.attributes.pop([x.value for x in package.attributes].index(attrib))
else:
log.debug("%s is unique, we'll keep it", attrib)
else:
log.error("Something went wrong with search, and it doesn't have an 'attribute' or a 'response' key: {}".format(search.keys()))
else:
log.debug("Skipping deduplication")
# Push the event to MISP
# TODO: There's probably a proper method to do this rather than json_full
# But I don't wanna read docs
if (len(package.attributes) > 0):
log.info("Uploading event to MISP with attributes %s", [x.value for x in package.attributes])
MISP.add_event(package)
log.debug("Uploading event to MISP with attributes %s", [x.value for x in package.attributes])
event = ''
try:
if MISP:
event = MISP.add_event(package)
TOTAL_ATTRIBUTES_SENT = len(package.attributes)
except ConnectionError:
log.error("Cannot push to MISP; please ensure that MISP is up and running at {}. Skipping MISP upload.".format(CONFIG['misp']['url']))
if (
CONFIG["misp"]["publish"] == True or
CONFIG["misp"]["publish"] == "True"
):
log.info("Publishing event to MISP with ID {}".format(event.get('uuid')))
if MISP:
MISP.publish(event)
else:
log.debug("Skipping MISP event publishing")
else:
log.info("No attributes, not bothering.")
log.debug("total_attributes_analyzed={}, total_attributes_sent={}".format(TOTAL_ATTRIBUTES_ANALYZED, TOTAL_ATTRIBUTES_SENT))
# Make TAXII call our push function whenever it gets new data
CONTENT_BLOCK_CREATED.connect(post_stix)

3
scripts/push_published_to_taxii.py
View File

@ -4,6 +4,7 @@ import sys
import json
import pymisp
from pyaml import yaml
from yaml import Loader
from cabby import create_client
from misp_stix_converter.converters import lint_roller
import logging
@ -20,7 +21,7 @@ log.setLevel(logging.DEBUG)
log.info("Starting...")
# Try to load in config
if "OPENTAXII_CONFIG" in os.environ:
config = yaml.load(open(os.environ["OPENTAXII_CONFIG"], "r"))
config = yaml.load(open(os.environ["OPENTAXII_CONFIG"], "r"), Loader=Loader)
else:
print("OPENTAXII CONFIG NOT EXPORTED")
sys.exit()

5
scripts/run-taxii-poll.py
View File

@ -2,6 +2,7 @@
from cabby import create_client
from pyaml import yaml
from yaml import Loader
import pytz
import argparse
import os
@ -51,14 +52,14 @@ config_file = "{}/remote-servers.yml".format(
log.debug("Opening config file %s", config_file)
with open(config_file, "r") as f:
config = yaml.load(f.read())
config = yaml.load(f.read(), Loader=Loader)
log.debug("Config read %s", config)
# Read in the local server configuration
local_config = "{}/local-server.yml".format(os.path.expanduser(args.configdir))
log.debug("Reading local server config")
with open(local_config, "r") as f:
local_config = yaml.load(f.read())
local_config = yaml.load(f.read(), Loader=Loader)
# Attempt to make contact with the local server
log.info("Connecting to local server...")