Merge pull request #82 from arcsector/master

Bringing up-to-date with recent PyMISP and Optional Deduplication
master
Alexandre Dulaunoy 2022-06-14 09:44:27 +02:00 committed by GitHub
commit 28dcfb6f39
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 177 additions and 36 deletions

2
.gitignore vendored
View File

@ -6,3 +6,5 @@ config.yaml
__pycache__ __pycache__
build build
dist dist
src
vscode/

View File

@ -29,6 +29,11 @@ zmq:
misp: misp:
url: "http://localhost" url: "http://localhost"
api: KEY api: KEY
dedup: true
collections:
- my_collection
- my_collection2
publish: false
taxii: taxii:
auth: auth:

View File

@ -0,0 +1,12 @@
host: localhost
port: 9000
discovery_path: /services/discovery
inbox_path: /services/inbox
use_https: False
taxii_version: '1.1'
headers:
auth:
username: test
password: test
collections:
- collection

View File

@ -0,0 +1,18 @@
- name: 'default'
host: localhost
port: 9000
discovery_path:
use_https: False
taxii_version: '1.1'
headers:
auth:
username:
password:
cacert_path:
cert_file:
key_file:
key_password:
jwt_auth_url:
verify_ssl: True
collections:
- collection

View File

@ -9,43 +9,84 @@ import pymisp
import tempfile import tempfile
import logging import logging
from pyaml import yaml from pyaml import yaml
from yaml import Loader
from io import StringIO from io import StringIO
from requests.exceptions import ConnectionError
from pymisp.exceptions import PyMISPError
import warnings
warnings.filterwarnings("ignore")
log = logging.getLogger("__main__") TOTAL_ATTRIBUTES_SENT = 0
TOTAL_ATTRIBUTES_ANALYZED = 0
logging_level = logging.INFO
log = logging.getLogger("misp_taxii_server")
log.setLevel(logging_level)
handler = logging.StreamHandler()
handler.setLevel(logging_level)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)
if log.handlers:
log.handlers = []
log.addHandler(handler)
from opentaxii.signals import ( from opentaxii.signals import (
CONTENT_BLOCK_CREATED, INBOX_MESSAGE_CREATED CONTENT_BLOCK_CREATED, INBOX_MESSAGE_CREATED
) )
def env_config_helper(env_name):
if env_name in os.environ:
if env_name == "MISP_COLLECTIONS":
name = os.environ[env_name]
return name.split(',')
return os.environ[env_name]
else:
log.error("Missing env setting {0}. Set OPENTAXII_CONFIG or {0}.".format(env_name))
return "UNKNOWN"
def yaml_config_helper(config_name, CONFIG):
if config_name in CONFIG["misp"]:
if not CONFIG["misp"][config_name] and CONFIG["misp"][config_name] != False:
CONFIG["misp"][config_name] = "UNKNOWN"
else:
CONFIG["misp"][config_name] = "UNKNOWN"
return CONFIG
## CONFIG ## CONFIG
if "OPENTAXII_CONFIG" in os.environ: if "OPENTAXII_CONFIG" in os.environ:
print("Using config from {}".format(os.environ["OPENTAXII_CONFIG"])) log.debug("Using config from {}".format(os.environ["OPENTAXII_CONFIG"]))
CONFIG = yaml.load(open(os.environ["OPENTAXII_CONFIG"], "r")) CONFIG = yaml.load(open(os.environ["OPENTAXII_CONFIG"], "r"), Loader=Loader)
# validate dedup and collections and publish
CONFIG = yaml_config_helper("dedup", CONFIG)
CONFIG = yaml_config_helper("collections", CONFIG)
CONFIG = yaml_config_helper("publish", CONFIG)
else: else:
print("Trying to use env variables...") log.debug("Trying to use env variables...")
if "MISP_URL" in os.environ: misp_url = env_config_helper("MISP_URL")
misp_url = os.environ["MISP_URL"] misp_api = env_config_helper("MISP_API")
else: misp_dedup = env_config_helper("MISP_DEDUP")
print("Unkown misp URL. Set OPENTAXII_CONFIG or MISP_URL.") misp_collections = env_config_helper("MISP_COLLECTIONS")
misp_url = "UNKNOWN" misp_publish = env_config_helper("MISP_PUBLISH")
if "MISP_API" in os.environ:
misp_api = os.environ["MISP_API"]
else:
print("Unknown misp API key. Set OPENTAXII_CONFIG or MISP_API.")
misp_api = "UNKNOWN"
CONFIG = { CONFIG = {
"misp" : { "misp" : {
"url" : misp_url, "url" : misp_url,
"api" : misp_api "api" : misp_api,
"dedup" : misp_dedup,
"collections": misp_collections
} }
} }
MISP = ''
try:
MISP = pymisp.PyMISP( MISP = pymisp.PyMISP(
CONFIG["misp"]["url"], CONFIG["misp"]["url"],
CONFIG["misp"]["api"], CONFIG["misp"]["api"],
ssl = CONFIG["misp"].get("verifySSL", True) ssl = CONFIG["misp"].get("verifySSL", True)
) )
except PyMISPError:
log.error("Cannot connect to MISP; please ensure that MISP is up and running at {}. Skipping MISP upload.".format(CONFIG['misp']['url']))
def post_stix(manager, content_block, collection_ids, service_id): def post_stix(manager, content_block, collection_ids, service_id):
''' '''
@ -53,34 +94,95 @@ def post_stix(manager, content_block, collection_ids, service_id):
Will convert it to a MISPEvent and push to the server Will convert it to a MISPEvent and push to the server
''' '''
# make sure collections, if specified are supposed to be sent to
if CONFIG["misp"]["collections"] != "UNKNOWN" or CONFIG["misp"]["collections"] == False:
log.debug("Using collections")
should_send_to_misp = False
collection_names = [collection.name for collection in manager.get_collections(service_id) if collection.id in collection_ids]
for collection in CONFIG["misp"]["collections"]:
if collection in collection_names or collection in collection_ids:
log.debug("Collection specified matches push collection: {}".format(collection))
should_send_to_misp = True
break
if should_send_to_misp == False:
log.debug('''No collections match misp.collections; aborting MISP extraction.''')
log.debug("Collection ids whitelisted: {}".format(CONFIG["misp"]["collections"]))
log.debug("Collection ids sent to: {}".format(collection_ids))
log.debug('''Collection names sent to: {}'''.format(collection_names))
return None
# Load the package # Load the package
log.info("Posting STIX...") log.debug("Posting STIX...")
block = content_block.content block = content_block.content
if isinstance(block, bytes): if isinstance(block, bytes):
block = block.decode() block = block.decode()
try:
package = pymisp.tools.stix.load_stix(StringIO(block)) package = pymisp.tools.stix.load_stix(StringIO(block))
log.info("STIX loaded succesfully.") except Exception:
log.error('Could not load stix into MISP format; exiting.')
return 0
log.debug("STIX loaded succesfully.")
values = [x.value for x in package.attributes] values = [x.value for x in package.attributes]
log.info("Extracted %s", values) log.debug("Extracted %s", values)
TOTAL_ATTRIBUTES_ANALYZED = len(values)
# if deduping is enabled, start deduping
if (
CONFIG["misp"]["dedup"] or
CONFIG["misp"]["dedup"] == "True" or
CONFIG["misp"]["dedup"] == "UNKNOWN"
):
for attrib in values: for attrib in values:
log.info("Checking for existence of %s", attrib) log.debug("Checking for existence of %s", attrib)
search = ''
if MISP:
search = MISP.search("attributes", values=str(attrib)) search = MISP.search("attributes", values=str(attrib))
else:
return 0
if 'response' in search:
if search["response"]["Attribute"] != []: if search["response"]["Attribute"] != []:
# This means we have it! # This means we have it!
log.info("%s is a duplicate, we'll ignore it.", attrib) log.debug("%s is a duplicate, we'll ignore it.", attrib)
package.attributes.pop([x.value for x in package.attributes].index(attrib)) package.attributes.pop([x.value for x in package.attributes].index(attrib))
else: else:
log.info("%s is unique, we'll keep it", attrib) log.debug("%s is unique, we'll keep it", attrib)
elif 'Attribute' in search:
if search["Attribute"] != []:
# This means we have it!
log.debug("%s is a duplicate, we'll ignore it.", attrib)
package.attributes.pop([x.value for x in package.attributes].index(attrib))
else:
log.debug("%s is unique, we'll keep it", attrib)
else:
log.error("Something went wrong with search, and it doesn't have an 'attribute' or a 'response' key: {}".format(search.keys()))
else:
log.debug("Skipping deduplication")
# Push the event to MISP # Push the event to MISP
# TODO: There's probably a proper method to do this rather than json_full # TODO: There's probably a proper method to do this rather than json_full
# But I don't wanna read docs # But I don't wanna read docs
if (len(package.attributes) > 0): if (len(package.attributes) > 0):
log.info("Uploading event to MISP with attributes %s", [x.value for x in package.attributes]) log.debug("Uploading event to MISP with attributes %s", [x.value for x in package.attributes])
MISP.add_event(package) event = ''
try:
if MISP:
event = MISP.add_event(package)
TOTAL_ATTRIBUTES_SENT = len(package.attributes)
except ConnectionError:
log.error("Cannot push to MISP; please ensure that MISP is up and running at {}. Skipping MISP upload.".format(CONFIG['misp']['url']))
if (
CONFIG["misp"]["publish"] == True or
CONFIG["misp"]["publish"] == "True"
):
log.info("Publishing event to MISP with ID {}".format(event.get('uuid')))
if MISP:
MISP.publish(event)
else:
log.debug("Skipping MISP event publishing")
else: else:
log.info("No attributes, not bothering.") log.info("No attributes, not bothering.")
log.debug("total_attributes_analyzed={}, total_attributes_sent={}".format(TOTAL_ATTRIBUTES_ANALYZED, TOTAL_ATTRIBUTES_SENT))
# Make TAXII call our push function whenever it gets new data # Make TAXII call our push function whenever it gets new data
CONTENT_BLOCK_CREATED.connect(post_stix) CONTENT_BLOCK_CREATED.connect(post_stix)

View File

@ -4,6 +4,7 @@ import sys
import json import json
import pymisp import pymisp
from pyaml import yaml from pyaml import yaml
from yaml import Loader
from cabby import create_client from cabby import create_client
from misp_stix_converter.converters import lint_roller from misp_stix_converter.converters import lint_roller
import logging import logging
@ -20,7 +21,7 @@ log.setLevel(logging.DEBUG)
log.info("Starting...") log.info("Starting...")
# Try to load in config # Try to load in config
if "OPENTAXII_CONFIG" in os.environ: if "OPENTAXII_CONFIG" in os.environ:
config = yaml.load(open(os.environ["OPENTAXII_CONFIG"], "r")) config = yaml.load(open(os.environ["OPENTAXII_CONFIG"], "r"), Loader=Loader)
else: else:
print("OPENTAXII CONFIG NOT EXPORTED") print("OPENTAXII CONFIG NOT EXPORTED")
sys.exit() sys.exit()

View File

@ -2,6 +2,7 @@
from cabby import create_client from cabby import create_client
from pyaml import yaml from pyaml import yaml
from yaml import Loader
import pytz import pytz
import argparse import argparse
import os import os
@ -51,14 +52,14 @@ config_file = "{}/remote-servers.yml".format(
log.debug("Opening config file %s", config_file) log.debug("Opening config file %s", config_file)
with open(config_file, "r") as f: with open(config_file, "r") as f:
config = yaml.load(f.read()) config = yaml.load(f.read(), Loader=Loader)
log.debug("Config read %s", config) log.debug("Config read %s", config)
# Read in the local server configuration # Read in the local server configuration
local_config = "{}/local-server.yml".format(os.path.expanduser(args.configdir)) local_config = "{}/local-server.yml".format(os.path.expanduser(args.configdir))
log.debug("Reading local server config") log.debug("Reading local server config")
with open(local_config, "r") as f: with open(local_config, "r") as f:
local_config = yaml.load(f.read()) local_config = yaml.load(f.read(), Loader=Loader)
# Attempt to make contact with the local server # Attempt to make contact with the local server
log.info("Connecting to local server...") log.info("Connecting to local server...")