2016-11-18 10:28:00 +01:00
#!/usr/bin/env python3
2016-11-18 12:54:58 +01:00
######
# TODO: DETECT DUPLICATE DATA
#####
2016-11-23 13:16:30 +01:00
import os
2016-11-18 10:28:00 +01:00
import pymisp
2016-11-18 12:54:58 +01:00
import tempfile
2017-06-29 15:37:54 +02:00
import logging
2016-11-23 13:16:30 +01:00
from pyaml import yaml
2020-05-07 02:00:04 +02:00
from yaml import Loader
2017-10-19 17:31:06 +02:00
from io import StringIO
2020-06-16 03:08:33 +02:00
from requests . exceptions import ConnectionError
2016-11-18 10:28:00 +01:00
2020-06-16 03:08:33 +02:00
logging_level = logging . INFO
2017-06-29 15:37:54 +02:00
log = logging . getLogger ( " __main__ " )
2020-06-16 03:12:24 +02:00
handler = logging . StreamHandler ( )
2020-06-16 03:08:33 +02:00
log . setLevel ( logging_level )
handler . setLevel ( logging_level )
formatter = logging . Formatter ( ' %(asctime)s - %(name)s - %(levelname)s - %(message)s ' )
handler . setFormatter ( formatter )
2020-06-16 03:12:24 +02:00
log . addHandler ( handler )
2020-06-16 03:08:33 +02:00
2017-06-29 15:37:54 +02:00
2016-11-18 10:28:00 +01:00
from opentaxii . signals import (
CONTENT_BLOCK_CREATED , INBOX_MESSAGE_CREATED
)
2020-05-14 04:21:52 +02:00
def env_config_helper ( env_name ) :
if env_name in os . environ :
if env_name == " MISP_COLLECTIONS " :
name = os . environ [ env_name ]
return name . split ( ' , ' )
return os . environ [ env_name ]
else :
2020-06-16 03:08:33 +02:00
log . error ( " Missing env setting {0} . Set OPENTAXII_CONFIG or {0} . " . format ( env_name ) )
2020-05-14 04:21:52 +02:00
return " UNKNOWN "
def yaml_config_helper ( config_name , CONFIG ) :
if config_name in CONFIG [ " misp " ] :
2020-05-15 01:26:51 +02:00
if not CONFIG [ " misp " ] [ config_name ] and CONFIG [ " misp " ] [ config_name ] != False :
2020-05-14 04:21:52 +02:00
CONFIG [ " misp " ] [ config_name ] = " UNKNOWN "
else :
CONFIG [ " misp " ] [ config_name ] = " UNKNOWN "
return CONFIG
2016-11-18 10:28:00 +01:00
## CONFIG
2016-11-23 16:31:56 +01:00
if " OPENTAXII_CONFIG " in os . environ :
2020-06-16 03:08:33 +02:00
log . info ( " Using config from {} " . format ( os . environ [ " OPENTAXII_CONFIG " ] ) )
2020-05-07 02:00:04 +02:00
CONFIG = yaml . load ( open ( os . environ [ " OPENTAXII_CONFIG " ] , " r " ) , Loader = Loader )
2020-05-14 04:21:52 +02:00
# validate dedup and collections and publish
CONFIG = yaml_config_helper ( " dedup " , CONFIG )
CONFIG = yaml_config_helper ( " collections " , CONFIG )
CONFIG = yaml_config_helper ( " publish " , CONFIG )
2016-11-23 13:16:30 +01:00
else :
2020-06-16 03:08:33 +02:00
log . debug ( " Trying to use env variables... " )
2020-05-14 04:21:52 +02:00
misp_url = env_config_helper ( " MISP_URL " )
misp_api = env_config_helper ( " MISP_API " )
misp_dedup = env_config_helper ( " MISP_DEDUP " )
misp_collections = env_config_helper ( " MISP_COLLECTIONS " )
misp_publish = env_config_helper ( " MISP_PUBLISH " )
2016-11-18 10:28:00 +01:00
2016-11-23 13:16:30 +01:00
CONFIG = {
2016-11-23 16:31:56 +01:00
" misp " : {
" url " : misp_url ,
2020-05-13 22:41:12 +02:00
" api " : misp_api ,
2020-05-14 04:21:52 +02:00
" dedup " : misp_dedup ,
" collections " : misp_collections
2016-11-23 16:31:56 +01:00
}
2016-11-23 13:16:30 +01:00
}
2016-11-18 12:54:58 +01:00
MISP = pymisp . PyMISP (
2016-11-23 16:31:56 +01:00
CONFIG [ " misp " ] [ " url " ] ,
CONFIG [ " misp " ] [ " api " ] ,
2017-05-22 17:45:06 +02:00
ssl = CONFIG [ " misp " ] . get ( " verifySSL " , True )
2016-11-18 12:54:58 +01:00
)
2016-11-18 12:45:30 +01:00
def post_stix ( manager , content_block , collection_ids , service_id ) :
2016-11-18 12:54:58 +01:00
'''
Callback function for when our taxii server gets new data
Will convert it to a MISPEvent and push to the server
'''
2020-05-14 04:21:52 +02:00
# make sure collections, if specified are supposed to be sent to
if CONFIG [ " misp " ] [ " collections " ] != " UNKNOWN " or CONFIG [ " misp " ] [ " collections " ] == False :
2020-05-15 01:18:53 +02:00
log . info ( " Using collections " )
2020-05-14 04:21:52 +02:00
should_send_to_misp = False
2020-05-15 01:38:28 +02:00
collection_names = [ collection . name for collection in manager . get_collections ( service_id ) if collection . id in collection_ids ]
2020-05-14 04:21:52 +02:00
for collection in CONFIG [ " misp " ] [ " collections " ] :
2020-05-15 01:15:23 +02:00
if collection in collection_names or collection in collection_ids :
2020-05-15 01:18:53 +02:00
log . info ( " Collection specified matches push collection: {} " . format ( collection ) )
2020-05-14 04:21:52 +02:00
should_send_to_misp = True
2020-05-15 01:18:53 +02:00
break
2020-05-14 04:21:52 +02:00
if should_send_to_misp == False :
log . info ( ''' No collections match misp.collections; aborting MISP extraction.
Collection ids whitelisted : { }
2020-05-15 01:15:23 +02:00
Collection ids sent to : { }
Collection names sent to : { } ''' .format(
2020-05-14 04:21:52 +02:00
CONFIG [ " misp " ] [ " collections " ] ,
2020-05-15 01:15:23 +02:00
collection_ids ,
collection_names
2020-05-14 04:21:52 +02:00
) )
return None
2016-11-18 12:54:58 +01:00
# Load the package
2017-06-29 15:37:54 +02:00
log . info ( " Posting STIX... " )
2017-11-03 12:47:27 +01:00
block = content_block . content
if isinstance ( block , bytes ) :
block = block . decode ( )
package = pymisp . tools . stix . load_stix ( StringIO ( block ) )
2017-06-29 15:37:54 +02:00
log . info ( " STIX loaded succesfully. " )
2017-06-14 14:52:29 +02:00
values = [ x . value for x in package . attributes ]
2017-06-29 15:37:54 +02:00
log . info ( " Extracted %s " , values )
2020-05-14 04:21:52 +02:00
# if deduping is enabled, start deduping
2020-05-13 22:49:34 +02:00
if (
2020-05-14 04:21:52 +02:00
CONFIG [ " misp " ] [ " dedup " ] or
2020-05-13 22:49:34 +02:00
CONFIG [ " misp " ] [ " dedup " ] == " True " or
CONFIG [ " misp " ] [ " dedup " ] == " UNKNOWN "
) :
2020-05-13 22:41:12 +02:00
for attrib in values :
log . info ( " Checking for existence of %s " , attrib )
search = MISP . search ( " attributes " , values = str ( attrib ) )
if ' response ' in search :
if search [ " response " ] [ " Attribute " ] != [ ] :
# This means we have it!
log . info ( " %s is a duplicate, we ' ll ignore it. " , attrib )
package . attributes . pop ( [ x . value for x in package . attributes ] . index ( attrib ) )
else :
log . info ( " %s is unique, we ' ll keep it " , attrib )
elif ' Attribute ' in search :
if search [ " Attribute " ] != [ ] :
# This means we have it!
log . info ( " %s is a duplicate, we ' ll ignore it. " , attrib )
package . attributes . pop ( [ x . value for x in package . attributes ] . index ( attrib ) )
else :
log . info ( " %s is unique, we ' ll keep it " , attrib )
2020-05-07 06:05:36 +02:00
else :
2020-05-13 22:41:12 +02:00
log . error ( " Something went wrong with search, and it doesn ' t have an ' attribute ' or a ' response ' key: {} " . format ( search . keys ( ) ) )
2020-05-15 01:33:15 +02:00
else :
log . info ( " Skipping deduplication " )
2017-06-29 15:37:54 +02:00
2016-11-18 12:54:58 +01:00
# Push the event to MISP
# TODO: There's probably a proper method to do this rather than json_full
# But I don't wanna read docs
2016-11-18 15:42:53 +01:00
if ( len ( package . attributes ) > 0 ) :
2017-06-29 15:37:54 +02:00
log . info ( " Uploading event to MISP with attributes %s " , [ x . value for x in package . attributes ] )
2020-06-16 03:08:33 +02:00
try :
event = MISP . add_event ( package )
except ConnectionError :
log . error ( " MISP-Taxii-Server - Cannot connect to MISP; please ensure that MISP is up and running at {} . Skipping MISP upload. " . format ( CONFIG [ ' misp ' ] [ ' url ' ] ) )
2020-05-14 04:21:52 +02:00
if (
2020-05-15 01:33:15 +02:00
CONFIG [ " misp " ] [ " publish " ] == True or
2020-05-14 04:21:52 +02:00
CONFIG [ " misp " ] [ " publish " ] == " True "
) :
2020-05-15 01:33:15 +02:00
log . info ( " Publishing event to MISP with ID {} " . format ( event . get ( ' uuid ' ) ) )
2020-05-14 04:21:52 +02:00
MISP . publish ( event )
2020-05-15 01:43:01 +02:00
else :
log . info ( " Skipping MISP event publishing " )
2017-06-29 15:37:54 +02:00
else :
log . info ( " No attributes, not bothering. " )
2016-11-18 10:28:00 +01:00
2016-11-18 12:54:58 +01:00
# Make TAXII call our push function whenever it gets new data
2016-11-18 10:28:00 +01:00
CONTENT_BLOCK_CREATED . connect ( post_stix )