Merge 29ce499f35 into ec5756bbe1
Davide Baglieri
GitHub
commit
c6805f27a9
22
README.md
22
README.md
|
|
@ -137,6 +137,28 @@ This will run the polling script every 6 hours to keep things all synced up.
|
|||
|
||||
## Troubleshooting
|
||||
|
||||
### Data truncated for column...
|
||||
|
||||
```python
|
||||
Warning: (1265, "Data truncated for column 'original_message' at row 1")
|
||||
|
||||
Warning: (1265, "Data truncated for column 'content' at row 1")
|
||||
```
|
||||
|
||||
If you encounter the error above, this means you tried to push a STIX file bigger than 65,535 bytes. To fix it run the following commands.
|
||||
```bash
|
||||
mysql -u [database user] -p
|
||||
# Enter Database password
|
||||
|
||||
mysql> use taxiipersist;
|
||||
|
||||
mysql> alter table `inbox_messages` modify `original_message` LONGTEXT;
|
||||
|
||||
mysql> alter table `content_blocks` modify `content` LONGTEXT;
|
||||
|
||||
mysql> exit;
|
||||
```
|
||||
|
||||
### Specified key was too long
|
||||
|
||||
```python
|
||||
|
|
|
|||
|
|
@ -8,6 +8,13 @@ import os
|
|||
import pymisp
|
||||
import tempfile
|
||||
import logging
|
||||
import datetime
|
||||
import time
|
||||
import base64
|
||||
import json
|
||||
from pymisp.abstract import MISPEncode
|
||||
from misp_stix_converter.converters import convert
|
||||
from misp_stix_converter.converters import buildMISPAttribute
|
||||
from pyaml import yaml
|
||||
from io import StringIO
|
||||
|
||||
|
|
@ -19,19 +26,19 @@ from opentaxii.signals import (
|
|||
|
||||
## CONFIG
|
||||
if "OPENTAXII_CONFIG" in os.environ:
|
||||
print("Using config from {}".format(os.environ["OPENTAXII_CONFIG"]))
|
||||
log.info("Using config from {}".format(os.environ["OPENTAXII_CONFIG"]))
|
||||
CONFIG = yaml.load(open(os.environ["OPENTAXII_CONFIG"], "r"))
|
||||
else:
|
||||
print("Trying to use env variables...")
|
||||
log.info("Trying to use env variables...")
|
||||
if "MISP_URL" in os.environ:
|
||||
misp_url = os.environ["MISP_URL"]
|
||||
else:
|
||||
print("Unkown misp URL. Set OPENTAXII_CONFIG or MISP_URL.")
|
||||
log.info("Unkown misp URL. Set OPENTAXII_CONFIG or MISP_URL.")
|
||||
misp_url = "UNKNOWN"
|
||||
if "MISP_API" in os.environ:
|
||||
misp_api = os.environ["MISP_API"]
|
||||
else:
|
||||
print("Unknown misp API key. Set OPENTAXII_CONFIG or MISP_API.")
|
||||
log.info("Unknown misp API key. Set OPENTAXII_CONFIG or MISP_API.")
|
||||
misp_api = "UNKNOWN"
|
||||
|
||||
CONFIG = {
|
||||
|
|
@ -41,12 +48,94 @@ else:
|
|||
}
|
||||
}
|
||||
|
||||
MISP = pymisp.PyMISP(
|
||||
MISP = pymisp.PyMISP(
|
||||
CONFIG["misp"]["url"],
|
||||
CONFIG["misp"]["api"],
|
||||
ssl = CONFIG["misp"].get("verifySSL", True)
|
||||
ssl = CONFIG["misp"].get("verifySSL", False)
|
||||
)
|
||||
|
||||
def mytimestamp():
|
||||
return datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d_%H:%M:%S')
|
||||
|
||||
def sanitizer(s):
|
||||
return s.strip(' \t\n\r')
|
||||
|
||||
|
||||
def detect_tlp(package):
|
||||
|
||||
handling = {}
|
||||
|
||||
if hasattr(package, "stix_header"):
|
||||
if hasattr(package.stix_header, "handling"):
|
||||
|
||||
for markingitem in package.stix_header.handling:
|
||||
|
||||
if hasattr(markingitem, "controlled_structure"):
|
||||
|
||||
handling["marking"] = markingitem.controlled_structure
|
||||
|
||||
for itemcolor in markingitem.marking_structures:
|
||||
if hasattr(itemcolor, "color"):
|
||||
handling["color"] = itemcolor.color
|
||||
else:
|
||||
log.info("NO Handling found in this package")
|
||||
else:
|
||||
log.info("No Header found in this package")
|
||||
|
||||
return handling
|
||||
|
||||
|
||||
def detect_source(package):
|
||||
|
||||
source = False
|
||||
|
||||
if hasattr(package, "stix_header"):
|
||||
if hasattr(package.stix_header, "information_source"):
|
||||
if hasattr(package.stix_header.information_source.identity, "name"):
|
||||
source = package.stix_header.information_source.identity.name
|
||||
|
||||
return source
|
||||
|
||||
|
||||
def detect_title(package):
|
||||
'''
|
||||
Return a list
|
||||
- event tile + attachment filename
|
||||
- 1 for detectable event (and possible event update), 0 for undetectable event
|
||||
'''
|
||||
timestamp = mytimestamp()
|
||||
|
||||
if hasattr(package, "stix_header"):
|
||||
if hasattr(package.stix_header, "title"):
|
||||
return package.stix_header.title, 1
|
||||
|
||||
elif hasattr(package.stix_header, "description"):
|
||||
return package.stix_header.description, 1
|
||||
else:
|
||||
return "STIX_FILE_NO_TITLE_" + timestamp, 0
|
||||
else:
|
||||
return "STIX_FILE_NO_TITLE_" + timestamp, 0
|
||||
|
||||
def searchEvent(search):
|
||||
result = MISP.search("events", values=str(sanitizer(search)))
|
||||
|
||||
if result["response"] == []:
|
||||
# New event
|
||||
return 0
|
||||
else:
|
||||
# Maybe an update
|
||||
# Check if a stix file named with `title`.xml exists
|
||||
event = result["response"][0]["Event"]["Attribute"][0]["event_id"]
|
||||
filename = search + ".xml"
|
||||
attachment = MISP.search("events", values=str(filename), type_attribute="attachment", eventid=event)
|
||||
|
||||
if attachment["response"] == []:
|
||||
# New event
|
||||
return 0
|
||||
else:
|
||||
# Event to update
|
||||
return event
|
||||
|
||||
def post_stix(manager, content_block, collection_ids, service_id):
|
||||
'''
|
||||
Callback function for when our taxii server gets new data
|
||||
|
|
@ -58,29 +147,116 @@ def post_stix(manager, content_block, collection_ids, service_id):
|
|||
block = content_block.content
|
||||
if isinstance(block, bytes):
|
||||
block = block.decode()
|
||||
|
||||
package = pymisp.tools.stix.load_stix(StringIO(block))
|
||||
log.info("STIX loaded succesfully.")
|
||||
values = [x.value for x in package.attributes]
|
||||
log.info("Extracted %s", values)
|
||||
for attrib in values:
|
||||
log.info("Checking for existence of %s", attrib)
|
||||
search = MISP.search("attributes", values=str(attrib))
|
||||
if search["response"] != []:
|
||||
# This means we have it!
|
||||
log.info("%s is a duplicate, we'll ignore it.", attrib)
|
||||
package.attributes.pop([x.value for x in package.attributes].index(attrib))
|
||||
else:
|
||||
log.info("%s is unique, we'll keep it", attrib)
|
||||
|
||||
# Push the event to MISP
|
||||
# TODO: There's probably a proper method to do this rather than json_full
|
||||
# But I don't wanna read docs
|
||||
if (len(package.attributes) > 0):
|
||||
log.info("Uploading event to MISP with attributes %s", [x.value for x in package.attributes])
|
||||
MISP.add_event(package)
|
||||
package = convert.load_stix(StringIO(block))
|
||||
# Building event obj
|
||||
distribution=3
|
||||
threat_level_id=2
|
||||
analysis=0
|
||||
|
||||
misp_event = buildMISPAttribute.buildEvent(package, distribution=distribution, threat_level_id=threat_level_id, analysis=analysis)
|
||||
log.info("STIX loaded succesfully. Let's go!")
|
||||
|
||||
evaluatePackage = detect_title(package)
|
||||
tlp = detect_tlp(package)
|
||||
source = detect_source(package)
|
||||
|
||||
title = evaluatePackage[0]
|
||||
detectable = evaluatePackage[1]
|
||||
|
||||
if detectable == 1:
|
||||
|
||||
search = searchEvent(title)
|
||||
|
||||
if search == 0 :
|
||||
# New Event!
|
||||
b64Pkg = base64.b64encode(package.to_xml()).decode("utf-8")
|
||||
|
||||
if misp_event.attributes:
|
||||
filename = title + ".xml"
|
||||
misp_event.add_attribute(type="attachment", value=filename, data=b64Pkg)
|
||||
|
||||
if tlp:
|
||||
misp_event.add_tag("tlp:"+tlp['color'])
|
||||
misp_event.add_tag("Marking_Controlled_Structure:"+tlp['marking'])
|
||||
|
||||
if source:
|
||||
misp_event.add_tag("source:"+source)
|
||||
|
||||
response = MISP.add_event(json.dumps(misp_event, cls=MISPEncode))
|
||||
|
||||
if response.get('errors'):
|
||||
raise Exception("PACKAGE: {}\nERROR: {}".format(json.dumps(misp_event, cls=MISPEncode),response.get('errors')))
|
||||
|
||||
else:
|
||||
MISP.fast_publish(response["Event"]["id"])
|
||||
else:
|
||||
log.info("No attributes detected")
|
||||
else:
|
||||
myeventid = search
|
||||
|
||||
# Just the library default!
|
||||
# Edit if you need
|
||||
distribution=3
|
||||
threat_level_id=2
|
||||
analysis=0
|
||||
buildattribute = buildMISPAttribute.buildEvent(package, distribution=distribution, threat_level_id=threat_level_id, analysis=analysis)
|
||||
|
||||
items = [x for x in buildattribute.attributes]
|
||||
|
||||
for attrib in items:
|
||||
|
||||
searchatt = MISP.search("attributes", values=str(sanitizer(attrib.value)), type_attribute=str(attrib.type), eventid=myeventid)
|
||||
|
||||
if searchatt["response"] != []:
|
||||
log.info("%s is a duplicate, we'll ignore it.", attrib.value)
|
||||
buildattribute.attributes.pop([x.value for x in buildattribute.attributes].index(attrib.value))
|
||||
else:
|
||||
log.info("%s is unique, we'll keep it", attrib.value)
|
||||
|
||||
if (len(buildattribute.attributes) > 0 ):
|
||||
b64Pkg = base64.b64encode(package.to_xml()).decode("utf-8")
|
||||
updatetimestamp = mytimestamp()
|
||||
filename = title + "_" + updatetimestamp + ".xml"
|
||||
misp_event.add_attribute(type="attachment", value=filename, data=b64Pkg)
|
||||
|
||||
if tlp:
|
||||
misp_event.add_tag("tlp:"+tlp['color'])
|
||||
misp_event.add_tag("Marking_Controlled_Structure:"+tlp['marking'])
|
||||
|
||||
if source:
|
||||
misp_event.add_tag("source:"+source)
|
||||
|
||||
MISP.update_event(myeventid,json.dumps(misp_event, cls=MISPEncode))
|
||||
MISP.fast_publish(myeventid)
|
||||
log.info("Updating event: " +myeventid)
|
||||
else:
|
||||
log.info("Nothing to update for event: "+ myeventid)
|
||||
else:
|
||||
log.info("No attributes, not bothering.")
|
||||
log.info("Event undetectable. Will be used old style import!")
|
||||
values = [x.value for x in misp_event.attributes]
|
||||
log.info("Extracted %s", values)
|
||||
|
||||
for attrib in values:
|
||||
log.info("Checking for existence of %s", attrib)
|
||||
search = MISP.search("attributes", values=str(sanitizer(attrib)))
|
||||
|
||||
if search["response"] != []:
|
||||
# This means we have it!
|
||||
log.info("%s is a duplicate, we'll ignore it.", attrib)
|
||||
misp_event.attributes.pop([x.value for x in misp_event.attributes].index(attrib))
|
||||
else:
|
||||
log.info("%s is unique, we'll keep it", attrib)
|
||||
|
||||
# Push the event to MISP
|
||||
# TODO: There's probably a proper method to do this rather than json_full
|
||||
# But I don't wanna read docs
|
||||
if (len(misp_event.attributes) > 0):
|
||||
log.info("Uploading event to MISP with attributes %s", [x.value for x in misp_event.attributes])
|
||||
MISP.add_event(misp_event)
|
||||
else:
|
||||
log.info("No attributes, not bothering.")
|
||||
|
||||
# Make TAXII call our push function whenever it gets new data
|
||||
CONTENT_BLOCK_CREATED.connect(post_stix)
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue