MISP
/
cti-python-stix2
mirror of https://github.com/MISP/cti-python-stix2
2
0
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #52 from oasis-open/environment 

Environment layer
stix2.1
Greg Back 2017-09-12 13:59:08 +00:00 committed by GitHub
parent f070a4358e 190b639126
commit 463d1e6b28
6 changed files with 272 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
stix2/__init__.py
View File

@ -7,7 +7,7 @@ from .common import (TLP_AMBER, TLP_GREEN, TLP_RED, TLP_WHITE, CustomMarking,
ExternalReference, GranularMarking, KillChainPhase,
MarkingDefinition, StatementMarking, TLPMarking)
from .core import Bundle, _register_type, parse
from .environment import ObjectFactory
from .environment import Environment, ObjectFactory
from .observables import (URL, AlternateDataStream, ArchiveExt, Artifact,
AutonomousSystem, CustomObservable, Directory,
DomainName, EmailAddress, EmailMessage,
@ -42,6 +42,13 @@ from .patterns import (AndBooleanExpression, AndObservationExpression,
from .sdo import (AttackPattern, Campaign, CourseOfAction, CustomObject,
Identity, Indicator, IntrusionSet, Malware, ObservedData,
Report, ThreatActor, Tool, Vulnerability)
from .sources import CompositeDataSource
from .sources.filesystem import (FileSystemSink, FileSystemSource,
FileSystemStore)
from .sources.filters import Filter
from .sources.memory import MemorySink, MemorySource, MemoryStore
from .sources.taxii import (TAXIICollectionSink, TAXIICollectionSource,
TAXIICollectionStore)
from .sro import Relationship, Sighting
from .utils import get_dict, new_version, revoke
from .version import __version__

6
stix2/core.py
View File

@ -75,13 +75,13 @@ def parse(data, allow_custom=False):
"""Deserialize a string or file-like object into a STIX object.
Args:
data: The STIX 2 string to be parsed.
data (str, dict, file-like object): The STIX 2 content to be parsed.
allow_custom (bool): Whether to allow custom properties or not. Default: False.
Returns:
An instantiated Python STIX object.
"""
"""
obj = get_dict(data)
if 'type' not in obj:
@ -96,6 +96,6 @@ def parse(data, allow_custom=False):
def _register_type(new_type):
"""Register a custom STIX Object type.
"""
"""
OBJ_MAP[new_type._type] = new_type

106
stix2/environment.py
View File

@ -1,22 +1,22 @@
import copy
from .core import parse as _parse
from .sources import CompositeDataSource, DataSource, DataStore
class ObjectFactory(object):
"""Object Factory
Used to easily create STIX objects with default values for certain
properties.
"""Easily create STIX objects with default values for certain properties.
Args:
created_by_ref: Default created_by_ref value to apply to all
created_by_ref (optional): Default created_by_ref value to apply to all
objects created by this factory.
created: Default created value to apply to all
created (optional): Default created value to apply to all
objects created by this factory.
external_references: Default `external_references` value to apply
external_references (optional): Default `external_references` value to apply
to all objects created by this factory.
object_marking_refs: Default `object_marking_refs` value to apply
object_marking_refs (optional): Default `object_marking_refs` value to apply
to all objects created by this factory.
list_append: When a default is set for a list property like
list_append (bool, optional): When a default is set for a list property like
`external_references` or `object_marking_refs` and a value for
that property is passed into `create()`, if this is set to True,
that value will be added to the list alongside the default. If
@ -44,6 +44,13 @@ class ObjectFactory(object):
self._list_properties = ['external_references', 'object_marking_refs']
def create(self, cls, **kwargs):
"""Create a STIX object using object factory defaults.
Args:
cls: the python-stix2 class of the object to be created (eg. Indicator)
**kwargs: The property/value pairs of the STIX object to be created
"""
# Use self.defaults as the base, but update with any explicit args
# provided by the user.
properties = copy.deepcopy(self._defaults)
@ -66,3 +73,84 @@ class ObjectFactory(object):
properties.update(**kwargs)
return cls(**properties)
class Environment(object):
"""
Args:
factory (ObjectFactory, optional): Factory for creating objects with common
defaults for certain properties.
store (DataStore, optional): Data store providing the source and sink for the
environment.
source (DataSource, optional): Source for retrieving STIX objects.
sink (DataSink, optional): Destination for saving STIX objects.
Invalid if `store` is also provided.
"""
def __init__(self, factory=ObjectFactory(), store=None, source=None, sink=None):
self.factory = factory
self.source = CompositeDataSource()
if store:
self.source.add_data_source(store.source)
self.sink = store.sink
if source:
self.source.add_data_source(source)
if sink:
if store:
raise ValueError("Data store already provided! Environment may only have one data sink.")
self.sink = sink
def create(self, *args, **kwargs):
return self.factory.create(*args, **kwargs)
create.__doc__ = ObjectFactory.create.__doc__
def get(self, *args, **kwargs):
try:
return self.source.get(*args, **kwargs)
except AttributeError:
raise AttributeError('Environment has no data source to query')
get.__doc__ = DataStore.get.__doc__
def all_versions(self, *args, **kwargs):
"""Retrieve all versions of a single STIX object by ID.
"""
try:
return self.source.all_versions(*args, **kwargs)
except AttributeError:
raise AttributeError('Environment has no data source to query')
all_versions.__doc__ = DataStore.all_versions.__doc__
def query(self, *args, **kwargs):
"""Retrieve STIX objects matching a set of filters.
"""
try:
return self.source.query(*args, **kwargs)
except AttributeError:
raise AttributeError('Environment has no data source to query')
query.__doc__ = DataStore.query.__doc__
def add_filters(self, *args, **kwargs):
try:
return self.source.add_filters(*args, **kwargs)
except AttributeError:
raise AttributeError('Environment has no data source')
add_filters.__doc__ = DataSource.add_filters.__doc__
def add_filter(self, *args, **kwargs):
try:
return self.source.add_filter(*args, **kwargs)
except AttributeError:
raise AttributeError('Environment has no data source')
add_filter.__doc__ = DataSource.add_filter.__doc__
def add(self, *args, **kwargs):
try:
return self.sink.add(*args, **kwargs)
except AttributeError:
raise AttributeError('Environment has no data sink to put objects in')
add.__doc__ = DataStore.add.__doc__
def parse(self, *args, **kwargs):
return _parse(*args, **kwargs)
parse.__doc__ = _parse.__doc__

73
stix2/sources/__init__.py
View File

@ -5,7 +5,7 @@ Classes:
DataStore
DataSink
DataSource
STIXCommonPropertyFilters
CompositeDataSource
TODO:Test everything
@ -45,30 +45,29 @@ class DataStore(object):
self.sink = sink
def get(self, stix_id):
"""
"""Retrieve the most recent version of a single STIX object by ID.
Notes:
Translate API get() call to the appropriate DataSource call.
Args:
stix_id (str): the id of the STIX 2.0 object to retrieve. Should
return a single object, the most recent version of the object
specified by the "id".
stix_id (str): the id of the STIX 2.0 object to retrieve.
Returns:
stix_obj (dictionary): the STIX object to be returned
stix_obj (dictionary): the single most recent version of the STIX
object specified by the "id".
"""
return self.source.get(stix_id)
def all_versions(self, stix_id):
"""
"""Retrieve all versions of a single STIX object by ID.
Implement:
Translate all_versions() call to the appropriate DataSource call
Args:
stix_id (str): the id of the STIX 2.0 object to retrieve. Should
return a single object, the most recent version of the object
specified by the "id".
stix_id (str): the id of the STIX 2.0 object to retrieve.
Returns:
stix_objs (list): a list of STIX objects (where each object is a
@ -78,7 +77,8 @@ class DataStore(object):
return self.source.all_versions(stix_id)
def query(self, query):
"""
"""Retrieve STIX objects matching a set of filters.
Notes:
Implement the specific data source API calls, processing,
functionality required for retrieving query from the data source.
@ -95,10 +95,15 @@ class DataStore(object):
return self.source.query(query=query)
def add(self, stix_objs):
"""
"""Store STIX objects.
Notes:
Translate add() to the appropriate DataSink call().
Args:
stix_objs (list): a list of STIX objects (where each object is a
STIX object)
"""
return self.sink.add(stix_objs)
@ -116,11 +121,16 @@ class DataSink(object):
self.id = make_id()
def add(self, stix_objs):
"""
"""Store STIX objects.
Notes:
Implement the specific data sink API calls, processing,
functionality required for adding data to the sink
Args:
stix_objs (list): a list of STIX objects (where each object is a
STIX object)
"""
raise NotImplementedError()
@ -201,16 +211,22 @@ class DataSource(object):
raise NotImplementedError()
def add_filters(self, filters):
"""Add multiple filters to the DataSource.
"""Add multiple filters to be applied to all queries for STIX objects.
Args:
filters (list): list of filters (dict) to add to the Data Source.
"""
for filter in filters:
self.add_filter(filter)
def add_filter(self, filter):
"""Add a filter."""
"""Add a filter to be applied to all queries for STIX objects.
Args:
filter: filter to add to the Data Source.
"""
# check filter field is a supported STIX 2.0 common field
if filter.field not in STIX_COMMON_FIELDS:
raise ValueError("Filter 'field' is not a STIX 2.0 common property. Currently only STIX object common properties supported")
@ -226,7 +242,7 @@ class DataSource(object):
self.filters.add(filter)
def apply_common_filters(self, stix_objs, query):
"""Evaluates filters against a set of STIX 2.0 objects
"""Evaluate filters against a set of STIX 2.0 objects.
Supports only STIX 2.0 common property fields
@ -300,11 +316,10 @@ class DataSource(object):
class CompositeDataSource(DataSource):
"""Composite Data Source
"""Controller for all the defined/configured STIX Data Sources.
Acts as a controller for all the defined/configured STIX Data Sources
e.g. a user can define n Data Sources - creating Data Source (objects)
for each. There is only one instance of this for any python STIX 2.0
E.g. a user can define n Data Sources - creating Data Source (objects)
for each. There is only one instance of this for any Python STIX 2.0
application.
Attributes:
@ -314,8 +329,7 @@ class CompositeDataSource(DataSource):
"""
def __init__(self):
"""
Creates a new STIX Data Source.
"""Create a new STIX Data Source.
Args:
name (str): A string containing the name to attach in the
@ -348,6 +362,9 @@ class CompositeDataSource(DataSource):
stix_obj (dict): the STIX object to be returned.
"""
if not self.get_all_data_sources():
raise AttributeError('CompositeDataSource has no data sources')
all_data = []
# for every configured Data Source, call its retrieve handler
@ -384,6 +401,9 @@ class CompositeDataSource(DataSource):
all_data (list): list of STIX objects that have the specified id
"""
if not self.get_all_data_sources():
raise AttributeError('CompositeDataSource has no data sources')
all_data = []
all_filters = self.filters
@ -403,9 +423,7 @@ class CompositeDataSource(DataSource):
return all_data
def query(self, query=None, _composite_filters=None):
"""Composite data source query
Federate the query to all Data Sources attached to the
"""Federate the query to all Data Sources attached to the
Composite Data Source.
Args:
@ -418,6 +436,9 @@ class CompositeDataSource(DataSource):
all_data (list): list of STIX objects to be returned
"""
if not self.get_all_data_sources():
raise AttributeError('CompositeDataSource has no data sources')
if not query:
query = []
@ -448,6 +469,8 @@ class CompositeDataSource(DataSource):
to the Composite Data Source
"""
if not isinstance(data_sources, list):
data_sources = [data_sources]
for ds in data_sources:
if issubclass(ds.__class__, DataSource):
if ds.id in self.data_sources:

34
stix2/sources/memory.py
View File

@ -22,33 +22,22 @@ import collections
import json
import os
from stix2validator import validate_instance
from stix2 import Bundle
from stix2.sources import DataSink, DataSource, DataStore
from stix2.sources.filters import Filter
def _add(store, stix_data):
def _add(store, stix_data=None):
"""Adds stix objects to MemoryStore/Source/Sink."""
if isinstance(stix_data, collections.Mapping):
# stix objects are in a bundle
# verify STIX json data
r = validate_instance(stix_data)
# make dictionary of the objects for easy lookup
if r.is_valid:
for stix_obj in stix_data["objects"]:
store.data[stix_obj["id"]] = stix_obj
else:
raise ValueError("Error: data passed was found to not be valid by the STIX 2 Validator: \n%s", r.as_dict())
for stix_obj in stix_data["objects"]:
store.data[stix_obj["id"]] = stix_obj
elif isinstance(stix_data, list):
# stix objects are in a list
for stix_obj in stix_data:
r = validate_instance(stix_obj)
if r.is_valid:
store.data[stix_obj["id"]] = stix_obj
else:
raise ValueError("Error: STIX object %s is not valid under STIX 2 validator.\n%s", stix_obj["id"], r)
store.data[stix_obj["id"]] = stix_obj
else:
raise ValueError("stix_data must be in bundle format or raw list")
@ -56,7 +45,7 @@ def _add(store, stix_data):
class MemoryStore(DataStore):
"""
"""
def __init__(self, stix_data):
def __init__(self, stix_data=None):
"""
Notes:
It doesn't make sense to create a MemoryStore by passing
@ -83,7 +72,7 @@ class MemoryStore(DataStore):
class MemorySink(DataSink):
"""
"""
def __init__(self, stix_data, _store=False):
def __init__(self, stix_data=None, _store=False):
"""
Args:
stix_data (dictionary OR list): valid STIX 2.0 content in
@ -114,7 +103,7 @@ class MemorySink(DataSink):
class MemorySource(DataSource):
def __init__(self, stix_data, _store=False):
def __init__(self, stix_data=None, _store=False):
"""
Args:
stix_data (dictionary OR list): valid STIX 2.0 content in
@ -193,10 +182,5 @@ class MemorySource(DataSource):
file_path = os.path.abspath(file_path)
stix_data = json.load(open(file_path, "r"))
r = validate_instance(stix_data)
if r.is_valid:
for stix_obj in stix_data["objects"]:
self.data[stix_obj["id"]] = stix_obj
raise ValueError("Error: STIX data loaded from file (%s) was found to not be validated by STIX 2 Validator.\n%s", file_path, r)
for stix_obj in stix_data["objects"]:
self.data[stix_obj["id"]] = stix_obj

109
stix2/test/test_environment.py
View File

@ -1,7 +1,9 @@
import pytest
import stix2
from .constants import (FAKE_TIME, IDENTITY_ID, IDENTITY_KWARGS,
INDICATOR_KWARGS)
from .constants import (FAKE_TIME, IDENTITY_ID, IDENTITY_KWARGS, INDICATOR_ID,
INDICATOR_KWARGS, MALWARE_ID)
def test_object_factory_created_by_ref_str():
@ -81,3 +83,106 @@ def test_object_factory_list_replace():
ind = factory.create(stix2.Indicator, external_references=ext_ref2, **INDICATOR_KWARGS)
assert len(ind.external_references) == 1
assert ind.external_references[0].source_name == "Yet Another Threat Report"
def test_environment_functions():
env = stix2.Environment(stix2.ObjectFactory(created_by_ref=IDENTITY_ID),
stix2.MemoryStore())
# Create a STIX object
ind = env.create(stix2.Indicator, id=INDICATOR_ID, **INDICATOR_KWARGS)
assert ind.created_by_ref == IDENTITY_ID
# Add objects to datastore
ind2 = ind.new_version(labels=['benign'])
env.add([ind, ind2])
# Get both versions of the object
resp = env.all_versions(INDICATOR_ID)
assert len(resp) == 1 # should be 2, but MemoryStore only keeps 1 version of objects
# Get just the most recent version of the object
resp = env.get(INDICATOR_ID)
assert resp['labels'][0] == 'benign'
# Search on something other than id
query = [stix2.Filter('type', '=', 'vulnerability')]
resp = env.query(query)
assert len(resp) == 0
# See different results after adding filters to the environment
env.add_filters([stix2.Filter('type', '=', 'indicator'),
stix2.Filter('created_by_ref', '=', IDENTITY_ID)])
env.add_filter(stix2.Filter('labels', '=', 'benign')) # should be 'malicious-activity'
resp = env.get(INDICATOR_ID)
assert resp['labels'][0] == 'benign' # should be 'malicious-activity'
def test_environment_source_and_sink():
ind = stix2.Indicator(id=INDICATOR_ID, **INDICATOR_KWARGS)
env = stix2.Environment(source=stix2.MemorySource([ind]), sink=stix2.MemorySink([ind]))
assert env.get(INDICATOR_ID).labels[0] == 'malicious-activity'
def test_environment_datastore_and_sink():
with pytest.raises(ValueError) as excinfo:
stix2.Environment(factory=stix2.ObjectFactory(),
store=stix2.MemoryStore(), sink=stix2.MemorySink)
assert 'Data store already provided' in str(excinfo.value)
def test_environment_no_datastore():
env = stix2.Environment(factory=stix2.ObjectFactory())
with pytest.raises(AttributeError) as excinfo:
env.add(stix2.Indicator(**INDICATOR_KWARGS))
assert 'Environment has no data sink to put objects in' in str(excinfo.value)
with pytest.raises(AttributeError) as excinfo:
env.get(INDICATOR_ID)
assert 'Environment has no data source' in str(excinfo.value)
with pytest.raises(AttributeError) as excinfo:
env.all_versions(INDICATOR_ID)
assert 'Environment has no data source' in str(excinfo.value)
with pytest.raises(AttributeError) as excinfo:
env.query(INDICATOR_ID)
assert 'Environment has no data source' in str(excinfo.value)
with pytest.raises(AttributeError) as excinfo:
env.add_filters(INDICATOR_ID)
assert 'Environment has no data source' in str(excinfo.value)
with pytest.raises(AttributeError) as excinfo:
env.add_filter(INDICATOR_ID)
assert 'Environment has no data source' in str(excinfo.value)
def test_environment_datastore_and_no_object_factory():
# Uses a default object factory
env = stix2.Environment(store=stix2.MemoryStore())
ind = env.create(stix2.Indicator, id=INDICATOR_ID, **INDICATOR_KWARGS)
assert ind.id == INDICATOR_ID
def test_parse_malware():
env = stix2.Environment()
data = """{
"type": "malware",
"id": "malware--fedcba98-7654-3210-fedc-ba9876543210",
"created": "2017-01-01T12:34:56.000Z",
"modified": "2017-01-01T12:34:56.000Z",
"name": "Cryptolocker",
"labels": [
"ransomware"
]
}"""
mal = env.parse(data)
assert mal.type == 'malware'
assert mal.id == MALWARE_ID
assert mal.created == FAKE_TIME
assert mal.modified == FAKE_TIME
assert mal.labels == ['ransomware']
assert mal.name == "Cryptolocker"