2020-09-11 19:54:13 +02:00
|
|
|
"""Python APIs for STIX 2 Pattern Semantic Equivalence.
|
|
|
|
|
|
|
|
.. autosummary::
|
2020-10-16 23:12:52 +02:00
|
|
|
:toctree: pattern
|
2020-09-11 19:54:13 +02:00
|
|
|
|
|
|
|
compare
|
|
|
|
transform
|
|
|
|
|
|
|
|
|
|
|
|
|
"""
|
|
|
|
|
2020-08-15 01:55:00 +02:00
|
|
|
import stix2
|
Graph Equivalence (#449)
* new packages for graph and object-based semantic equivalence
* new method graphically_equivalent for Environment, move equivalence methods out
* object equivalence function, methods used for object-based moved here.
* new graph_equivalence methods
* add notes
* add support for versioning checks (default disabled)
* new tests to cover graph equivalence and new methods
* added more imports to environment.py to prevent breaking changes
* variable changes, new fields for checks, reset depth check per call
* flexibility when object is not available on graph.
* refactor debug logging message
* new file stix2.equivalence.graph_equivalence.rst and stix2.equivalence.object_equivalence.rst for docs
* API documentation for new modules
* additional text required to build docs
* add more test methods for list_semantic_check an graphically_equivalent/versioning
* add logging debug messages, code clean-up
* include individual scoring on results dict, fix issue on list_semantic_check not keeping highest score
* include results as summary in prop_scores, minor tweaks
* Update __init__.py
doctrings update
* apply feedback from pull request
- rename semantic_check to reference_check
- rename modules to graph and object respectively to eliminate redundancy
- remove created_by_ref and object_marking_refs from graph WEIGHTS and rebalance
* update docs/ entries
* add more checks, make max score based on actual objects checked instead of the full list, only create entry when type is present in WEIGHTS dictionary
update tests to reflect changes
* rename package patterns -> pattern
* documentation, moving weights around
* more documentation moving
* rename WEIGHTS variable for graph_equivalence
2020-10-16 17:35:26 +02:00
|
|
|
from stix2.equivalence.pattern.compare.observation import (
|
2020-08-13 23:44:42 +02:00
|
|
|
observation_expression_cmp,
|
|
|
|
)
|
Graph Equivalence (#449)
* new packages for graph and object-based semantic equivalence
* new method graphically_equivalent for Environment, move equivalence methods out
* object equivalence function, methods used for object-based moved here.
* new graph_equivalence methods
* add notes
* add support for versioning checks (default disabled)
* new tests to cover graph equivalence and new methods
* added more imports to environment.py to prevent breaking changes
* variable changes, new fields for checks, reset depth check per call
* flexibility when object is not available on graph.
* refactor debug logging message
* new file stix2.equivalence.graph_equivalence.rst and stix2.equivalence.object_equivalence.rst for docs
* API documentation for new modules
* additional text required to build docs
* add more test methods for list_semantic_check an graphically_equivalent/versioning
* add logging debug messages, code clean-up
* include individual scoring on results dict, fix issue on list_semantic_check not keeping highest score
* include results as summary in prop_scores, minor tweaks
* Update __init__.py
doctrings update
* apply feedback from pull request
- rename semantic_check to reference_check
- rename modules to graph and object respectively to eliminate redundancy
- remove created_by_ref and object_marking_refs from graph WEIGHTS and rebalance
* update docs/ entries
* add more checks, make max score based on actual objects checked instead of the full list, only create entry when type is present in WEIGHTS dictionary
update tests to reflect changes
* rename package patterns -> pattern
* documentation, moving weights around
* more documentation moving
* rename WEIGHTS variable for graph_equivalence
2020-10-16 17:35:26 +02:00
|
|
|
from stix2.equivalence.pattern.transform import (
|
2020-08-13 23:44:42 +02:00
|
|
|
ChainTransformer, SettleTransformer,
|
2020-08-11 00:33:26 +02:00
|
|
|
)
|
Graph Equivalence (#449)
* new packages for graph and object-based semantic equivalence
* new method graphically_equivalent for Environment, move equivalence methods out
* object equivalence function, methods used for object-based moved here.
* new graph_equivalence methods
* add notes
* add support for versioning checks (default disabled)
* new tests to cover graph equivalence and new methods
* added more imports to environment.py to prevent breaking changes
* variable changes, new fields for checks, reset depth check per call
* flexibility when object is not available on graph.
* refactor debug logging message
* new file stix2.equivalence.graph_equivalence.rst and stix2.equivalence.object_equivalence.rst for docs
* API documentation for new modules
* additional text required to build docs
* add more test methods for list_semantic_check an graphically_equivalent/versioning
* add logging debug messages, code clean-up
* include individual scoring on results dict, fix issue on list_semantic_check not keeping highest score
* include results as summary in prop_scores, minor tweaks
* Update __init__.py
doctrings update
* apply feedback from pull request
- rename semantic_check to reference_check
- rename modules to graph and object respectively to eliminate redundancy
- remove created_by_ref and object_marking_refs from graph WEIGHTS and rebalance
* update docs/ entries
* add more checks, make max score based on actual objects checked instead of the full list, only create entry when type is present in WEIGHTS dictionary
update tests to reflect changes
* rename package patterns -> pattern
* documentation, moving weights around
* more documentation moving
* rename WEIGHTS variable for graph_equivalence
2020-10-16 17:35:26 +02:00
|
|
|
from stix2.equivalence.pattern.transform.observation import (
|
2020-08-13 23:44:42 +02:00
|
|
|
AbsorptionTransformer, CanonicalizeComparisonExpressionsTransformer,
|
|
|
|
DNFTransformer, FlattenTransformer, OrderDedupeTransformer,
|
2020-08-11 00:33:26 +02:00
|
|
|
)
|
2020-08-13 23:44:42 +02:00
|
|
|
import stix2.pattern_visitor
|
2020-08-11 00:33:26 +02:00
|
|
|
|
|
|
|
# Lazy-initialize
|
|
|
|
_pattern_canonicalizer = None
|
|
|
|
|
|
|
|
|
|
|
|
def _get_pattern_canonicalizer():
|
|
|
|
"""
|
|
|
|
Get a canonicalization transformer for STIX patterns.
|
|
|
|
|
|
|
|
:return: The transformer
|
|
|
|
"""
|
|
|
|
|
|
|
|
# The transformers are either stateless or contain no state which changes
|
|
|
|
# with each use. So we can setup the transformers once and keep reusing
|
|
|
|
# them.
|
|
|
|
global _pattern_canonicalizer
|
|
|
|
|
|
|
|
if not _pattern_canonicalizer:
|
|
|
|
canonicalize_comp_expr = \
|
|
|
|
CanonicalizeComparisonExpressionsTransformer()
|
|
|
|
|
|
|
|
obs_expr_flatten = FlattenTransformer()
|
|
|
|
obs_expr_order = OrderDedupeTransformer()
|
|
|
|
obs_expr_absorb = AbsorptionTransformer()
|
|
|
|
obs_simplify = ChainTransformer(
|
2020-08-13 23:44:42 +02:00
|
|
|
obs_expr_flatten, obs_expr_order, obs_expr_absorb,
|
2020-08-11 00:33:26 +02:00
|
|
|
)
|
|
|
|
obs_settle_simplify = SettleTransformer(obs_simplify)
|
|
|
|
|
|
|
|
obs_dnf = DNFTransformer()
|
|
|
|
|
|
|
|
_pattern_canonicalizer = ChainTransformer(
|
|
|
|
canonicalize_comp_expr,
|
2020-08-13 23:44:42 +02:00
|
|
|
obs_settle_simplify, obs_dnf, obs_settle_simplify,
|
2020-08-11 00:33:26 +02:00
|
|
|
)
|
|
|
|
|
|
|
|
return _pattern_canonicalizer
|
|
|
|
|
|
|
|
|
2020-08-15 01:55:00 +02:00
|
|
|
def equivalent_patterns(pattern1, pattern2, stix_version=stix2.DEFAULT_VERSION):
|
2020-08-11 00:33:26 +02:00
|
|
|
"""
|
|
|
|
Determine whether two STIX patterns are semantically equivalent.
|
|
|
|
|
|
|
|
:param pattern1: The first STIX pattern
|
|
|
|
:param pattern2: The second STIX pattern
|
2020-08-15 01:55:00 +02:00
|
|
|
:param stix_version: The STIX version to use for pattern parsing, as a
|
|
|
|
string ("2.0", "2.1", etc). Defaults to library-wide default version.
|
2020-08-11 00:33:26 +02:00
|
|
|
:return: True if the patterns are semantically equivalent; False if not
|
|
|
|
"""
|
2020-08-15 01:55:00 +02:00
|
|
|
patt_ast1 = stix2.pattern_visitor.create_pattern_object(
|
|
|
|
pattern1, version=stix_version,
|
|
|
|
)
|
|
|
|
patt_ast2 = stix2.pattern_visitor.create_pattern_object(
|
|
|
|
pattern2, version=stix_version,
|
|
|
|
)
|
2020-08-11 00:33:26 +02:00
|
|
|
|
|
|
|
pattern_canonicalizer = _get_pattern_canonicalizer()
|
|
|
|
canon_patt1, _ = pattern_canonicalizer.transform(patt_ast1)
|
|
|
|
canon_patt2, _ = pattern_canonicalizer.transform(patt_ast2)
|
|
|
|
|
|
|
|
result = observation_expression_cmp(canon_patt1, canon_patt2)
|
|
|
|
|
|
|
|
return result == 0
|
2020-08-13 23:09:04 +02:00
|
|
|
|
|
|
|
|
2020-08-15 01:55:00 +02:00
|
|
|
def find_equivalent_patterns(
|
|
|
|
search_pattern, patterns, stix_version=stix2.DEFAULT_VERSION,
|
|
|
|
):
|
2020-08-13 23:09:04 +02:00
|
|
|
"""
|
|
|
|
Find patterns from a sequence which are equivalent to a given pattern.
|
|
|
|
This is more efficient than using equivalent_patterns() in a loop, because
|
|
|
|
it doesn't re-canonicalize the search pattern over and over. This works
|
|
|
|
on an input iterable and is implemented as a generator of matches. So you
|
|
|
|
can "stream" patterns in and matching patterns will be streamed out.
|
|
|
|
|
|
|
|
:param search_pattern: A search pattern as a string
|
|
|
|
:param patterns: An iterable over patterns as strings
|
2020-08-15 01:55:00 +02:00
|
|
|
:param stix_version: The STIX version to use for pattern parsing, as a
|
|
|
|
string ("2.0", "2.1", etc). Defaults to library-wide default version.
|
2020-08-13 23:09:04 +02:00
|
|
|
:return: A generator iterator producing the semantically equivalent
|
|
|
|
patterns
|
|
|
|
"""
|
|
|
|
search_pattern_ast = stix2.pattern_visitor.create_pattern_object(
|
2020-08-15 01:55:00 +02:00
|
|
|
search_pattern, version=stix_version,
|
2020-08-13 23:09:04 +02:00
|
|
|
)
|
|
|
|
|
|
|
|
pattern_canonicalizer = _get_pattern_canonicalizer()
|
|
|
|
canon_search_pattern_ast, _ = pattern_canonicalizer.transform(
|
2020-08-13 23:44:42 +02:00
|
|
|
search_pattern_ast,
|
2020-08-13 23:09:04 +02:00
|
|
|
)
|
|
|
|
|
|
|
|
for pattern in patterns:
|
2020-08-15 01:55:00 +02:00
|
|
|
pattern_ast = stix2.pattern_visitor.create_pattern_object(
|
|
|
|
pattern, version=stix_version,
|
|
|
|
)
|
2020-08-13 23:09:04 +02:00
|
|
|
canon_pattern_ast, _ = pattern_canonicalizer.transform(pattern_ast)
|
|
|
|
|
|
|
|
result = observation_expression_cmp(
|
2020-08-13 23:44:42 +02:00
|
|
|
canon_search_pattern_ast, canon_pattern_ast,
|
2020-08-13 23:09:04 +02:00
|
|
|
)
|
|
|
|
|
|
|
|
if result == 0:
|
|
|
|
yield pattern
|