MISP
/
cti-python-stix2
mirror of https://github.com/MISP/cti-python-stix2
2
0
Fork 0
Code Issues Releases Wiki Activity

pre-commit hook stylistic changes

pull/1/head
Michael Chisholm 2020-08-13 17:44:42 -04:00
parent 16a8c544ac
commit c21b230edb
11 changed files with 211 additions and 203 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
stix2/equivalence/patterns/__init__.py
View File

@ -1,18 +1,14 @@
import stix2.pattern_visitor
from stix2.equivalence.patterns.transform import (
ChainTransformer, SettleTransformer
)
from stix2.equivalence.patterns.compare.observation import (
observation_expression_cmp
observation_expression_cmp,
)
from stix2.equivalence.patterns.transform import (
ChainTransformer, SettleTransformer,
)
from stix2.equivalence.patterns.transform.observation import (
CanonicalizeComparisonExpressionsTransformer,
AbsorptionTransformer,
FlattenTransformer,
DNFTransformer,
OrderDedupeTransformer
AbsorptionTransformer, CanonicalizeComparisonExpressionsTransformer,
DNFTransformer, FlattenTransformer, OrderDedupeTransformer,
)
import stix2.pattern_visitor
# Lazy-initialize
_pattern_canonicalizer = None
@ -38,7 +34,7 @@ def _get_pattern_canonicalizer():
obs_expr_order = OrderDedupeTransformer()
obs_expr_absorb = AbsorptionTransformer()
obs_simplify = ChainTransformer(
obs_expr_flatten, obs_expr_order, obs_expr_absorb
obs_expr_flatten, obs_expr_order, obs_expr_absorb,
)
obs_settle_simplify = SettleTransformer(obs_simplify)
@ -46,7 +42,7 @@ def _get_pattern_canonicalizer():
_pattern_canonicalizer = ChainTransformer(
canonicalize_comp_expr,
obs_settle_simplify, obs_dnf, obs_settle_simplify
obs_settle_simplify, obs_dnf, obs_settle_simplify,
)
return _pattern_canonicalizer
@ -86,12 +82,12 @@ def find_equivalent_patterns(search_pattern, patterns):
patterns
"""
search_pattern_ast = stix2.pattern_visitor.create_pattern_object(
search_pattern
search_pattern,
)
pattern_canonicalizer = _get_pattern_canonicalizer()
canon_search_pattern_ast, _ = pattern_canonicalizer.transform(
search_pattern_ast
search_pattern_ast,
)
for pattern in patterns:
@ -99,7 +95,7 @@ def find_equivalent_patterns(search_pattern, patterns):
canon_pattern_ast, _ = pattern_canonicalizer.transform(pattern_ast)
result = observation_expression_cmp(
canon_search_pattern_ast, canon_pattern_ast
canon_search_pattern_ast, canon_pattern_ast,
)
if result == 0:

1
stix2/equivalence/patterns/compare/__init__.py
View File

@ -2,6 +2,7 @@
Some generic comparison utility functions.
"""
def generic_cmp(value1, value2):
"""
Generic comparator of values which uses the builtin '<' and '>' operators.

28
stix2/equivalence/patterns/compare/comparison.py
View File

@ -3,18 +3,18 @@ Comparison utilities for STIX pattern comparison expressions.
"""
import base64
import functools
from stix2.patterns import (
_ComparisonExpression, AndBooleanExpression, OrBooleanExpression,
ListObjectPathComponent, IntegerConstant, FloatConstant, StringConstant,
BooleanConstant, TimestampConstant, HexConstant, BinaryConstant,
ListConstant
)
from stix2.equivalence.patterns.compare import generic_cmp, iter_lex_cmp
from stix2.equivalence.patterns.compare import generic_cmp, iter_lex_cmp
from stix2.patterns import (
AndBooleanExpression, BinaryConstant, BooleanConstant, FloatConstant,
HexConstant, IntegerConstant, ListConstant, ListObjectPathComponent,
OrBooleanExpression, StringConstant, TimestampConstant,
_ComparisonExpression,
)
_COMPARISON_OP_ORDER = (
"=", "!=", "<>", "<", "<=", ">", ">=",
"IN", "LIKE", "MATCHES", "ISSUBSET", "ISSUPERSET"
"IN", "LIKE", "MATCHES", "ISSUBSET", "ISSUPERSET",
)
@ -23,7 +23,7 @@ _CONSTANT_TYPE_ORDER = (
# treated equally as a generic "number" type. So they aren't in this list.
# See constant_cmp().
StringConstant, BooleanConstant,
TimestampConstant, HexConstant, BinaryConstant, ListConstant
TimestampConstant, HexConstant, BinaryConstant, ListConstant,
)
@ -111,11 +111,11 @@ def list_cmp(value1, value2):
# Achieve order-independence by sorting the lists first.
sorted_value1 = sorted(
value1.value, key=functools.cmp_to_key(constant_cmp)
value1.value, key=functools.cmp_to_key(constant_cmp),
)
sorted_value2 = sorted(
value2.value, key=functools.cmp_to_key(constant_cmp)
value2.value, key=functools.cmp_to_key(constant_cmp),
)
result = iter_lex_cmp(sorted_value1, sorted_value2, constant_cmp)
@ -131,7 +131,7 @@ _CONSTANT_COMPARATORS = {
TimestampConstant: generic_constant_cmp,
HexConstant: hex_cmp,
BinaryConstant: bin_cmp,
ListConstant: list_cmp
ListConstant: list_cmp,
}
@ -214,7 +214,7 @@ def object_path_cmp(path1, path2):
path_vals1 = object_path_to_raw_values(path1)
path_vals2 = object_path_to_raw_values(path2)
result = iter_lex_cmp(
path_vals1, path_vals2, object_path_component_cmp
path_vals1, path_vals2, object_path_component_cmp,
)
return result
@ -345,7 +345,7 @@ def comparison_expression_cmp(expr1, expr2):
# This will order according to recursive invocations of this comparator,
# on sub-expressions.
result = iter_lex_cmp(
expr1.operands, expr2.operands, comparison_expression_cmp
expr1.operands, expr2.operands, comparison_expression_cmp,
)
return result

29
stix2/equivalence/patterns/compare/observation.py
View File

@ -3,24 +3,23 @@ Comparison utilities for STIX pattern observation expressions.
"""
from stix2.equivalence.patterns.compare import generic_cmp, iter_lex_cmp
from stix2.equivalence.patterns.compare.comparison import (
comparison_expression_cmp, generic_constant_cmp
comparison_expression_cmp, generic_constant_cmp,
)
from stix2.patterns import (
ObservationExpression, AndObservationExpression, OrObservationExpression,
QualifiedObservationExpression, _CompoundObservationExpression,
RepeatQualifier, WithinQualifier, StartStopQualifier,
FollowedByObservationExpression
AndObservationExpression, FollowedByObservationExpression,
ObservationExpression, OrObservationExpression,
QualifiedObservationExpression, RepeatQualifier, StartStopQualifier,
WithinQualifier, _CompoundObservationExpression,
)
_OBSERVATION_EXPRESSION_TYPE_ORDER = (
ObservationExpression, AndObservationExpression, OrObservationExpression,
FollowedByObservationExpression, QualifiedObservationExpression
FollowedByObservationExpression, QualifiedObservationExpression,
)
_QUALIFIER_TYPE_ORDER = (
RepeatQualifier, WithinQualifier, StartStopQualifier
RepeatQualifier, WithinQualifier, StartStopQualifier,
)
@ -36,7 +35,7 @@ def within_cmp(qual1, qual2):
Compare WITHIN qualifiers. This orders by number of seconds.
"""
return generic_constant_cmp(
qual1.number_of_seconds, qual2.number_of_seconds
qual1.number_of_seconds, qual2.number_of_seconds,
)
@ -48,14 +47,14 @@ def startstop_cmp(qual1, qual2):
return iter_lex_cmp(
(qual1.start_time, qual1.stop_time),
(qual2.start_time, qual2.stop_time),
generic_constant_cmp
generic_constant_cmp,
)
_QUALIFIER_COMPARATORS = {
RepeatQualifier: repeats_cmp,
WithinQualifier: within_cmp,
StartStopQualifier: startstop_cmp
StartStopQualifier: startstop_cmp,
}
@ -84,14 +83,14 @@ def observation_expression_cmp(expr1, expr2):
# If they're simple, use contained comparison expression order
elif type1 is ObservationExpression:
result = comparison_expression_cmp(
expr1.operand, expr2.operand
expr1.operand, expr2.operand,
)
elif isinstance(expr1, _CompoundObservationExpression):
# Both compound, and of same type (and/or/followedby): sort according
# to contents.
result = iter_lex_cmp(
expr1.operands, expr2.operands, observation_expression_cmp
expr1.operands, expr2.operands, observation_expression_cmp,
)
else: # QualifiedObservationExpression
@ -112,13 +111,13 @@ def observation_expression_cmp(expr1, expr2):
result = qual_cmp(expr1.qualifier, expr2.qualifier)
else:
raise TypeError(
"Can't compare qualifier type: " + qual1_type.__name__
"Can't compare qualifier type: " + qual1_type.__name__,
)
if result == 0:
# Same qualifier type and details; use qualified expression order
result = observation_expression_cmp(
expr1.observation_expression, expr2.observation_expression
expr1.observation_expression, expr2.observation_expression,
)
return result

3
stix2/equivalence/patterns/transform/__init__.py
View File

@ -2,6 +2,7 @@
Generic AST transformation classes.
"""
class Transformer:
"""
Base class for AST transformers.
@ -16,7 +17,7 @@ class Transformer:
is useful in situations where a transformation needs to be repeated
until the AST stops changing.
"""
raise NotImplemented("transform")
raise NotImplementedError("transform")
class ChainTransformer(Transformer):

31
stix2/equivalence/patterns/transform/comparison.py
View File

@ -3,18 +3,19 @@ Transformation utilities for STIX pattern comparison expressions.
"""
import functools
import itertools
from stix2.equivalence.patterns.compare import iter_in, iter_lex_cmp
from stix2.equivalence.patterns.compare.comparison import (
comparison_expression_cmp,
)
from stix2.equivalence.patterns.transform import Transformer
from stix2.equivalence.patterns.transform.specials import (
windows_reg_key, ipv4_addr, ipv6_addr
ipv4_addr, ipv6_addr, windows_reg_key,
)
from stix2.patterns import (
_BooleanExpression, _ComparisonExpression, AndBooleanExpression,
OrBooleanExpression, ParentheticalExpression
AndBooleanExpression, OrBooleanExpression, ParentheticalExpression,
_BooleanExpression, _ComparisonExpression,
)
from stix2.equivalence.patterns.compare.comparison import (
comparison_expression_cmp
)
from stix2.equivalence.patterns.compare import iter_lex_cmp, iter_in
def _dupe_ast(ast):
@ -119,7 +120,7 @@ class ComparisonExpressionTransformer(Transformer):
elif isinstance(ast, _ComparisonExpression):
meth = getattr(
self, "transform_comparison", self.transform_default
self, "transform_comparison", self.transform_default,
)
else:
@ -156,7 +157,7 @@ class OrderDedupeTransformer(
:return: The same AST node, but with sorted children
"""
sorted_children = sorted(
ast.operands, key=functools.cmp_to_key(comparison_expression_cmp)
ast.operands, key=functools.cmp_to_key(comparison_expression_cmp),
)
deduped_children = [
@ -165,13 +166,13 @@ class OrderDedupeTransformer(
# need key wrappers in our ASTs!
k.obj for k, _ in itertools.groupby(
sorted_children, key=functools.cmp_to_key(
comparison_expression_cmp
)
comparison_expression_cmp,
),
)
]
changed = iter_lex_cmp(
ast.operands, deduped_children, comparison_expression_cmp
ast.operands, deduped_children, comparison_expression_cmp,
) != 0
ast.operands = deduped_children
@ -268,7 +269,7 @@ class AbsorptionTransformer(
# The simple check: is child1 contained in child2?
if iter_in(
child1, child2.operands, comparison_expression_cmp
child1, child2.operands, comparison_expression_cmp,
):
to_delete.add(j)
@ -278,7 +279,7 @@ class AbsorptionTransformer(
if all(
iter_in(
child1_operand, child2.operands,
comparison_expression_cmp
comparison_expression_cmp,
)
for child1_operand in child1.operands
):
@ -326,7 +327,7 @@ class DNFTransformer(ComparisonExpressionTransformer):
# we should ensure each repetition is independent of the
# others.
_dupe_ast(sub_ast) for sub_ast in itertools.chain(
other_children, prod_seq
other_children, prod_seq,
)
])
for prod_seq in itertools.product(*or_children)

59
stix2/equivalence/patterns/transform/observation.py
View File

@ -3,23 +3,30 @@ Transformation utilities for STIX pattern observation expressions.
"""
import functools
import itertools
from stix2.patterns import (
ObservationExpression, AndObservationExpression, OrObservationExpression,
QualifiedObservationExpression, _CompoundObservationExpression,
ParentheticalExpression, FollowedByObservationExpression
from stix2.equivalence.patterns.compare import iter_in, iter_lex_cmp
from stix2.equivalence.patterns.compare.observation import (
observation_expression_cmp,
)
from stix2.equivalence.patterns.transform import (
ChainTransformer, SettleTransformer, Transformer
ChainTransformer, SettleTransformer, Transformer,
)
from stix2.equivalence.patterns.transform.comparison import (
FlattenTransformer as CFlattenTransformer,
OrderDedupeTransformer as COrderDedupeTransformer,
AbsorptionTransformer as CAbsorptionTransformer,
DNFTransformer as CDNFTransformer,
SpecialValueCanonicalization
SpecialValueCanonicalization,
)
from stix2.equivalence.patterns.transform.comparison import \
AbsorptionTransformer as CAbsorptionTransformer
from stix2.equivalence.patterns.transform.comparison import \
DNFTransformer as CDNFTransformer
from stix2.equivalence.patterns.transform.comparison import \
FlattenTransformer as CFlattenTransformer
from stix2.equivalence.patterns.transform.comparison import \
OrderDedupeTransformer as COrderDedupeTransformer
from stix2.patterns import (
AndObservationExpression, FollowedByObservationExpression,
ObservationExpression, OrObservationExpression, ParentheticalExpression,
QualifiedObservationExpression, _CompoundObservationExpression,
)
from stix2.equivalence.patterns.compare import iter_lex_cmp, iter_in
from stix2.equivalence.patterns.compare.observation import observation_expression_cmp
def _dupe_ast(ast):
@ -52,7 +59,7 @@ def _dupe_ast(ast):
elif isinstance(ast, QualifiedObservationExpression):
# Don't need to dupe the qualifier object at this point
result = QualifiedObservationExpression(
_dupe_ast(ast.observation_expression), ast.qualifier
_dupe_ast(ast.observation_expression), ast.qualifier,
)
elif isinstance(ast, ObservationExpression):
@ -100,7 +107,7 @@ class ObservationExpressionTransformer(Transformer):
AndObservationExpression: "and",
OrObservationExpression: "or",
FollowedByObservationExpression: "followedby",
QualifiedObservationExpression: "qualified"
QualifiedObservationExpression: "qualified",
}
def transform(self, ast):
@ -143,7 +150,7 @@ class ObservationExpressionTransformer(Transformer):
else:
raise TypeError("Not an observation expression: {}: {}".format(
type(ast).__name__, str(ast)
type(ast).__name__, str(ast),
))
return result, changed
@ -228,7 +235,7 @@ class OrderDedupeTransformer(
def __transform(self, ast):
sorted_children = sorted(
ast.operands, key=functools.cmp_to_key(observation_expression_cmp)
ast.operands, key=functools.cmp_to_key(observation_expression_cmp),
)
# Deduping only applies to ORs
@ -236,15 +243,15 @@ class OrderDedupeTransformer(
deduped_children = [
key.obj for key, _ in itertools.groupby(
sorted_children, key=functools.cmp_to_key(
observation_expression_cmp
)
observation_expression_cmp,
),
)
]
else:
deduped_children = sorted_children
changed = iter_lex_cmp(
ast.operands, deduped_children, observation_expression_cmp
ast.operands, deduped_children, observation_expression_cmp,
) != 0
ast.operands = deduped_children
@ -376,12 +383,12 @@ class AbsorptionTransformer(
if isinstance(
child2, (
AndObservationExpression,
FollowedByObservationExpression
)
FollowedByObservationExpression,
),
):
# The simple check: is child1 contained in child2?
if iter_in(
child1, child2.operands, observation_expression_cmp
child1, child2.operands, observation_expression_cmp,
):
to_delete.add(j)
@ -390,11 +397,11 @@ class AbsorptionTransformer(
elif type(child1) is type(child2):
if isinstance(child1, AndObservationExpression):
can_simplify = self.__is_contained_and(
child1.operands, child2.operands
child1.operands, child2.operands,
)
else: # child1 and 2 are followedby nodes
can_simplify = self.__is_contained_followedby(
child1.operands, child2.operands
child1.operands, child2.operands,
)
if can_simplify:
@ -434,7 +441,7 @@ class DNFTransformer(ObservationExpressionTransformer):
distributed_children = [
root_type([
_dupe_ast(sub_ast) for sub_ast in itertools.chain(
other_children, prod_seq
other_children, prod_seq,
)
])
for prod_seq in itertools.product(*or_children)
@ -477,7 +484,7 @@ class CanonicalizeComparisonExpressionsTransformer(
comp_special = SpecialValueCanonicalization()
comp_dnf = CDNFTransformer()
self.__comp_canonicalize = ChainTransformer(
comp_special, settle_simplify, comp_dnf, settle_simplify
comp_special, settle_simplify, comp_dnf, settle_simplify,
)
def transform_observation(self, ast):

6
stix2/equivalence/patterns/transform/specials.py
View File

@ -2,10 +2,10 @@
Some simple comparison expression canonicalization functions.
"""
import socket
from stix2.equivalence.patterns.compare.comparison import (
object_path_to_raw_values
)
from stix2.equivalence.patterns.compare.comparison import (
object_path_to_raw_values,
)
# Values we can use as wildcards in path patterns
_ANY_IDX = object()

8
stix2/pattern_visitor.py
View File

@ -2,8 +2,8 @@
import importlib
import inspect
from six import text_type
from six import text_type
from stix2patterns.exceptions import ParseException
from stix2patterns.grammars.STIXPatternParser import TerminalNode
from stix2patterns.v20.grammars.STIXPatternParser import \
@ -261,9 +261,11 @@ class STIXPatternVisitorForSTIX2():
property_path.append(self.instantiate("ListObjectPathComponent", current.property_name, next.getText()))
i += 2
elif isinstance(next, IntegerConstant):
property_path.append(self.instantiate("ListObjectPathComponent",
property_path.append(self.instantiate(
"ListObjectPathComponent",
current.property_name if isinstance(current, BasicObjectPathComponent) else text_type(current),
next.value))
next.value,
))
i += 2
else:
property_path.append(current)

218
stix2/test/test_pattern_equivalence.py
View File

@ -1,8 +1,8 @@
import pytest
from stix2.equivalence.patterns import (
equivalent_patterns, find_equivalent_patterns
)
from stix2.equivalence.patterns import (
equivalent_patterns, find_equivalent_patterns,
)
# # # #
# # Observation expression equivalence tests # #
@ -13,13 +13,13 @@ from stix2.equivalence.patterns import (
"patt1, patt2", [
(
"[a:b=1] OR [a:b=1]",
"[a:b=1]"
"[a:b=1]",
),
(
"[a:b=1] OR [a:b=1] OR [a:b=1]",
"[a:b=1]"
"[a:b=1]",
),
]
],
)
def test_obs_dupe_equivalent(patt1, patt2):
assert equivalent_patterns(patt1, patt2)
@ -29,13 +29,13 @@ def test_obs_dupe_equivalent(patt1, patt2):
"patt1, patt2", [
(
"[a:b=1] AND [a:b=1]",
"[a:b=1]"
"[a:b=1]",
),
(
"[a:b=1] FOLLOWEDBY [a:b=1]",
"[a:b=1]"
"[a:b=1]",
),
]
],
)
def test_obs_dupe_not_equivalent(patt1, patt2):
assert not equivalent_patterns(patt1, patt2)
@ -72,8 +72,8 @@ def test_obs_dupe_not_equivalent(patt1, patt2):
(
"[a:b=1] AND ([a:b=2] AND ([a:b=3] AND [a:b=4])) AND ([a:b=5])",
"([a:b=1] AND ([a:b=2] AND [a:b=3]) AND ([a:b=4] AND [a:b=5]))",
)
]
),
],
)
def test_obs_flatten_equivalent(patt1, patt2):
assert equivalent_patterns(patt1, patt2)
@ -103,7 +103,7 @@ def test_obs_flatten_equivalent(patt1, patt2):
"[a:b=1] FOLLOWEDBY ([a:b=2] FOLLOWEDBY [a:b=3]) WITHIN 2 SECONDS",
"[a:b=1] WITHIN 2 SECONDS FOLLOWEDBY [a:b=2] FOLLOWEDBY [a:b=3]",
),
]
],
)
def test_obs_flatten_not_equivalent(patt1, patt2):
assert not equivalent_patterns(patt1, patt2)
@ -113,21 +113,21 @@ def test_obs_flatten_not_equivalent(patt1, patt2):
"patt1, patt2", [
(
"[a:b=1] AND [a:b=2]",
"[a:b=2] AND [a:b=1]"
"[a:b=2] AND [a:b=1]",
),
(
"[a:b=1] OR [a:b=2]",
"[a:b=2] OR [a:b=1]"
"[a:b=2] OR [a:b=1]",
),
(
"[a:b=1] OR ([a:b=2] AND [a:b=3])",
"([a:b=3] AND [a:b=2]) OR [a:b=1]"
"([a:b=3] AND [a:b=2]) OR [a:b=1]",
),
(
"[a:b=1] WITHIN 2 SECONDS AND [a:b=2] REPEATS 2 TIMES",
"[a:b=2] REPEATS 2 TIMES AND [a:b=1] WITHIN 2 SECONDS"
)
]
"[a:b=2] REPEATS 2 TIMES AND [a:b=1] WITHIN 2 SECONDS",
),
],
)
def test_obs_order_equivalent(patt1, patt2):
assert equivalent_patterns(patt1, patt2)
@ -137,13 +137,13 @@ def test_obs_order_equivalent(patt1, patt2):
"patt1, patt2", [
(
"[a:b=1] FOLLOWEDBY [a:b=2]",
"[a:b=2] FOLLOWEDBY [a:b=1]"
"[a:b=2] FOLLOWEDBY [a:b=1]",
),
(
"[a:b=1] WITHIN 2 SECONDS AND [a:b=2] REPEATS 2 TIMES",
"[a:b=1] REPEATS 2 TIMES AND [a:b=2] WITHIN 2 SECONDS"
)
]
"[a:b=1] REPEATS 2 TIMES AND [a:b=2] WITHIN 2 SECONDS",
),
],
)
def test_obs_order_not_equivalent(patt1, patt2):
assert not equivalent_patterns(patt1, patt2)
@ -153,29 +153,29 @@ def test_obs_order_not_equivalent(patt1, patt2):
"patt1, patt2", [
(
"[a:b=1] OR ([a:b=1] AND [a:b=2])",
"[a:b=1]"
"[a:b=1]",
),
(
"[a:b=1] OR ([a:b=1] FOLLOWEDBY [a:b=2])",
"[a:b=1]"
"[a:b=1]",
),
(
"([a:b=3] AND [a:b=1]) OR ([a:b=1] AND [a:b=2] AND [a:b=3])",
"[a:b=3] AND [a:b=1]"
"[a:b=3] AND [a:b=1]",
),
(
"([a:b=1] FOLLOWEDBY [a:b=3]) OR ([a:b=4] FOLLOWEDBY [a:b=1] FOLLOWEDBY [a:b=2] FOLLOWEDBY [a:b=3])",
"[a:b=1] FOLLOWEDBY [a:b=3]"
"[a:b=1] FOLLOWEDBY [a:b=3]",
),
(
"([a:b=1] FOLLOWEDBY [a:b=2]) OR (([a:b=1] FOLLOWEDBY [a:b=2]) AND [a:b=3])",
"[a:b=1] FOLLOWEDBY [a:b=2]"
"[a:b=1] FOLLOWEDBY [a:b=2]",
),
(
"([a:b=1] AND [a:b=2]) OR (([a:b=1] AND [a:b=2]) FOLLOWEDBY [a:b=3])",
"[a:b=1] AND [a:b=2]"
"[a:b=1] AND [a:b=2]",
),
]
],
)
def test_obs_absorb_equivalent(patt1, patt2):
assert equivalent_patterns(patt1, patt2)
@ -185,13 +185,13 @@ def test_obs_absorb_equivalent(patt1, patt2):
"patt1, patt2", [
(
"([a:b=1] AND [a:b=2]) OR ([a:b=2] AND [a:b=3] AND [a:b=4])",
"[a:b=1] AND [a:b=2]"
"[a:b=1] AND [a:b=2]",
),
(
"([a:b=2] FOLLOWEDBY [a:b=1]) OR ([a:b=1] FOLLOWEDBY [a:b=2] FOLLOWEDBY [a:b=3])",
"[a:b=2] FOLLOWEDBY [a:b=1]"
)
]
"[a:b=2] FOLLOWEDBY [a:b=1]",
),
],
)
def test_obs_absorb_not_equivalent(patt1, patt2):
assert not equivalent_patterns(patt1, patt2)
@ -201,29 +201,29 @@ def test_obs_absorb_not_equivalent(patt1, patt2):
"patt1, patt2", [
(
"[a:b=1] AND ([a:b=2] OR [a:b=3])",
"([a:b=1] AND [a:b=2]) OR ([a:b=1] AND [a:b=3])"
"([a:b=1] AND [a:b=2]) OR ([a:b=1] AND [a:b=3])",
),
(
"[a:b=1] FOLLOWEDBY ([a:b=2] OR [a:b=3])",
"([a:b=1] FOLLOWEDBY [a:b=2]) OR ([a:b=1] FOLLOWEDBY [a:b=3])"
"([a:b=1] FOLLOWEDBY [a:b=2]) OR ([a:b=1] FOLLOWEDBY [a:b=3])",
),
(
"[a:b=1] AND ([a:b=2] AND ([a:b=3] OR [a:b=4]))",
"([a:b=1] AND [a:b=2] AND [a:b=3]) OR ([a:b=1] AND [a:b=2] AND [a:b=4])"
"([a:b=1] AND [a:b=2] AND [a:b=3]) OR ([a:b=1] AND [a:b=2] AND [a:b=4])",
),
(
"[a:b=1] FOLLOWEDBY ([a:b=2] FOLLOWEDBY ([a:b=3] OR [a:b=4]))",
"([a:b=1] FOLLOWEDBY [a:b=2] FOLLOWEDBY [a:b=3]) OR ([a:b=1] FOLLOWEDBY [a:b=2] FOLLOWEDBY [a:b=4])"
"([a:b=1] FOLLOWEDBY [a:b=2] FOLLOWEDBY [a:b=3]) OR ([a:b=1] FOLLOWEDBY [a:b=2] FOLLOWEDBY [a:b=4])",
),
(
"([a:b=1] OR [a:b=2]) AND ([a:b=3] OR [a:b=4])",
"([a:b=1] AND [a:b=3]) OR ([a:b=1] AND [a:b=4]) OR ([a:b=2] AND [a:b=3]) OR ([a:b=2] AND [a:b=4])"
"([a:b=1] AND [a:b=3]) OR ([a:b=1] AND [a:b=4]) OR ([a:b=2] AND [a:b=3]) OR ([a:b=2] AND [a:b=4])",
),
(
"([a:b=1] OR [a:b=2]) FOLLOWEDBY ([a:b=3] OR [a:b=4])",
"([a:b=1] FOLLOWEDBY [a:b=3]) OR ([a:b=1] FOLLOWEDBY [a:b=4]) OR ([a:b=2] FOLLOWEDBY [a:b=3]) OR ([a:b=2] FOLLOWEDBY [a:b=4])"
"([a:b=1] FOLLOWEDBY [a:b=3]) OR ([a:b=1] FOLLOWEDBY [a:b=4]) OR ([a:b=2] FOLLOWEDBY [a:b=3]) OR ([a:b=2] FOLLOWEDBY [a:b=4])",
),
]
],
)
def test_obs_dnf_equivalent(patt1, patt2):
assert equivalent_patterns(patt1, patt2)
@ -233,17 +233,17 @@ def test_obs_dnf_equivalent(patt1, patt2):
"patt1, patt2", [
(
"[a:b=1] AND [a:b=2]",
"[a:b=1] OR [a:b=2]"
"[a:b=1] OR [a:b=2]",
),
(
"[a:b=1] AND ([a:b=2] OR [a:b=3])",
"([a:b=1] AND [a:b=2]) OR [a:b=3]"
"([a:b=1] AND [a:b=2]) OR [a:b=3]",
),
(
"[a:b=1] WITHIN 2 SECONDS",
"[a:b=1] REPEATS 2 TIMES"
)
]
"[a:b=1] REPEATS 2 TIMES",
),
],
)
def test_obs_not_equivalent(patt1, patt2):
assert not equivalent_patterns(patt1, patt2)
@ -258,21 +258,21 @@ def test_obs_not_equivalent(patt1, patt2):
"patt1, patt2", [
(
"[a:b=1 AND a:b=1]",
"[a:b=1]"
"[a:b=1]",
),
(
"[a:b=1 AND a:b=1 AND a:b=1]",
"[a:b=1]"
"[a:b=1]",
),
(
"[a:b=1 OR a:b=1]",
"[a:b=1]"
"[a:b=1]",
),
(
"[a:b=1 OR a:b=1 OR a:b=1]",
"[a:b=1]"
)
]
"[a:b=1]",
),
],
)
def test_comp_dupe_equivalent(patt1, patt2):
assert equivalent_patterns(patt1, patt2)
@ -282,29 +282,29 @@ def test_comp_dupe_equivalent(patt1, patt2):
"patt1, patt2", [
(
"[(a:b=1)]",
"[a:b=1]"
"[a:b=1]",
),
(
"[(((((a:b=1)))))]",
"[(a:b=1)]"
"[(a:b=1)]",
),
(
"[a:b=1 AND (a:b=2 AND a:b=3)]",
"[(a:b=1 AND a:b=2) AND a:b=3]"
"[(a:b=1 AND a:b=2) AND a:b=3]",
),
(
"[a:b=1 OR (a:b=2 OR a:b=3)]",
"[(a:b=1 OR a:b=2) OR a:b=3]"
"[(a:b=1 OR a:b=2) OR a:b=3]",
),
(
"[(((a:b=1 AND ((a:b=2) AND a:b=3) AND (a:b=4))))]",
"[a:b=1 AND a:b=2 AND a:b=3 AND a:b=4]"
"[a:b=1 AND a:b=2 AND a:b=3 AND a:b=4]",
),
(
"[(((a:b=1 OR ((a:b=2) OR a:b=3) OR (a:b=4))))]",
"[a:b=1 OR a:b=2 OR a:b=3 OR a:b=4]"
"[a:b=1 OR a:b=2 OR a:b=3 OR a:b=4]",
),
]
],
)
def test_comp_flatten_equivalent(patt1, patt2):
assert equivalent_patterns(patt1, patt2)
@ -314,17 +314,17 @@ def test_comp_flatten_equivalent(patt1, patt2):
"patt1, patt2", [
(
"[a:b=1 AND a:b=2]",
"[a:b=2 AND a:b=1]"
"[a:b=2 AND a:b=1]",
),
(
"[a:b=1 OR a:b=2]",
"[a:b=2 OR a:b=1]"
"[a:b=2 OR a:b=1]",
),
(
"[(a:b=1 OR a:b=2) AND a:b=3]",
"[a:b=3 AND (a:b=2 OR a:b=1)]",
)
]
),
],
)
def test_comp_order_equivalent(patt1, patt2):
assert equivalent_patterns(patt1, patt2)
@ -334,21 +334,21 @@ def test_comp_order_equivalent(patt1, patt2):
"patt1, patt2", [
(
"[a:b=1 OR (a:b=1 AND a:b=2)]",
"[a:b=1]"
"[a:b=1]",
),
(
"[a:b=1 AND (a:b=1 OR a:b=2)]",
"[a:b=1]"
"[a:b=1]",
),
(
"[(a:b=1 AND a:b=2) OR (a:b=3 AND a:b=2 AND a:b=1)]",
"[a:b=1 AND a:b=2]"
"[a:b=1 AND a:b=2]",
),
(
"[(a:b=1 OR a:b=2) AND (a:b=3 OR a:b=2 OR a:b=1)]",
"[a:b=1 OR a:b=2]"
)
]
"[a:b=1 OR a:b=2]",
),
],
)
def test_comp_absorb_equivalent(patt1, patt2):
assert equivalent_patterns(patt1, patt2)
@ -358,25 +358,25 @@ def test_comp_absorb_equivalent(patt1, patt2):
"patt1, patt2", [
(
"[a:b=1 OR (a:b=2 AND a:b=3)]",
"[(a:b=1 OR a:b=2) AND (a:b=1 OR a:b=3)]"
"[(a:b=1 OR a:b=2) AND (a:b=1 OR a:b=3)]",
),
(
"[a:b=1 AND (a:b=2 OR a:b=3)]",
"[(a:b=1 AND a:b=2) OR (a:b=1 AND a:b=3)]"
"[(a:b=1 AND a:b=2) OR (a:b=1 AND a:b=3)]",
),
(
"[(a:b=1 AND a:b=2) OR (a:b=3 AND a:b=4)]",
"[(a:b=1 OR a:b=3) AND (a:b=1 OR a:b=4) AND (a:b=2 OR a:b=3) AND (a:b=2 OR a:b=4)]"
"[(a:b=1 OR a:b=3) AND (a:b=1 OR a:b=4) AND (a:b=2 OR a:b=3) AND (a:b=2 OR a:b=4)]",
),
(
"[(a:b=1 OR a:b=2) AND (a:b=3 OR a:b=4)]",
"[(a:b=1 AND a:b=3) OR (a:b=1 AND a:b=4) OR (a:b=2 AND a:b=3) OR (a:b=2 AND a:b=4)]"
"[(a:b=1 AND a:b=3) OR (a:b=1 AND a:b=4) OR (a:b=2 AND a:b=3) OR (a:b=2 AND a:b=4)]",
),
(
"[a:b=1 AND (a:b=2 AND (a:b=3 OR a:b=4))]",
"[(a:b=1 AND a:b=2 AND a:b=3) OR (a:b=1 AND a:b=2 AND a:b=4)]"
)
]
"[(a:b=1 AND a:b=2 AND a:b=3) OR (a:b=1 AND a:b=2 AND a:b=4)]",
),
],
)
def test_comp_dnf_equivalent(patt1, patt2):
assert equivalent_patterns(patt1, patt2)
@ -386,17 +386,17 @@ def test_comp_dnf_equivalent(patt1, patt2):
"patt1, patt2", [
(
"[a:b=1]",
"[a:b=2]"
"[a:b=2]",
),
(
"[a:b=1 AND a:b=2]",
"[a:b=1 OR a:b=2]"
"[a:b=1 OR a:b=2]",
),
(
"[(a:b=1 AND a:b=2) OR a:b=3]",
"[a:b=1 AND (a:b=2 OR a:b=3)]"
"[a:b=1 AND (a:b=2 OR a:b=3)]",
),
]
],
)
def test_comp_not_equivalent(patt1, patt2):
assert not equivalent_patterns(patt1, patt2)
@ -406,41 +406,41 @@ def test_comp_not_equivalent(patt1, patt2):
"patt1, patt2", [
(
"[ipv4-addr:value='1.2.3.4/32']",
"[ipv4-addr:value='1.2.3.4']"
"[ipv4-addr:value='1.2.3.4']",
),
(
"[ipv4-addr:value='1.2.3.4/24']",
"[ipv4-addr:value='1.2.3.0/24']"
"[ipv4-addr:value='1.2.3.0/24']",
),
(
"[ipv4-addr:value='1.2.255.4/23']",
"[ipv4-addr:value='1.2.254.0/23']"
"[ipv4-addr:value='1.2.254.0/23']",
),
(
"[ipv4-addr:value='1.2.255.4/20']",
"[ipv4-addr:value='1.2.240.0/20']"
"[ipv4-addr:value='1.2.240.0/20']",
),
(
"[ipv4-addr:value='1.2.255.4/0']",
"[ipv4-addr:value='0.0.0.0/0']"
"[ipv4-addr:value='0.0.0.0/0']",
),
(
"[ipv4-addr:value='01.02.03.04']",
"[ipv4-addr:value='1.2.3.4']"
"[ipv4-addr:value='1.2.3.4']",
),
(
"[ipv4-addr:value='1.2.3.4/-5']",
"[ipv4-addr:value='1.2.3.4/-5']"
"[ipv4-addr:value='1.2.3.4/-5']",
),
(
"[ipv4-addr:value='1.2.3.4/99']",
"[ipv4-addr:value='1.2.3.4/99']"
"[ipv4-addr:value='1.2.3.4/99']",
),
(
"[ipv4-addr:value='foo']",
"[ipv4-addr:value='foo']"
"[ipv4-addr:value='foo']",
),
]
],
)
def test_comp_special_canonicalization_ipv4(patt1, patt2):
assert equivalent_patterns(patt1, patt2)
@ -450,17 +450,17 @@ def test_comp_special_canonicalization_ipv4(patt1, patt2):
"patt1, patt2", [
(
"[ipv4-addr:value='1.2.3.4']",
"[ipv4-addr:value='1.2.3.5']"
"[ipv4-addr:value='1.2.3.5']",
),
(
"[ipv4-addr:value='1.2.3.4/1']",
"[ipv4-addr:value='1.2.3.4/2']"
"[ipv4-addr:value='1.2.3.4/2']",
),
(
"[ipv4-addr:value='foo']",
"[ipv4-addr:value='bar']"
"[ipv4-addr:value='bar']",
),
]
],
)
def test_comp_special_canonicalization_ipv4_not_equivalent(patt1, patt2):
assert not equivalent_patterns(patt1, patt2)
@ -470,45 +470,45 @@ def test_comp_special_canonicalization_ipv4_not_equivalent(patt1, patt2):
"patt1, patt2", [
(
"[ipv6-addr:value='1:2:3:4:5:6:7:8/128']",
"[ipv6-addr:value='1:2:3:4:5:6:7:8']"
"[ipv6-addr:value='1:2:3:4:5:6:7:8']",
),
(
"[ipv6-addr:value='1:2:3:4:5:6:7:8/112']",
"[ipv6-addr:value='1:2:3:4:5:6:7:0/112']"
"[ipv6-addr:value='1:2:3:4:5:6:7:0/112']",
),
(
"[ipv6-addr:value='1:2:3:4:5:6:ffff:8/111']",
"[ipv6-addr:value='1:2:3:4:5:6:fffe:0/111']"
"[ipv6-addr:value='1:2:3:4:5:6:fffe:0/111']",
),
(
"[ipv6-addr:value='1:2:3:4:5:6:ffff:8/104']",
"[ipv6-addr:value='1:2:3:4:5:6:ff00:0/104']"
"[ipv6-addr:value='1:2:3:4:5:6:ff00:0/104']",
),
(
"[ipv6-addr:value='1:2:3:4:5:6:7:8/0']",
"[ipv6-addr:value='0:0:0:0:0:0:0:0/0']"
"[ipv6-addr:value='0:0:0:0:0:0:0:0/0']",
),
(
"[ipv6-addr:value='0001:0000:0000:0000:0000:0000:0000:0001']",
"[ipv6-addr:value='1::1']"
"[ipv6-addr:value='1::1']",
),
(
"[ipv6-addr:value='0000:0000:0000:0000:0000:0000:0000:0000']",
"[ipv6-addr:value='::']"
"[ipv6-addr:value='::']",
),
(
"[ipv6-addr:value='1:2:3:4:5:6:7:8/-5']",
"[ipv6-addr:value='1:2:3:4:5:6:7:8/-5']"
"[ipv6-addr:value='1:2:3:4:5:6:7:8/-5']",
),
(
"[ipv6-addr:value='1:2:3:4:5:6:7:8/99']",
"[ipv6-addr:value='1:2:3:4:5:6:7:8/99']"
"[ipv6-addr:value='1:2:3:4:5:6:7:8/99']",
),
(
"[ipv6-addr:value='foo']",
"[ipv6-addr:value='foo']"
"[ipv6-addr:value='foo']",
),
]
],
)
def test_comp_special_canonicalization_ipv6(patt1, patt2):
assert equivalent_patterns(patt1, patt2)
@ -528,7 +528,7 @@ def test_comp_special_canonicalization_ipv6(patt1, patt2):
"[ipv6-addr:value='foo']",
"[ipv6-addr:value='bar']",
),
]
],
)
def test_comp_special_canonicalization_ipv6_not_equivalent(patt1, patt2):
assert not equivalent_patterns(patt1, patt2)
@ -548,7 +548,7 @@ def test_comp_special_canonicalization_ipv6_not_equivalent(patt1, patt2):
"[windows-registry-key:values[*].name = 'aaa']",
"[windows-registry-key:values[*].name = 'AAA']",
),
]
],
)
def test_comp_special_canonicalization_win_reg_key(patt1, patt2):
assert equivalent_patterns(patt1, patt2)
@ -572,7 +572,7 @@ def test_comp_special_canonicalization_win_reg_key(patt1, patt2):
"[windows-registry-key:values[*].data='foo']",
"[windows-registry-key:values[*].data='FOO']",
),
]
],
)
def test_comp_special_canonicalization_win_reg_key_not_equivalent(patt1, patt2):
assert not equivalent_patterns(patt1, patt2)
@ -591,11 +591,11 @@ def test_find_equivalent_patterns():
"[a:b=1] OR ([a:b=2] AND [a:b=1])",
"[(a:b=2 OR a:b=1) AND a:b=1]",
"[c:d=1]",
"[a:b>1]"
"[a:b>1]",
]
result = list(
find_equivalent_patterns(search_pattern, other_patterns)
find_equivalent_patterns(search_pattern, other_patterns),
)
assert result == [

1
stix2/test/v21/test_pattern_expressions.py
View File

@ -658,6 +658,7 @@ def test_parsing_integer_index():
patt_obj = create_pattern_object("[a:b[1]=2]")
assert str(patt_obj) == "[a:b[1] = 2]"
# This should never occur, because the first component will always be a property_name, and they should not be quoted.
def test_parsing_quoted_first_path_component():
patt_obj = create_pattern_object("[a:'b'[1]=2]")