From cfb7c4c73bdad7597ae9f815bb9b8f5a33e919aa Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Mon, 17 Feb 2020 19:26:21 -0500 Subject: [PATCH 1/9] Fix stix2.pattern_visitor.create_pattern_object() so its documentation at least isn't wrong, and it behaves better. --- stix2/pattern_visitor.py | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/stix2/pattern_visitor.py b/stix2/pattern_visitor.py index b2d7a53..e4601ba 100644 --- a/stix2/pattern_visitor.py +++ b/stix2/pattern_visitor.py @@ -1,16 +1,15 @@ import importlib import inspect -from antlr4 import CommonTokenStream, InputStream -from antlr4.tree.Trees import Trees +from antlr4 import BailErrorStrategy, CommonTokenStream, InputStream +import antlr4.error.Errors import six -from stix2patterns.exceptions import ParseException +from stix2patterns.exceptions import ParseException, ParserErrorListener from stix2patterns.grammars.STIXPatternLexer import STIXPatternLexer from stix2patterns.grammars.STIXPatternParser import ( STIXPatternParser, TerminalNode, ) from stix2patterns.grammars.STIXPatternVisitor import STIXPatternVisitor -from stix2patterns.validator import STIXPatternErrorListener from .patterns import * from .patterns import _BooleanExpression @@ -328,20 +327,12 @@ class STIXPatternVisitorForSTIX2(STIXPatternVisitor): def create_pattern_object(pattern, module_suffix="", module_name=""): """ - Validates a pattern against the STIX Pattern grammar. Error messages are - returned in a list. The test passed if the returned list is empty. + Create a STIX pattern AST from a pattern string. """ - start = '' - if isinstance(pattern, six.string_types): - start = pattern[:2] - pattern = InputStream(pattern) + pattern = InputStream(pattern) - if not start: - start = pattern.readline()[:2] - pattern.seek(0) - - parseErrListener = STIXPatternErrorListener() + parseErrListener = ParserErrorListener() lexer = STIXPatternLexer(pattern) # it always adds a console listener by default... remove it. @@ -350,6 +341,7 @@ def create_pattern_object(pattern, module_suffix="", module_name=""): stream = CommonTokenStream(lexer) parser = STIXPatternParser(stream) + parser._errHandler = BailErrorStrategy() parser.buildParseTrees = True # it always adds a console listener by default... remove it. @@ -363,6 +355,15 @@ def create_pattern_object(pattern, module_suffix="", module_name=""): if lit_name == u"": parser.literalNames[i] = parser.symbolicNames[i] - tree = parser.pattern() + try: + tree = parser.pattern() + except antlr4.error.Errors.ParseCancellationException as e: + real_exc = e.args[0] + parser._errHandler.reportError(parser, real_exc) + six.raise_from( + ParseException(parseErrListener.error_message), + real_exc, + ) + builder = STIXPatternVisitorForSTIX2(module_suffix, module_name) return builder.visit(tree) From 14daa1edae88cb6c08474143b6b7b26d775a4e3c Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Wed, 19 Feb 2020 15:39:23 -0500 Subject: [PATCH 2/9] Add a test case to test parse exceptions from create_pattern_object(). --- stix2/test/v21/test_pattern_expressions.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/stix2/test/v21/test_pattern_expressions.py b/stix2/test/v21/test_pattern_expressions.py index 76880be..0c298f8 100644 --- a/stix2/test/v21/test_pattern_expressions.py +++ b/stix2/test/v21/test_pattern_expressions.py @@ -1,6 +1,7 @@ import datetime import pytest +from stix2patterns.exceptions import ParseException import stix2 from stix2.pattern_visitor import create_pattern_object @@ -515,3 +516,8 @@ def test_list_constant(): def test_parsing_multiple_slashes_quotes(): patt_obj = create_pattern_object("[ file:name = 'weird_name\\'' ]") assert str(patt_obj) == "[file:name = 'weird_name\\'']" + + +def test_parse_error(): + with pytest.raises(ParseException): + create_pattern_object("[ file: name = 'weirdname]") From 76a6eb58736e3ebd566914ca8e1b53a430a9e35d Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Wed, 19 Feb 2020 16:39:15 -0500 Subject: [PATCH 3/9] Greatly simplify the create_pattern_object() function to take advantage of the pattern validator library's Pattern.visit() method. --- stix2/pattern_visitor.py | 39 +++------------------------------------ 1 file changed, 3 insertions(+), 36 deletions(-) diff --git a/stix2/pattern_visitor.py b/stix2/pattern_visitor.py index e4601ba..571c66f 100644 --- a/stix2/pattern_visitor.py +++ b/stix2/pattern_visitor.py @@ -10,6 +10,7 @@ from stix2patterns.grammars.STIXPatternParser import ( STIXPatternParser, TerminalNode, ) from stix2patterns.grammars.STIXPatternVisitor import STIXPatternVisitor +from stix2patterns.v20.pattern import Pattern from .patterns import * from .patterns import _BooleanExpression @@ -330,40 +331,6 @@ def create_pattern_object(pattern, module_suffix="", module_name=""): Create a STIX pattern AST from a pattern string. """ - pattern = InputStream(pattern) - - parseErrListener = ParserErrorListener() - - lexer = STIXPatternLexer(pattern) - # it always adds a console listener by default... remove it. - lexer.removeErrorListeners() - - stream = CommonTokenStream(lexer) - - parser = STIXPatternParser(stream) - parser._errHandler = BailErrorStrategy() - - parser.buildParseTrees = True - # it always adds a console listener by default... remove it. - parser.removeErrorListeners() - parser.addErrorListener(parseErrListener) - - # To improve error messages, replace "" in the literal - # names with symbolic names. This is a hack, but seemed like - # the simplest workaround. - for i, lit_name in enumerate(parser.literalNames): - if lit_name == u"": - parser.literalNames[i] = parser.symbolicNames[i] - - try: - tree = parser.pattern() - except antlr4.error.Errors.ParseCancellationException as e: - real_exc = e.args[0] - parser._errHandler.reportError(parser, real_exc) - six.raise_from( - ParseException(parseErrListener.error_message), - real_exc, - ) - + pattern_obj = Pattern(pattern) builder = STIXPatternVisitorForSTIX2(module_suffix, module_name) - return builder.visit(tree) + return pattern_obj.visit(builder) From 1959cc609737ebf734768be8ec1b53104b0f7859 Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Wed, 19 Feb 2020 16:45:15 -0500 Subject: [PATCH 4/9] Removed a bunch of no-longer-used imports from pattern_visitor.py --- stix2/pattern_visitor.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/stix2/pattern_visitor.py b/stix2/pattern_visitor.py index 571c66f..6ac3e98 100644 --- a/stix2/pattern_visitor.py +++ b/stix2/pattern_visitor.py @@ -1,11 +1,7 @@ import importlib import inspect -from antlr4 import BailErrorStrategy, CommonTokenStream, InputStream -import antlr4.error.Errors -import six -from stix2patterns.exceptions import ParseException, ParserErrorListener -from stix2patterns.grammars.STIXPatternLexer import STIXPatternLexer +from stix2patterns.exceptions import ParseException from stix2patterns.grammars.STIXPatternParser import ( STIXPatternParser, TerminalNode, ) From 41e541959d3c174917030fcee99c07150f8a05f4 Mon Sep 17 00:00:00 2001 From: "Desai, Kartikey H" Date: Mon, 24 Feb 2020 21:11:42 -0500 Subject: [PATCH 5/9] Add _id_contributing_properties functionality to custom SCOs. Tests coming soon. Fixes #351 --- stix2/custom.py | 4 +++- stix2/v21/observables.py | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/stix2/custom.py b/stix2/custom.py index a00498b..81991e9 100644 --- a/stix2/custom.py +++ b/stix2/custom.py @@ -53,7 +53,7 @@ def _custom_marking_builder(cls, type, properties, version): return _CustomMarking -def _custom_observable_builder(cls, type, properties, version): +def _custom_observable_builder(cls, type, properties, version, id_contrib_props=[]): class _CustomObservable(cls, _Observable): if not re.match(TYPE_REGEX, type): @@ -98,6 +98,8 @@ def _custom_observable_builder(cls, type, properties, version): _type = type _properties = OrderedDict(properties) + if version != '2.0': + _id_contributing_properties = id_contrib_props def __init__(self, **kwargs): _Observable.__init__(self, **kwargs) diff --git a/stix2/v21/observables.py b/stix2/v21/observables.py index ed560a6..5c23362 100644 --- a/stix2/v21/observables.py +++ b/stix2/v21/observables.py @@ -966,7 +966,7 @@ class X509Certificate(_Observable): self._check_at_least_one_property(att_list) -def CustomObservable(type='x-custom-observable', properties=None): +def CustomObservable(type='x-custom-observable', properties=None, id_contrib_props=[]): """Custom STIX Cyber Observable Object type decorator. Example: @@ -987,7 +987,7 @@ def CustomObservable(type='x-custom-observable', properties=None): properties, [('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=type))], ])) - return _custom_observable_builder(cls, type, _properties, '2.1') + return _custom_observable_builder(cls, type, _properties, '2.1', id_contrib_props) return wrapper From 055ad97a7aee4e14d1acd8258e9eeccf38254578 Mon Sep 17 00:00:00 2001 From: "Desai, Kartikey H" Date: Thu, 27 Feb 2020 15:15:37 -0500 Subject: [PATCH 6/9] Add tests for _id_contributing_properties for custom observables --- stix2/test/v21/test_custom.py | 72 +++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/stix2/test/v21/test_custom.py b/stix2/test/v21/test_custom.py index 9c650eb..b46288d 100644 --- a/stix2/test/v21/test_custom.py +++ b/stix2/test/v21/test_custom.py @@ -1,3 +1,5 @@ +import uuid + import pytest import stix2 @@ -665,6 +667,76 @@ def test_observed_data_with_custom_observable_object(): assert ob_data.objects['0'].property1 == 'something' +def test_custom_observable_object_det_id_1(): + @stix2.v21.CustomObservable( + 'x-det-id-observable-1', [ + ('property1', stix2.properties.StringProperty(required=True)), + ('property2', stix2.properties.IntegerProperty()), + ], [ + 'property1', + ], + ) + class DetIdObs1(): + pass + + dio_1 = DetIdObs1(property1='I am property1!', property2=42) + dio_2 = DetIdObs1(property1='I am property1!', property2=24) + assert dio_1.property1 == dio_2.property1 == 'I am property1!' + assert dio_1.id == dio_2.id + + uuid_obj = uuid.UUID(dio_1.id[-36:]) + assert uuid_obj.variant == uuid.RFC_4122 + assert uuid_obj.version == 5 + + dio_3 = DetIdObs1(property1='I am property1!', property2=42) + dio_4 = DetIdObs1(property1='I am also property1!', property2=24) + assert dio_3.property1 == 'I am property1!' + assert dio_4.property1 == 'I am also property1!' + assert dio_3.id != dio_4.id + + +def test_custom_observable_object_det_id_2(): + @stix2.v21.CustomObservable( + 'x-det-id-observable-2', [ + ('property1', stix2.properties.StringProperty(required=True)), + ('property2', stix2.properties.IntegerProperty()), + ], [ + 'property1', 'property2', + ], + ) + class DetIdObs2(): + pass + + dio_1 = DetIdObs2(property1='I am property1!', property2=42) + dio_2 = DetIdObs2(property1='I am property1!', property2=42) + assert dio_1.property1 == dio_2.property1 == 'I am property1!' + assert dio_1.property2 == dio_2.property2 == 42 + assert dio_1.id == dio_2.id + + dio_3 = DetIdObs2(property1='I am property1!', property2=42) + dio_4 = DetIdObs2(property1='I am also property1!', property2=42) + assert dio_3.property1 == 'I am property1!' + assert dio_4.property1 == 'I am also property1!' + assert dio_3.property2 == dio_4.property2 == 42 + assert dio_3.id != dio_4.id + + +def test_custom_observable_object_no_id_contrib_props(): + @stix2.v21.CustomObservable( + 'x-det-id-observable-3', [ + ('property1', stix2.properties.StringProperty(required=True)), + ], + ) + class DetIdObs3(): + pass + + dio = DetIdObs3(property1="I am property1!") + + uuid_obj = uuid.UUID(dio.id[-36:]) + assert uuid_obj.variant == uuid.RFC_4122 + assert uuid_obj.version == 4 + + @stix2.v21.CustomExtension( stix2.v21.DomainName, 'x-new-ext', [ ('property1', stix2.properties.StringProperty(required=True)), From 4f00c7ca4fe3088adb664b368d8c1c65ee323362 Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Wed, 4 Mar 2020 13:33:54 -0500 Subject: [PATCH 7/9] Fix patterning test --- stix2/test/v21/test_indicator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stix2/test/v21/test_indicator.py b/stix2/test/v21/test_indicator.py index 152f253..23bad29 100644 --- a/stix2/test/v21/test_indicator.py +++ b/stix2/test/v21/test_indicator.py @@ -271,7 +271,7 @@ def test_indicator_stix20_invalid_pattern(): ) assert excinfo.value.cls == stix2.v21.Indicator - assert "FAIL: The same qualifier is used more than once" in str(excinfo.value) + assert "FAIL: Duplicate qualifier type encountered" in str(excinfo.value) ind = stix2.v21.Indicator( type="indicator", From fc95b400ff4794b0418fab427fd6324163aee352 Mon Sep 17 00:00:00 2001 From: "Desai, Kartikey H" Date: Wed, 4 Mar 2020 14:29:35 -0500 Subject: [PATCH 8/9] Change default parameters from empty lists to None. Fixes #351 --- stix2/custom.py | 2 +- stix2/v21/observables.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/stix2/custom.py b/stix2/custom.py index 81991e9..0db9d25 100644 --- a/stix2/custom.py +++ b/stix2/custom.py @@ -53,7 +53,7 @@ def _custom_marking_builder(cls, type, properties, version): return _CustomMarking -def _custom_observable_builder(cls, type, properties, version, id_contrib_props=[]): +def _custom_observable_builder(cls, type, properties, version, id_contrib_props=None): class _CustomObservable(cls, _Observable): if not re.match(TYPE_REGEX, type): diff --git a/stix2/v21/observables.py b/stix2/v21/observables.py index 5c23362..88d2f6c 100644 --- a/stix2/v21/observables.py +++ b/stix2/v21/observables.py @@ -966,7 +966,7 @@ class X509Certificate(_Observable): self._check_at_least_one_property(att_list) -def CustomObservable(type='x-custom-observable', properties=None, id_contrib_props=[]): +def CustomObservable(type='x-custom-observable', properties=None, id_contrib_props=None): """Custom STIX Cyber Observable Object type decorator. Example: @@ -980,6 +980,9 @@ def CustomObservable(type='x-custom-observable', properties=None, id_contrib_pro ... pass """ + if id_contrib_props is None: + id_contrib_props = [] + def wrapper(cls): _properties = list(itertools.chain.from_iterable([ [('type', TypeProperty(type))], From a5cd0fdc505e5cc6376331232ea4f71e0f4c131d Mon Sep 17 00:00:00 2001 From: "Desai, Kartikey H" Date: Wed, 4 Mar 2020 14:46:55 -0500 Subject: [PATCH 9/9] Change location of None-check for id_contrib_props. Fixes #351 --- stix2/custom.py | 3 +++ stix2/v21/observables.py | 3 --- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/stix2/custom.py b/stix2/custom.py index 0db9d25..802fd07 100644 --- a/stix2/custom.py +++ b/stix2/custom.py @@ -54,6 +54,9 @@ def _custom_marking_builder(cls, type, properties, version): def _custom_observable_builder(cls, type, properties, version, id_contrib_props=None): + if id_contrib_props is None: + id_contrib_props = [] + class _CustomObservable(cls, _Observable): if not re.match(TYPE_REGEX, type): diff --git a/stix2/v21/observables.py b/stix2/v21/observables.py index 88d2f6c..e8c1925 100644 --- a/stix2/v21/observables.py +++ b/stix2/v21/observables.py @@ -980,9 +980,6 @@ def CustomObservable(type='x-custom-observable', properties=None, id_contrib_pro ... pass """ - if id_contrib_props is None: - id_contrib_props = [] - def wrapper(cls): _properties = list(itertools.chain.from_iterable([ [('type', TypeProperty(type))],