From cfb7c4c73bdad7597ae9f815bb9b8f5a33e919aa Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Mon, 17 Feb 2020 19:26:21 -0500 Subject: [PATCH 1/5] Fix stix2.pattern_visitor.create_pattern_object() so its documentation at least isn't wrong, and it behaves better. --- stix2/pattern_visitor.py | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/stix2/pattern_visitor.py b/stix2/pattern_visitor.py index b2d7a53..e4601ba 100644 --- a/stix2/pattern_visitor.py +++ b/stix2/pattern_visitor.py @@ -1,16 +1,15 @@ import importlib import inspect -from antlr4 import CommonTokenStream, InputStream -from antlr4.tree.Trees import Trees +from antlr4 import BailErrorStrategy, CommonTokenStream, InputStream +import antlr4.error.Errors import six -from stix2patterns.exceptions import ParseException +from stix2patterns.exceptions import ParseException, ParserErrorListener from stix2patterns.grammars.STIXPatternLexer import STIXPatternLexer from stix2patterns.grammars.STIXPatternParser import ( STIXPatternParser, TerminalNode, ) from stix2patterns.grammars.STIXPatternVisitor import STIXPatternVisitor -from stix2patterns.validator import STIXPatternErrorListener from .patterns import * from .patterns import _BooleanExpression @@ -328,20 +327,12 @@ class STIXPatternVisitorForSTIX2(STIXPatternVisitor): def create_pattern_object(pattern, module_suffix="", module_name=""): """ - Validates a pattern against the STIX Pattern grammar. Error messages are - returned in a list. The test passed if the returned list is empty. + Create a STIX pattern AST from a pattern string. """ - start = '' - if isinstance(pattern, six.string_types): - start = pattern[:2] - pattern = InputStream(pattern) + pattern = InputStream(pattern) - if not start: - start = pattern.readline()[:2] - pattern.seek(0) - - parseErrListener = STIXPatternErrorListener() + parseErrListener = ParserErrorListener() lexer = STIXPatternLexer(pattern) # it always adds a console listener by default... remove it. @@ -350,6 +341,7 @@ def create_pattern_object(pattern, module_suffix="", module_name=""): stream = CommonTokenStream(lexer) parser = STIXPatternParser(stream) + parser._errHandler = BailErrorStrategy() parser.buildParseTrees = True # it always adds a console listener by default... remove it. @@ -363,6 +355,15 @@ def create_pattern_object(pattern, module_suffix="", module_name=""): if lit_name == u"": parser.literalNames[i] = parser.symbolicNames[i] - tree = parser.pattern() + try: + tree = parser.pattern() + except antlr4.error.Errors.ParseCancellationException as e: + real_exc = e.args[0] + parser._errHandler.reportError(parser, real_exc) + six.raise_from( + ParseException(parseErrListener.error_message), + real_exc, + ) + builder = STIXPatternVisitorForSTIX2(module_suffix, module_name) return builder.visit(tree) From 14daa1edae88cb6c08474143b6b7b26d775a4e3c Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Wed, 19 Feb 2020 15:39:23 -0500 Subject: [PATCH 2/5] Add a test case to test parse exceptions from create_pattern_object(). --- stix2/test/v21/test_pattern_expressions.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/stix2/test/v21/test_pattern_expressions.py b/stix2/test/v21/test_pattern_expressions.py index 76880be..0c298f8 100644 --- a/stix2/test/v21/test_pattern_expressions.py +++ b/stix2/test/v21/test_pattern_expressions.py @@ -1,6 +1,7 @@ import datetime import pytest +from stix2patterns.exceptions import ParseException import stix2 from stix2.pattern_visitor import create_pattern_object @@ -515,3 +516,8 @@ def test_list_constant(): def test_parsing_multiple_slashes_quotes(): patt_obj = create_pattern_object("[ file:name = 'weird_name\\'' ]") assert str(patt_obj) == "[file:name = 'weird_name\\'']" + + +def test_parse_error(): + with pytest.raises(ParseException): + create_pattern_object("[ file: name = 'weirdname]") From 76a6eb58736e3ebd566914ca8e1b53a430a9e35d Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Wed, 19 Feb 2020 16:39:15 -0500 Subject: [PATCH 3/5] Greatly simplify the create_pattern_object() function to take advantage of the pattern validator library's Pattern.visit() method. --- stix2/pattern_visitor.py | 39 +++------------------------------------ 1 file changed, 3 insertions(+), 36 deletions(-) diff --git a/stix2/pattern_visitor.py b/stix2/pattern_visitor.py index e4601ba..571c66f 100644 --- a/stix2/pattern_visitor.py +++ b/stix2/pattern_visitor.py @@ -10,6 +10,7 @@ from stix2patterns.grammars.STIXPatternParser import ( STIXPatternParser, TerminalNode, ) from stix2patterns.grammars.STIXPatternVisitor import STIXPatternVisitor +from stix2patterns.v20.pattern import Pattern from .patterns import * from .patterns import _BooleanExpression @@ -330,40 +331,6 @@ def create_pattern_object(pattern, module_suffix="", module_name=""): Create a STIX pattern AST from a pattern string. """ - pattern = InputStream(pattern) - - parseErrListener = ParserErrorListener() - - lexer = STIXPatternLexer(pattern) - # it always adds a console listener by default... remove it. - lexer.removeErrorListeners() - - stream = CommonTokenStream(lexer) - - parser = STIXPatternParser(stream) - parser._errHandler = BailErrorStrategy() - - parser.buildParseTrees = True - # it always adds a console listener by default... remove it. - parser.removeErrorListeners() - parser.addErrorListener(parseErrListener) - - # To improve error messages, replace "" in the literal - # names with symbolic names. This is a hack, but seemed like - # the simplest workaround. - for i, lit_name in enumerate(parser.literalNames): - if lit_name == u"": - parser.literalNames[i] = parser.symbolicNames[i] - - try: - tree = parser.pattern() - except antlr4.error.Errors.ParseCancellationException as e: - real_exc = e.args[0] - parser._errHandler.reportError(parser, real_exc) - six.raise_from( - ParseException(parseErrListener.error_message), - real_exc, - ) - + pattern_obj = Pattern(pattern) builder = STIXPatternVisitorForSTIX2(module_suffix, module_name) - return builder.visit(tree) + return pattern_obj.visit(builder) From 1959cc609737ebf734768be8ec1b53104b0f7859 Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Wed, 19 Feb 2020 16:45:15 -0500 Subject: [PATCH 4/5] Removed a bunch of no-longer-used imports from pattern_visitor.py --- stix2/pattern_visitor.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/stix2/pattern_visitor.py b/stix2/pattern_visitor.py index 571c66f..6ac3e98 100644 --- a/stix2/pattern_visitor.py +++ b/stix2/pattern_visitor.py @@ -1,11 +1,7 @@ import importlib import inspect -from antlr4 import BailErrorStrategy, CommonTokenStream, InputStream -import antlr4.error.Errors -import six -from stix2patterns.exceptions import ParseException, ParserErrorListener -from stix2patterns.grammars.STIXPatternLexer import STIXPatternLexer +from stix2patterns.exceptions import ParseException from stix2patterns.grammars.STIXPatternParser import ( STIXPatternParser, TerminalNode, ) From 4f00c7ca4fe3088adb664b368d8c1c65ee323362 Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Wed, 4 Mar 2020 13:33:54 -0500 Subject: [PATCH 5/5] Fix patterning test --- stix2/test/v21/test_indicator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stix2/test/v21/test_indicator.py b/stix2/test/v21/test_indicator.py index 152f253..23bad29 100644 --- a/stix2/test/v21/test_indicator.py +++ b/stix2/test/v21/test_indicator.py @@ -271,7 +271,7 @@ def test_indicator_stix20_invalid_pattern(): ) assert excinfo.value.cls == stix2.v21.Indicator - assert "FAIL: The same qualifier is used more than once" in str(excinfo.value) + assert "FAIL: Duplicate qualifier type encountered" in str(excinfo.value) ind = stix2.v21.Indicator( type="indicator",