From 03cceb827d069f0226af6833b347e79fe18bca5d Mon Sep 17 00:00:00 2001 From: Richard Piazza Date: Fri, 7 Dec 2018 12:43:23 -0500 Subject: [PATCH 01/18] add_visitor - take 2 --- .gitignore | 28 ++ stix2/STIXPatternVisitor.py | 350 +++++++++++++++ stix2/patterns.py | 73 +-- .../20170531213258226477.json | 60 +-- .../20181101232448456000.json | 50 +-- .../20181101232448457000.json | 50 +-- stix2/test/test_pattern_expressions.py | 414 ++++++++++++------ 7 files changed, 783 insertions(+), 242 deletions(-) create mode 100644 stix2/STIXPatternVisitor.py diff --git a/.gitignore b/.gitignore index 9e12d7d..5534a28 100644 --- a/.gitignore +++ b/.gitignore @@ -68,3 +68,31 @@ cache.sqlite # PyCharm .idea/ +### macOS template +# General +.DS_Store +.AppleDouble +.LSOverride + +# Icon must end with two \r +Icon + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + diff --git a/stix2/STIXPatternVisitor.py b/stix2/STIXPatternVisitor.py new file mode 100644 index 0000000..5339395 --- /dev/null +++ b/stix2/STIXPatternVisitor.py @@ -0,0 +1,350 @@ +import importlib +import inspect + +import six +from stix2patterns.grammars.STIXPatternLexer import STIXPatternLexer +from stix2patterns.grammars.STIXPatternParser import ( + STIXPatternParser, TerminalNode, +) +from stix2patterns.grammars.STIXPatternVisitor import STIXPatternVisitor +from stix2patterns.validator import STIXPatternErrorListener +from stix2.patterns import _BooleanExpression + +from antlr4 import CommonTokenStream, InputStream + +# need to import all classes because we need to access them via globals() +from .patterns import * + +def collapse_lists(lists): + result = [] + for c in lists: + if isinstance(c, list): + result.extend(c) + else: + result.append(c) + return result + + +def values_only(things): + values = [] + for x in things: + if not isinstance(x, TerminalNode): + values.append(x) + return values + + + +# This class defines a complete generic visitor for a parse tree produced by STIXPatternParser. + + +class STIXPatternVisitorForSTIX2(STIXPatternVisitor): + classes = {} + + def __init__(self, module_suffix, module_name): + if module_suffix and module_name: + self.module_suffix = module_suffix + if STIXPatternVisitorForSTIX2.classes == {}: + module = importlib.import_module(module_name) + for k, c in inspect.getmembers(module, inspect.isclass): + STIXPatternVisitorForSTIX2.classes[k] = c + else: + self.module_suffix = None + super(STIXPatternVisitor, self).__init__() + + def get_class(self, class_name): + if class_name in STIXPatternVisitorForSTIX2.classes: + return STIXPatternVisitorForSTIX2.classes[class_name] + else: + return None + + def instantiate(self, klass_name, *args): + klass_to_instantiate = None + if self.module_suffix: + klass_to_instantiate = self.get_class(klass_name + "For" + self.module_suffix) + if not klass_to_instantiate: + # use the classes in python_stix2 + klass_to_instantiate = globals()[klass_name] + return klass_to_instantiate(*args) + + # Visit a parse tree produced by STIXPatternParser#pattern. + def visitPattern(self, ctx): + children = self.visitChildren(ctx) + return children[0] + + # Visit a parse tree produced by STIXPatternParser#observationExpressions. + def visitObservationExpressions(self, ctx): + children = self.visitChildren(ctx) + if len(children) == 1: + return children[0] + else: + return FollowedByObservationExpression([children[0], children[2]]) + + # Visit a parse tree produced by STIXPatternParser#observationExpressionOr. + def visitObservationExpressionOr(self, ctx): + children = self.visitChildren(ctx) + if len(children) == 1: + return children[0] + else: + return self.instantiate("OrObservationExpression", [children[0], children[2]]) + + # Visit a parse tree produced by STIXPatternParser#observationExpressionAnd. + def visitObservationExpressionAnd(self, ctx): + children = self.visitChildren(ctx) + if len(children) == 1: + return children[0] + else: + return self.instantiate("AndObservationExpression", [children[0], children[2]]) + + # Visit a parse tree produced by STIXPatternParser#observationExpressionRepeated. + def visitObservationExpressionRepeated(self, ctx): + children = self.visitChildren(ctx) + return self.instantiate("QualifiedObservationExpression", children[0], children[1]) + + # Visit a parse tree produced by STIXPatternParser#observationExpressionSimple. + def visitObservationExpressionSimple(self, ctx): + children = self.visitChildren(ctx) + return self.instantiate("ObservationExpression", children[1]) + + # Visit a parse tree produced by STIXPatternParser#observationExpressionCompound. + def visitObservationExpressionCompound(self, ctx): + children = self.visitChildren(ctx) + return self.instantiate("ObservationExpression", children[1]) + + # Visit a parse tree produced by STIXPatternParser#observationExpressionWithin. + def visitObservationExpressionWithin(self, ctx): + children = self.visitChildren(ctx) + return self.instantiate("QualifiedObservationExpression", children[0], children[1]) + + # Visit a parse tree produced by STIXPatternParser#observationExpressionStartStop. + def visitObservationExpressionStartStop(self, ctx): + children = self.visitChildren(ctx) + return self.instantiate("QualifiedObservationExpression", children[0], children[1]) + + # Visit a parse tree produced by STIXPatternParser#comparisonExpression. + def visitComparisonExpression(self, ctx): + children = self.visitChildren(ctx) + if len(children) == 1: + return children[0] + else: + if isinstance(children[0], _BooleanExpression): + children[0].operands.append(children[2]) + return children[0] + else: + return self.instantiate("OrBooleanExpression", [children[0], children[2]]) + + # Visit a parse tree produced by STIXPatternParser#comparisonExpressionAnd. + def visitComparisonExpressionAnd(self, ctx): + # TODO: NOT + children = self.visitChildren(ctx) + if len(children) == 1: + return children[0] + else: + if isinstance(children[0], _BooleanExpression): + children[0].operands.append(children[2]) + return children[0] + else: + return self.instantiate("AndBooleanExpression", [children[0], children[2]]) + + # Visit a parse tree produced by STIXPatternParser#propTestEqual. + def visitPropTestEqual(self, ctx): + children = self.visitChildren(ctx) + operator = children[1].symbol.type + negated = negated = operator != STIXPatternParser.EQ + return self.instantiate("EqualityComparisonExpression", children[0], children[3 if len(children) > 3 else 2], + negated) + + # Visit a parse tree produced by STIXPatternParser#propTestOrder. + def visitPropTestOrder(self, ctx): + children = self.visitChildren(ctx) + operator = children[1].symbol.type + if operator == STIXPatternParser.GT: + return self.instantiate("GreaterThanComparisonExpression", children[0], + children[3 if len(children) > 3 else 2], False) + elif operator == STIXPatternParser.LT: + return self.instantiate("LessThanComparisonExpression", children[0], + children[3 if len(children) > 3 else 2], False) + elif operator == STIXPatternParser.GE: + return self.instantiate("GreaterThanEqualComparisonExpression", children[0], + children[3 if len(children) > 3 else 2], False) + elif operator == STIXPatternParser.LE: + return self.instantiate("LessThanEqualComparisonExpression", children[0], + children[3 if len(children) > 3 else 2], False) + + # Visit a parse tree produced by STIXPatternParser#propTestSet. + def visitPropTestSet(self, ctx): + children = self.visitChildren(ctx) + return self.instantiate("InComparisonExpression", children[0], children[3 if len(children) > 3 else 2], False) + + # Visit a parse tree produced by STIXPatternParser#propTestLike. + def visitPropTestLike(self, ctx): + children = self.visitChildren(ctx) + return self.instantiate("LikeComparisonExpression", children[0], children[3 if len(children) > 3 else 2], False) + + # Visit a parse tree produced by STIXPatternParser#propTestRegex. + def visitPropTestRegex(self, ctx): + children = self.visitChildren(ctx) + return self.instantiate("MatchesComparisonExpression", children[0], children[3 if len(children) > 3 else 2], + False) + + # Visit a parse tree produced by STIXPatternParser#propTestIsSubset. + def visitPropTestIsSubset(self, ctx): + children = self.visitChildren(ctx) + return self.instantiate("IsSubsetComparisonExpression", children[0], children[3 if len(children) > 3 else 2]) + + # Visit a parse tree produced by STIXPatternParser#propTestIsSuperset. + def visitPropTestIsSuperset(self, ctx): + children = self.visitChildren(ctx) + return self.instantiate("IsSupersetComparisonExpression", children[0], children[3 if len(children) > 3 else 2]) + + # Visit a parse tree produced by STIXPatternParser#propTestParen. + def visitPropTestParen(self, ctx): + children = self.visitChildren(ctx) + return self.instantiate("ParentheticalExpression", children[1]) + + # Visit a parse tree produced by STIXPatternParser#startStopQualifier. + def visitStartStopQualifier(self, ctx): + children = self.visitChildren(ctx) + return StartStopQualifier(children[1], children[3]) + + # Visit a parse tree produced by STIXPatternParser#withinQualifier. + def visitWithinQualifier(self, ctx): + children = self.visitChildren(ctx) + return WithinQualifier(children[1]) + + # Visit a parse tree produced by STIXPatternParser#repeatedQualifier. + def visitRepeatedQualifier(self, ctx): + children = self.visitChildren(ctx) + return RepeatQualifier(children[1]) + + # Visit a parse tree produced by STIXPatternParser#objectPath. + def visitObjectPath(self, ctx): + children = self.visitChildren(ctx) + flat_list = collapse_lists(children[2:]) + property_path = [] + i = 0 + while i < len(flat_list): + current = flat_list[i] + if i == len(flat_list)-1: + property_path.append(current) + break + next = flat_list[i+1] + if isinstance(next, TerminalNode): + property_path.append(self.instantiate("ListObjectPathComponent", current.property_name, next.getText())) + i += 2 + else: + property_path.append(current) + i += 1 + return self.instantiate("ObjectPath", children[0].getText(), property_path) + + # Visit a parse tree produced by STIXPatternParser#objectType. + def visitObjectType(self, ctx): + children = self.visitChildren(ctx) + return children[0] + + # Visit a parse tree produced by STIXPatternParser#firstPathComponent. + def visitFirstPathComponent(self, ctx): + children = self.visitChildren(ctx) + step = children[0].getText() + # if step.endswith("_ref"): + # return stix2.ReferenceObjectPathComponent(step) + # else: + return self.instantiate("BasicObjectPathComponent", step, False) + + # Visit a parse tree produced by STIXPatternParser#indexPathStep. + def visitIndexPathStep(self, ctx): + children = self.visitChildren(ctx) + return children[1] + + # Visit a parse tree produced by STIXPatternParser#pathStep. + def visitPathStep(self, ctx): + return collapse_lists(self.visitChildren(ctx)) + + # Visit a parse tree produced by STIXPatternParser#keyPathStep. + def visitKeyPathStep(self, ctx): + children = self.visitChildren(ctx) + if isinstance(children[1], StringConstant): + # special case for hashes + return children[1].value + else: + return self.instantiate("BasicObjectPathComponent", children[1].getText(), True) + + # Visit a parse tree produced by STIXPatternParser#setLiteral. + def visitSetLiteral(self, ctx): + children = self.visitChildren(ctx) + return self.instantiate("ListConstant", values_only(children)) + + # Visit a parse tree produced by STIXPatternParser#primitiveLiteral. + def visitPrimitiveLiteral(self, ctx): + children = self.visitChildren(ctx) + return children[0] + + # Visit a parse tree produced by STIXPatternParser#orderableLiteral. + def visitOrderableLiteral(self, ctx): + children = self.visitChildren(ctx) + return children[0] + + def visitTerminal(self, node): + if node.symbol.type == STIXPatternParser.IntPosLiteral or node.symbol.type == STIXPatternParser.IntNegLiteral: + return IntegerConstant(node.getText()) + elif node.symbol.type == STIXPatternParser.FloatPosLiteral or node.symbol.type == STIXPatternParser.FloatNegLiteral: + return FloatConstant(node.getText()) + elif node.symbol.type == STIXPatternParser.HexLiteral: + return HexConstant(node.getText(), from_parse_tree=True) + elif node.symbol.type == STIXPatternParser.BinaryLiteral: + return BinaryConstant(node.getText(), from_parse_tree=True) + elif node.symbol.type == STIXPatternParser.StringLiteral: + return StringConstant(node.getText().strip('\''), from_parse_tree=True) + elif node.symbol.type == STIXPatternParser.BoolLiteral: + return BooleanConstant(node.getText()) + elif node.symbol.type == STIXPatternParser.TimestampLiteral: + return TimestampConstant(node.getText()) + else: + return node + + def aggregateResult(self, aggregate, nextResult): + if aggregate: + aggregate.append(nextResult) + elif nextResult: + aggregate = [nextResult] + return aggregate + + +def create_pattern_object(pattern, module_suffix="", module_name=""): + ''' + Validates a pattern against the STIX Pattern grammar. Error messages are + returned in a list. The test passed if the returned list is empty. + ''' + + start = '' + if isinstance(pattern, six.string_types): + start = pattern[:2] + pattern = InputStream(pattern) + + if not start: + start = pattern.readline()[:2] + pattern.seek(0) + + parseErrListener = STIXPatternErrorListener() + + lexer = STIXPatternLexer(pattern) + # it always adds a console listener by default... remove it. + lexer.removeErrorListeners() + + stream = CommonTokenStream(lexer) + + parser = STIXPatternParser(stream) + parser.buildParseTrees = True + # it always adds a console listener by default... remove it. + parser.removeErrorListeners() + parser.addErrorListener(parseErrListener) + + # To improve error messages, replace "" in the literal + # names with symbolic names. This is a hack, but seemed like + # the simplest workaround. + for i, lit_name in enumerate(parser.literalNames): + if lit_name == u"": + parser.literalNames[i] = parser.symbolicNames[i] + + tree = parser.pattern() + builder = STIXPatternVisitorForSTIX2(module_suffix, module_name) + return builder.visit(tree) diff --git a/stix2/patterns.py b/stix2/patterns.py index 146ec04..4828b07 100644 --- a/stix2/patterns.py +++ b/stix2/patterns.py @@ -13,6 +13,14 @@ def escape_quotes_and_backslashes(s): return s.replace(u'\\', u'\\\\').replace(u"'", u"\\'") +def quote_if_needed(x): + if isinstance(x, str): + if x.find("-") != -1: + if not x.startswith("'"): + return "'" + x + "'" + return x + + class _Constant(object): pass @@ -23,11 +31,13 @@ class StringConstant(_Constant): Args: value (str): string value """ - def __init__(self, value): + + def __init__(self, value, from_parse_tree=False): + self.needs_to_be_quoted = not from_parse_tree self.value = value def __str__(self): - return "'%s'" % escape_quotes_and_backslashes(self.value) + return "'%s'" % (escape_quotes_and_backslashes(self.value) if self.needs_to_be_quoted else self.value) class TimestampConstant(_Constant): @@ -86,8 +96,8 @@ class BooleanConstant(_Constant): self.value = value return - trues = ['true', 't'] - falses = ['false', 'f'] + trues = ['true', 't', '1'] + falses = ['false', 'f', '0'] try: if value.lower() in trues: self.value = True @@ -143,7 +153,7 @@ class HashConstant(StringConstant): vocab_key = _HASH_REGEX[key][1] if not re.match(_HASH_REGEX[key][0], value): raise ValueError("'%s' is not a valid %s hash" % (value, vocab_key)) - self.value = value + super(HashConstant, self).__init__(value) class BinaryConstant(_Constant): @@ -152,7 +162,13 @@ class BinaryConstant(_Constant): Args: value (str): base64 encoded string value """ - def __init__(self, value): + + def __init__(self, value, from_parse_tree=False): + # support with or without a 'b' + if from_parse_tree: + m = re.match("^b'(.+)'$", value) + if m: + value = m.group(1) try: base64.b64decode(value) self.value = value @@ -169,10 +185,17 @@ class HexConstant(_Constant): Args: value (str): hexadecimal value """ - def __init__(self, value): - if not re.match('^([a-fA-F0-9]{2})+$', value): - raise ValueError("must contain an even number of hexadecimal characters") - self.value = value + + def __init__(self, value, from_parse_tree=False): + # support with or without an 'h' + if not from_parse_tree and re.match('^([a-fA-F0-9]{2})+$', value): + self.value = value + else: + m = re.match("^h'(([a-fA-F0-9]{2})+)'$", value) + if m: + self.value = m.group(1) + else: + raise ValueError("must contain an even number of hexadecimal characters") def __str__(self): return "h'%s'" % self.value @@ -185,10 +208,11 @@ class ListConstant(_Constant): value (list): list of values """ def __init__(self, values): - self.value = values + # handle _Constants or make a _Constant + self.value = [x if isinstance(x, _Constant) else make_constant(x) for x in values] def __str__(self): - return "(" + ", ".join([("%s" % make_constant(x)) for x in self.value]) + ")" + return "(" + ", ".join(["%s" % x for x in self.value]) + ")" def make_constant(value): @@ -229,7 +253,10 @@ class _ObjectPathComponent(object): parse1 = component_name.split("[") return ListObjectPathComponent(parse1[0], parse1[1][:-1]) else: - return BasicObjectPathComponent(component_name) + return BasicObjectPathComponent(component_name, False) + + def __str__(self): + return quote_if_needed(self.property_name) class BasicObjectPathComponent(_ObjectPathComponent): @@ -243,14 +270,11 @@ class BasicObjectPathComponent(_ObjectPathComponent): property_name (str): object property name is_key (bool): is dictionary key, default: False """ - def __init__(self, property_name, is_key=False): + def __init__(self, property_name, is_key): self.property_name = property_name # TODO: set is_key to True if this component is a dictionary key # self.is_key = is_key - def __str__(self): - return self.property_name - class ListObjectPathComponent(_ObjectPathComponent): """List object path component (for an observation or expression) @@ -264,7 +288,7 @@ class ListObjectPathComponent(_ObjectPathComponent): self.index = index def __str__(self): - return "%s[%s]" % (self.property_name, self.index) + return "%s[%s]" % (quote_if_needed(self.property_name), self.index) class ReferenceObjectPathComponent(_ObjectPathComponent): @@ -276,9 +300,6 @@ class ReferenceObjectPathComponent(_ObjectPathComponent): def __init__(self, reference_property_name): self.property_name = reference_property_name - def __str__(self): - return self.property_name - class ObjectPath(object): """Pattern operand object (property) path @@ -289,12 +310,14 @@ class ObjectPath(object): """ def __init__(self, object_type_name, property_path): self.object_type_name = object_type_name - self.property_path = [x if isinstance(x, _ObjectPathComponent) else - _ObjectPathComponent.create_ObjectPathComponent(x) - for x in property_path] + self.property_path = [ + x if isinstance(x, _ObjectPathComponent) else + _ObjectPathComponent.create_ObjectPathComponent(x) + for x in property_path + ] def __str__(self): - return "%s:%s" % (self.object_type_name, ".".join(["%s" % x for x in self.property_path])) + return "%s:%s" % (self.object_type_name, ".".join(["%s" % quote_if_needed(x) for x in self.property_path])) def merge(self, other): """Extend the object property with that of the supplied object property path""" diff --git a/stix2/test/stix2_data/malware/malware--6b616fc1-1505-48e3-8b2c-0d19337bff38/20170531213258226477.json b/stix2/test/stix2_data/malware/malware--6b616fc1-1505-48e3-8b2c-0d19337bff38/20170531213258226477.json index c60200b..4c03007 100644 --- a/stix2/test/stix2_data/malware/malware--6b616fc1-1505-48e3-8b2c-0d19337bff38/20170531213258226477.json +++ b/stix2/test/stix2_data/malware/malware--6b616fc1-1505-48e3-8b2c-0d19337bff38/20170531213258226477.json @@ -1,34 +1,34 @@ { - "id": "bundle--f64de948-7067-4534-8018-85f03d470625", - "objects": [ + "id": "bundle--f64de948-7067-4534-8018-85f03d470625", + "objects": [ + { + "created": "2017-05-31T21:32:58.226477Z", + "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5", + "description": "Rover is malware suspected of being used for espionage purposes. It was used in 2015 in a targeted email sent to an Indian Ambassador to Afghanistan.[[Citation: Palo Alto Rover]]", + "external_references": [ { - "created": "2017-05-31T21:32:58.226477Z", - "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5", - "description": "Rover is malware suspected of being used for espionage purposes. It was used in 2015 in a targeted email sent to an Indian Ambassador to Afghanistan.[[Citation: Palo Alto Rover]]", - "external_references": [ - { - "external_id": "S0090", - "source_name": "mitre-attack", - "url": "https://attack.mitre.org/wiki/Software/S0090" - }, - { - "description": "Ray, V., Hayashi, K. (2016, February 29). New Malware \u2018Rover\u2019 Targets Indian Ambassador to Afghanistan. Retrieved February 29, 2016.", - "source_name": "Palo Alto Rover", - "url": "http://researchcenter.paloaltonetworks.com/2016/02/new-malware-rover-targets-indian-ambassador-to-afghanistan/" - } - ], - "id": "malware--6b616fc1-1505-48e3-8b2c-0d19337bff38", - "labels": [ - "malware" - ], - "modified": "2017-05-31T21:32:58.226477Z", - "name": "Rover", - "object_marking_refs": [ - "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168" - ], - "type": "malware" + "external_id": "S0090", + "source_name": "mitre-attack", + "url": "https://attack.mitre.org/wiki/Software/S0090" + }, + { + "description": "Ray, V., Hayashi, K. (2016, February 29). New Malware \u2018Rover\u2019 Targets Indian Ambassador to Afghanistan. Retrieved February 29, 2016.", + "source_name": "Palo Alto Rover", + "url": "http://researchcenter.paloaltonetworks.com/2016/02/new-malware-rover-targets-indian-ambassador-to-afghanistan/" } - ], - "spec_version": "2.0", - "type": "bundle" + ], + "id": "malware--6b616fc1-1505-48e3-8b2c-0d19337bff38", + "labels": [ + "malware" + ], + "modified": "2017-05-31T21:32:58.226477Z", + "name": "Rover", + "object_marking_refs": [ + "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168" + ], + "type": "malware" + } + ], + "spec_version": "2.0", + "type": "bundle" } diff --git a/stix2/test/stix2_data/malware/malware--6b616fc1-1505-48e3-8b2c-0d19337bff38/20181101232448456000.json b/stix2/test/stix2_data/malware/malware--6b616fc1-1505-48e3-8b2c-0d19337bff38/20181101232448456000.json index af47f27..1016dec 100644 --- a/stix2/test/stix2_data/malware/malware--6b616fc1-1505-48e3-8b2c-0d19337bff38/20181101232448456000.json +++ b/stix2/test/stix2_data/malware/malware--6b616fc1-1505-48e3-8b2c-0d19337bff38/20181101232448456000.json @@ -1,27 +1,27 @@ { - "type": "malware", - "id": "malware--6b616fc1-1505-48e3-8b2c-0d19337bff38", - "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5", - "created": "2017-05-31T21:32:58.226Z", - "modified": "2018-11-01T23:24:48.456Z", - "name": "Rover", - "description": "Rover is malware suspected of being used for espionage purposes. It was used in 2015 in a targeted email sent to an Indian Ambassador to Afghanistan.[[Citation: Palo Alto Rover]]", - "labels": [ - "version two" - ], - "external_references": [ - { - "source_name": "mitre-attack", - "url": "https://attack.mitre.org/wiki/Software/S0090", - "external_id": "S0090" - }, - { - "source_name": "Palo Alto Rover", - "description": "Ray, V., Hayashi, K. (2016, February 29). New Malware \u2018Rover\u2019 Targets Indian Ambassador to Afghanistan. Retrieved February 29, 2016.", - "url": "http://researchcenter.paloaltonetworks.com/2016/02/new-malware-rover-targets-indian-ambassador-to-afghanistan/" - } - ], - "object_marking_refs": [ - "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168" - ] + "type": "malware", + "id": "malware--6b616fc1-1505-48e3-8b2c-0d19337bff38", + "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5", + "created": "2017-05-31T21:32:58.226Z", + "modified": "2018-11-01T23:24:48.456Z", + "name": "Rover", + "description": "Rover is malware suspected of being used for espionage purposes. It was used in 2015 in a targeted email sent to an Indian Ambassador to Afghanistan.[[Citation: Palo Alto Rover]]", + "labels": [ + "version two" + ], + "external_references": [ + { + "source_name": "mitre-attack", + "url": "https://attack.mitre.org/wiki/Software/S0090", + "external_id": "S0090" + }, + { + "source_name": "Palo Alto Rover", + "description": "Ray, V., Hayashi, K. (2016, February 29). New Malware \u2018Rover\u2019 Targets Indian Ambassador to Afghanistan. Retrieved February 29, 2016.", + "url": "http://researchcenter.paloaltonetworks.com/2016/02/new-malware-rover-targets-indian-ambassador-to-afghanistan/" + } + ], + "object_marking_refs": [ + "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168" + ] } diff --git a/stix2/test/stix2_data/malware/malware--6b616fc1-1505-48e3-8b2c-0d19337bff38/20181101232448457000.json b/stix2/test/stix2_data/malware/malware--6b616fc1-1505-48e3-8b2c-0d19337bff38/20181101232448457000.json index 446fb26..e7328f8 100644 --- a/stix2/test/stix2_data/malware/malware--6b616fc1-1505-48e3-8b2c-0d19337bff38/20181101232448457000.json +++ b/stix2/test/stix2_data/malware/malware--6b616fc1-1505-48e3-8b2c-0d19337bff38/20181101232448457000.json @@ -1,27 +1,27 @@ { - "type": "malware", - "id": "malware--6b616fc1-1505-48e3-8b2c-0d19337bff38", - "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5", - "created": "2017-05-31T21:32:58.226Z", - "modified": "2018-11-01T23:24:48.457Z", - "name": "Rover", - "description": "Rover is malware suspected of being used for espionage purposes. It was used in 2015 in a targeted email sent to an Indian Ambassador to Afghanistan.[[Citation: Palo Alto Rover]]", - "labels": [ - "version three" - ], - "external_references": [ - { - "source_name": "mitre-attack", - "url": "https://attack.mitre.org/wiki/Software/S0090", - "external_id": "S0090" - }, - { - "source_name": "Palo Alto Rover", - "description": "Ray, V., Hayashi, K. (2016, February 29). New Malware \u2018Rover\u2019 Targets Indian Ambassador to Afghanistan. Retrieved February 29, 2016.", - "url": "http://researchcenter.paloaltonetworks.com/2016/02/new-malware-rover-targets-indian-ambassador-to-afghanistan/" - } - ], - "object_marking_refs": [ - "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168" - ] + "type": "malware", + "id": "malware--6b616fc1-1505-48e3-8b2c-0d19337bff38", + "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5", + "created": "2017-05-31T21:32:58.226Z", + "modified": "2018-11-01T23:24:48.457Z", + "name": "Rover", + "description": "Rover is malware suspected of being used for espionage purposes. It was used in 2015 in a targeted email sent to an Indian Ambassador to Afghanistan.[[Citation: Palo Alto Rover]]", + "labels": [ + "version three" + ], + "external_references": [ + { + "source_name": "mitre-attack", + "url": "https://attack.mitre.org/wiki/Software/S0090", + "external_id": "S0090" + }, + { + "source_name": "Palo Alto Rover", + "description": "Ray, V., Hayashi, K. (2016, February 29). New Malware \u2018Rover\u2019 Targets Indian Ambassador to Afghanistan. Retrieved February 29, 2016.", + "url": "http://researchcenter.paloaltonetworks.com/2016/02/new-malware-rover-targets-indian-ambassador-to-afghanistan/" + } + ], + "object_marking_refs": [ + "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168" + ] } diff --git a/stix2/test/test_pattern_expressions.py b/stix2/test/test_pattern_expressions.py index a4d0a5c..c136eaa 100644 --- a/stix2/test/test_pattern_expressions.py +++ b/stix2/test/test_pattern_expressions.py @@ -4,42 +4,61 @@ import pytest import stix2 +from stix2.STIXPatternVisitor import create_pattern_object + def test_create_comparison_expression(): - - exp = stix2.EqualityComparisonExpression("file:hashes.'SHA-256'", - stix2.HashConstant("aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f", "SHA-256")) # noqa + exp = stix2.EqualityComparisonExpression( + "file:hashes.'SHA-256'", + stix2.HashConstant("aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f", "SHA-256"), + ) # noqa assert str(exp) == "file:hashes.'SHA-256' = 'aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f'" def test_boolean_expression(): - exp1 = stix2.MatchesComparisonExpression("email-message:from_ref.value", - stix2.StringConstant(".+\\@example\\.com$")) - exp2 = stix2.MatchesComparisonExpression("email-message:body_multipart[*].body_raw_ref.name", - stix2.StringConstant("^Final Report.+\\.exe$")) + exp1 = stix2.MatchesComparisonExpression( + "email-message:from_ref.value", + stix2.StringConstant(".+\\@example\\.com$"), + ) + exp2 = stix2.MatchesComparisonExpression( + "email-message:body_multipart[*].body_raw_ref.name", + stix2.StringConstant("^Final Report.+\\.exe$"), + ) exp = stix2.AndBooleanExpression([exp1, exp2]) assert str(exp) == "email-message:from_ref.value MATCHES '.+\\\\@example\\\\.com$' AND email-message:body_multipart[*].body_raw_ref.name MATCHES '^Final Report.+\\\\.exe$'" # noqa def test_boolean_expression_with_parentheses(): - exp1 = stix2.MatchesComparisonExpression(stix2.ObjectPath("email-message", - [stix2.ReferenceObjectPathComponent("from_ref"), - stix2.BasicObjectPathComponent("value")]), - stix2.StringConstant(".+\\@example\\.com$")) - exp2 = stix2.MatchesComparisonExpression("email-message:body_multipart[*].body_raw_ref.name", - stix2.StringConstant("^Final Report.+\\.exe$")) + exp1 = stix2.MatchesComparisonExpression( + stix2.ObjectPath( + "email-message", + [ + stix2.ReferenceObjectPathComponent("from_ref"), + stix2.BasicObjectPathComponent("value", False), + ], + ), + stix2.StringConstant(".+\\@example\\.com$"), + ) + exp2 = stix2.MatchesComparisonExpression( + "email-message:body_multipart[*].body_raw_ref.name", + stix2.StringConstant("^Final Report.+\\.exe$"), + ) exp = stix2.ParentheticalExpression(stix2.AndBooleanExpression([exp1, exp2])) assert str(exp) == "(email-message:from_ref.value MATCHES '.+\\\\@example\\\\.com$' AND email-message:body_multipart[*].body_raw_ref.name MATCHES '^Final Report.+\\\\.exe$')" # noqa def test_hash_followed_by_registryKey_expression_python_constant(): - hash_exp = stix2.EqualityComparisonExpression("file:hashes.MD5", - stix2.HashConstant("79054025255fb1a26e4bc422aef54eb4", "MD5")) + hash_exp = stix2.EqualityComparisonExpression( + "file:hashes.MD5", + stix2.HashConstant("79054025255fb1a26e4bc422aef54eb4", "MD5"), + ) o_exp1 = stix2.ObservationExpression(hash_exp) - reg_exp = stix2.EqualityComparisonExpression(stix2.ObjectPath("windows-registry-key", ["key"]), - stix2.StringConstant("HKEY_LOCAL_MACHINE\\foo\\bar")) + reg_exp = stix2.EqualityComparisonExpression( + stix2.ObjectPath("windows-registry-key", ["key"]), + stix2.StringConstant("HKEY_LOCAL_MACHINE\\foo\\bar"), + ) o_exp2 = stix2.ObservationExpression(reg_exp) fb_exp = stix2.FollowedByObservationExpression([o_exp1, o_exp2]) para_exp = stix2.ParentheticalExpression(fb_exp) @@ -49,11 +68,15 @@ def test_hash_followed_by_registryKey_expression_python_constant(): def test_hash_followed_by_registryKey_expression(): - hash_exp = stix2.EqualityComparisonExpression("file:hashes.MD5", - stix2.HashConstant("79054025255fb1a26e4bc422aef54eb4", "MD5")) + hash_exp = stix2.EqualityComparisonExpression( + "file:hashes.MD5", + stix2.HashConstant("79054025255fb1a26e4bc422aef54eb4", "MD5"), + ) o_exp1 = stix2.ObservationExpression(hash_exp) - reg_exp = stix2.EqualityComparisonExpression(stix2.ObjectPath("windows-registry-key", ["key"]), - stix2.StringConstant("HKEY_LOCAL_MACHINE\\foo\\bar")) + reg_exp = stix2.EqualityComparisonExpression( + stix2.ObjectPath("windows-registry-key", ["key"]), + stix2.StringConstant("HKEY_LOCAL_MACHINE\\foo\\bar"), + ) o_exp2 = stix2.ObservationExpression(reg_exp) fb_exp = stix2.FollowedByObservationExpression([o_exp1, o_exp2]) para_exp = stix2.ParentheticalExpression(fb_exp) @@ -63,31 +86,44 @@ def test_hash_followed_by_registryKey_expression(): def test_file_observable_expression(): - exp1 = stix2.EqualityComparisonExpression("file:hashes.'SHA-256'", - stix2.HashConstant( - "aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f", - 'SHA-256')) + exp1 = stix2.EqualityComparisonExpression( + "file:hashes.'SHA-256'", + stix2.HashConstant( + "aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f", + 'SHA-256', + ), + ) exp2 = stix2.EqualityComparisonExpression("file:mime_type", stix2.StringConstant("application/x-pdf")) bool_exp = stix2.ObservationExpression(stix2.AndBooleanExpression([exp1, exp2])) assert str(bool_exp) == "[file:hashes.'SHA-256' = 'aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f' AND file:mime_type = 'application/x-pdf']" # noqa -@pytest.mark.parametrize("observation_class, op", [ - (stix2.AndObservationExpression, 'AND'), - (stix2.OrObservationExpression, 'OR'), -]) +@pytest.mark.parametrize( + "observation_class, op", [ + (stix2.AndObservationExpression, 'AND'), + (stix2.OrObservationExpression, 'OR'), + ], +) def test_multiple_file_observable_expression(observation_class, op): - exp1 = stix2.EqualityComparisonExpression("file:hashes.'SHA-256'", - stix2.HashConstant( - "bf07a7fbb825fc0aae7bf4a1177b2b31fcf8a3feeaf7092761e18c859ee52a9c", - 'SHA-256')) - exp2 = stix2.EqualityComparisonExpression("file:hashes.MD5", - stix2.HashConstant("cead3f77f6cda6ec00f57d76c9a6879f", "MD5")) + exp1 = stix2.EqualityComparisonExpression( + "file:hashes.'SHA-256'", + stix2.HashConstant( + "bf07a7fbb825fc0aae7bf4a1177b2b31fcf8a3feeaf7092761e18c859ee52a9c", + 'SHA-256', + ), + ) + exp2 = stix2.EqualityComparisonExpression( + "file:hashes.MD5", + stix2.HashConstant("cead3f77f6cda6ec00f57d76c9a6879f", "MD5"), + ) bool1_exp = stix2.OrBooleanExpression([exp1, exp2]) - exp3 = stix2.EqualityComparisonExpression("file:hashes.'SHA-256'", - stix2.HashConstant( - "aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f", - 'SHA-256')) + exp3 = stix2.EqualityComparisonExpression( + "file:hashes.'SHA-256'", + stix2.HashConstant( + "aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f", + 'SHA-256', + ), + ) op1_exp = stix2.ObservationExpression(bool1_exp) op2_exp = stix2.ObservationExpression(exp3) exp = observation_class([op1_exp, op2_exp]) @@ -97,34 +133,46 @@ def test_multiple_file_observable_expression(observation_class, op): def test_root_types(): ast = stix2.ObservationExpression( stix2.AndBooleanExpression( - [stix2.ParentheticalExpression( - stix2.OrBooleanExpression([ - stix2.EqualityComparisonExpression("a:b", stix2.StringConstant("1")), - stix2.EqualityComparisonExpression("b:c", stix2.StringConstant("2"))])), - stix2.EqualityComparisonExpression(u"b:d", stix2.StringConstant("3"))])) + [ + stix2.ParentheticalExpression( + stix2.OrBooleanExpression([ + stix2.EqualityComparisonExpression("a:b", stix2.StringConstant("1")), + stix2.EqualityComparisonExpression("b:c", stix2.StringConstant("2")), + ]), + ), + stix2.EqualityComparisonExpression(u"b:d", stix2.StringConstant("3")), + ], + ), + ) assert str(ast) == "[(a:b = '1' OR b:c = '2') AND b:d = '3']" def test_artifact_payload(): - exp1 = stix2.EqualityComparisonExpression("artifact:mime_type", - "application/vnd.tcpdump.pcap") - exp2 = stix2.MatchesComparisonExpression("artifact:payload_bin", - stix2.StringConstant("\\xd4\\xc3\\xb2\\xa1\\x02\\x00\\x04\\x00")) + exp1 = stix2.EqualityComparisonExpression( + "artifact:mime_type", + "application/vnd.tcpdump.pcap", + ) + exp2 = stix2.MatchesComparisonExpression( + "artifact:payload_bin", + stix2.StringConstant("\\xd4\\xc3\\xb2\\xa1\\x02\\x00\\x04\\x00"), + ) and_exp = stix2.ObservationExpression(stix2.AndBooleanExpression([exp1, exp2])) assert str(and_exp) == "[artifact:mime_type = 'application/vnd.tcpdump.pcap' AND artifact:payload_bin MATCHES '\\\\xd4\\\\xc3\\\\xb2\\\\xa1\\\\x02\\\\x00\\\\x04\\\\x00']" # noqa def test_greater_than_python_constant(): - exp1 = stix2.GreaterThanComparisonExpression("file:extensions.windows-pebinary-ext.sections[*].entropy", 7.0) + exp1 = stix2.GreaterThanComparisonExpression("file:extensions.'windows-pebinary-ext'.sections[*].entropy", 7.0) exp = stix2.ObservationExpression(exp1) - assert str(exp) == "[file:extensions.windows-pebinary-ext.sections[*].entropy > 7.0]" + assert str(exp) == "[file:extensions.'windows-pebinary-ext'.sections[*].entropy > 7.0]" def test_greater_than(): - exp1 = stix2.GreaterThanComparisonExpression("file:extensions.windows-pebinary-ext.sections[*].entropy", - stix2.FloatConstant(7.0)) + exp1 = stix2.GreaterThanComparisonExpression( + "file:extensions.'windows-pebinary-ext'.sections[*].entropy", + stix2.FloatConstant(7.0), + ) exp = stix2.ObservationExpression(exp1) - assert str(exp) == "[file:extensions.windows-pebinary-ext.sections[*].entropy > 7.0]" + assert str(exp) == "[file:extensions.'windows-pebinary-ext'.sections[*].entropy > 7.0]" def test_less_than(): @@ -133,73 +181,123 @@ def test_less_than(): def test_greater_than_or_equal(): - exp = stix2.GreaterThanEqualComparisonExpression("file:size", - 1024) + exp = stix2.GreaterThanEqualComparisonExpression( + "file:size", + 1024, + ) assert str(exp) == "file:size >= 1024" def test_less_than_or_equal(): - exp = stix2.LessThanEqualComparisonExpression("file:size", - 1024) + exp = stix2.LessThanEqualComparisonExpression( + "file:size", + 1024, + ) assert str(exp) == "file:size <= 1024" def test_not(): - exp = stix2.LessThanComparisonExpression("file:size", - 1024, - negated=True) + exp = stix2.LessThanComparisonExpression( + "file:size", + 1024, + negated=True, + ) assert str(exp) == "file:size NOT < 1024" def test_and_observable_expression(): - exp1 = stix2.AndBooleanExpression([stix2.EqualityComparisonExpression("user-account:account_type", - "unix"), - stix2.EqualityComparisonExpression("user-account:user_id", - stix2.StringConstant("1007")), - stix2.EqualityComparisonExpression("user-account:account_login", - "Peter")]) - exp2 = stix2.AndBooleanExpression([stix2.EqualityComparisonExpression("user-account:account_type", - "unix"), - stix2.EqualityComparisonExpression("user-account:user_id", - stix2.StringConstant("1008")), - stix2.EqualityComparisonExpression("user-account:account_login", - "Paul")]) - exp3 = stix2.AndBooleanExpression([stix2.EqualityComparisonExpression("user-account:account_type", - "unix"), - stix2.EqualityComparisonExpression("user-account:user_id", - stix2.StringConstant("1009")), - stix2.EqualityComparisonExpression("user-account:account_login", - "Mary")]) - exp = stix2.AndObservationExpression([stix2.ObservationExpression(exp1), - stix2.ObservationExpression(exp2), - stix2.ObservationExpression(exp3)]) + exp1 = stix2.AndBooleanExpression([ + stix2.EqualityComparisonExpression( + "user-account:account_type", + "unix", + ), + stix2.EqualityComparisonExpression( + "user-account:user_id", + stix2.StringConstant("1007"), + ), + stix2.EqualityComparisonExpression( + "user-account:account_login", + "Peter", + ), + ]) + exp2 = stix2.AndBooleanExpression([ + stix2.EqualityComparisonExpression( + "user-account:account_type", + "unix", + ), + stix2.EqualityComparisonExpression( + "user-account:user_id", + stix2.StringConstant("1008"), + ), + stix2.EqualityComparisonExpression( + "user-account:account_login", + "Paul", + ), + ]) + exp3 = stix2.AndBooleanExpression([ + stix2.EqualityComparisonExpression( + "user-account:account_type", + "unix", + ), + stix2.EqualityComparisonExpression( + "user-account:user_id", + stix2.StringConstant("1009"), + ), + stix2.EqualityComparisonExpression( + "user-account:account_login", + "Mary", + ), + ]) + exp = stix2.AndObservationExpression([ + stix2.ObservationExpression(exp1), + stix2.ObservationExpression(exp2), + stix2.ObservationExpression(exp3), + ]) assert str(exp) == "[user-account:account_type = 'unix' AND user-account:user_id = '1007' AND user-account:account_login = 'Peter'] AND [user-account:account_type = 'unix' AND user-account:user_id = '1008' AND user-account:account_login = 'Paul'] AND [user-account:account_type = 'unix' AND user-account:user_id = '1009' AND user-account:account_login = 'Mary']" # noqa def test_invalid_and_observable_expression(): with pytest.raises(ValueError) as excinfo: - stix2.AndBooleanExpression([stix2.EqualityComparisonExpression("user-account:display_name", - "admin"), - stix2.EqualityComparisonExpression("email-addr:display_name", - stix2.StringConstant("admin"))]) + stix2.AndBooleanExpression([ + stix2.EqualityComparisonExpression( + "user-account:display_name", + "admin", + ), + stix2.EqualityComparisonExpression( + "email-addr:display_name", + stix2.StringConstant("admin"), + ), + ]) assert "All operands to an 'AND' expression must have the same object type" in str(excinfo) def test_hex(): - exp_and = stix2.AndBooleanExpression([stix2.EqualityComparisonExpression("file:mime_type", - "image/bmp"), - stix2.EqualityComparisonExpression("file:magic_number_hex", - stix2.HexConstant("ffd8"))]) + exp_and = stix2.AndBooleanExpression([ + stix2.EqualityComparisonExpression( + "file:mime_type", + "image/bmp", + ), + stix2.EqualityComparisonExpression( + "file:magic_number_hex", + stix2.HexConstant("ffd8"), + ), + ]) exp = stix2.ObservationExpression(exp_and) assert str(exp) == "[file:mime_type = 'image/bmp' AND file:magic_number_hex = h'ffd8']" def test_multiple_qualifiers(): - exp_and = stix2.AndBooleanExpression([stix2.EqualityComparisonExpression("network-traffic:dst_ref.type", - "domain-name"), - stix2.EqualityComparisonExpression("network-traffic:dst_ref.value", - "example.com")]) + exp_and = stix2.AndBooleanExpression([ + stix2.EqualityComparisonExpression( + "network-traffic:dst_ref.type", + "domain-name", + ), + stix2.EqualityComparisonExpression( + "network-traffic:dst_ref.value", + "example.com", + ), + ]) exp_ob = stix2.ObservationExpression(exp_and) qual_rep = stix2.RepeatQualifier(5) qual_within = stix2.WithinQualifier(stix2.IntegerConstant(1800)) @@ -208,8 +306,10 @@ def test_multiple_qualifiers(): def test_set_op(): - exp = stix2.ObservationExpression(stix2.IsSubsetComparisonExpression("network-traffic:dst_ref.value", - "2001:0db8:dead:beef:0000:0000:0000:0000/64")) + exp = stix2.ObservationExpression(stix2.IsSubsetComparisonExpression( + "network-traffic:dst_ref.value", + "2001:0db8:dead:beef:0000:0000:0000:0000/64", + )) assert str(exp) == "[network-traffic:dst_ref.value ISSUBSET '2001:0db8:dead:beef:0000:0000:0000:0000/64']" @@ -219,35 +319,45 @@ def test_timestamp(): def test_boolean(): - exp = stix2.EqualityComparisonExpression("email-message:is_multipart", - True) + exp = stix2.EqualityComparisonExpression( + "email-message:is_multipart", + True, + ) assert str(exp) == "email-message:is_multipart = true" def test_binary(): const = stix2.BinaryConstant("dGhpcyBpcyBhIHRlc3Q=") - exp = stix2.EqualityComparisonExpression("artifact:payload_bin", - const) + exp = stix2.EqualityComparisonExpression( + "artifact:payload_bin", + const, + ) assert str(exp) == "artifact:payload_bin = b'dGhpcyBpcyBhIHRlc3Q='" def test_list(): - exp = stix2.InComparisonExpression("process:name", - ['proccy', 'proximus', 'badproc']) + exp = stix2.InComparisonExpression( + "process:name", + ['proccy', 'proximus', 'badproc'], + ) assert str(exp) == "process:name IN ('proccy', 'proximus', 'badproc')" def test_list2(): # alternate way to construct an "IN" Comparison Expression - exp = stix2.EqualityComparisonExpression("process:name", - ['proccy', 'proximus', 'badproc']) + exp = stix2.EqualityComparisonExpression( + "process:name", + ['proccy', 'proximus', 'badproc'], + ) assert str(exp) == "process:name IN ('proccy', 'proximus', 'badproc')" def test_invalid_constant_type(): with pytest.raises(ValueError) as excinfo: - stix2.EqualityComparisonExpression("artifact:payload_bin", - {'foo': 'bar'}) + stix2.EqualityComparisonExpression( + "artifact:payload_bin", + {'foo': 'bar'}, + ) assert 'Unable to create a constant' in str(excinfo) @@ -269,20 +379,22 @@ def test_invalid_float_constant(): assert 'must be a float' in str(excinfo) -@pytest.mark.parametrize("data, result", [ - (True, True), - (False, False), - ('True', True), - ('False', False), - ('true', True), - ('false', False), - ('t', True), - ('f', False), - ('T', True), - ('F', False), - (1, True), - (0, False), -]) +@pytest.mark.parametrize( + "data, result", [ + (True, True), + (False, False), + ('True', True), + ('False', False), + ('true', True), + ('false', False), + ('t', True), + ('f', False), + ('T', True), + ('F', False), + (1, True), + (0, False), + ], +) def test_boolean_constant(data, result): boolean = stix2.BooleanConstant(data) assert boolean.value == result @@ -294,10 +406,12 @@ def test_invalid_boolean_constant(): assert 'must be a boolean' in str(excinfo) -@pytest.mark.parametrize("hashtype, data", [ - ('MD5', 'zzz'), - ('ssdeep', 'zzz=='), -]) +@pytest.mark.parametrize( + "hashtype, data", [ + ('MD5', 'zzz'), + ('ssdeep', 'zzz=='), + ], +) def test_invalid_hash_constant(hashtype, data): with pytest.raises(ValueError) as excinfo: stix2.HashConstant(data, hashtype) @@ -317,20 +431,26 @@ def test_invalid_binary_constant(): def test_escape_quotes_and_backslashes(): - exp = stix2.MatchesComparisonExpression("file:name", - "^Final Report.+\\.exe$") + exp = stix2.MatchesComparisonExpression( + "file:name", + "^Final Report.+\\.exe$", + ) assert str(exp) == "file:name MATCHES '^Final Report.+\\\\.exe$'" def test_like(): - exp = stix2.LikeComparisonExpression("directory:path", - "C:\\Windows\\%\\foo") + exp = stix2.LikeComparisonExpression( + "directory:path", + "C:\\Windows\\%\\foo", + ) assert str(exp) == "directory:path LIKE 'C:\\\\Windows\\\\%\\\\foo'" def test_issuperset(): - exp = stix2.IsSupersetComparisonExpression("ipv4-addr:value", - "198.51.100.0/24") + exp = stix2.IsSupersetComparisonExpression( + "ipv4-addr:value", + "198.51.100.0/24", + ) assert str(exp) == "ipv4-addr:value ISSUPERSET '198.51.100.0/24'" @@ -352,24 +472,32 @@ def test_invalid_within_qualifier(): def test_startstop_qualifier(): - qual = stix2.StartStopQualifier(stix2.TimestampConstant('2016-06-01T00:00:00Z'), - datetime.datetime(2017, 3, 12, 8, 30, 0)) + qual = stix2.StartStopQualifier( + stix2.TimestampConstant('2016-06-01T00:00:00Z'), + datetime.datetime(2017, 3, 12, 8, 30, 0), + ) assert str(qual) == "START t'2016-06-01T00:00:00Z' STOP t'2017-03-12T08:30:00Z'" - qual2 = stix2.StartStopQualifier(datetime.date(2016, 6, 1), - stix2.TimestampConstant('2016-07-01T00:00:00Z')) + qual2 = stix2.StartStopQualifier( + datetime.date(2016, 6, 1), + stix2.TimestampConstant('2016-07-01T00:00:00Z'), + ) assert str(qual2) == "START t'2016-06-01T00:00:00Z' STOP t'2016-07-01T00:00:00Z'" def test_invalid_startstop_qualifier(): with pytest.raises(ValueError) as excinfo: - stix2.StartStopQualifier('foo', - stix2.TimestampConstant('2016-06-01T00:00:00Z')) + stix2.StartStopQualifier( + 'foo', + stix2.TimestampConstant('2016-06-01T00:00:00Z'), + ) assert 'is not a valid argument for a Start/Stop Qualifier' in str(excinfo) with pytest.raises(ValueError) as excinfo: - stix2.StartStopQualifier(datetime.date(2016, 6, 1), - 'foo') + stix2.StartStopQualifier( + datetime.date(2016, 6, 1), + 'foo', + ) assert 'is not a valid argument for a Start/Stop Qualifier' in str(excinfo) @@ -377,3 +505,15 @@ def test_make_constant_already_a_constant(): str_const = stix2.StringConstant('Foo') result = stix2.patterns.make_constant(str_const) assert result is str_const + + +def test_parsing_comparison_expression(): + patt_obj = create_pattern_object("[file:hashes.'SHA-256' = 'aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f']") + assert str(patt_obj) == "[file:hashes.'SHA-256' = 'aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f']" + + +def test_parsing_qualified_expression(): + patt_obj = create_pattern_object( + "[network-traffic:dst_ref.type = 'domain-name' AND network-traffic:dst_ref.value = 'example.com'] REPEATS 5 TIMES WITHIN 1800 SECONDS") + assert str( + patt_obj) == "[network-traffic:dst_ref.type = 'domain-name' AND network-traffic:dst_ref.value = 'example.com'] REPEATS 5 TIMES WITHIN 1800 SECONDS" From da4a91a3ca91b0242f6b9bfa759cbff94f70d6df Mon Sep 17 00:00:00 2001 From: Richard Piazza Date: Fri, 7 Dec 2018 13:30:05 -0500 Subject: [PATCH 02/18] flakey --- stix2/STIXPatternVisitor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stix2/STIXPatternVisitor.py b/stix2/STIXPatternVisitor.py index 5339395..a94aa40 100644 --- a/stix2/STIXPatternVisitor.py +++ b/stix2/STIXPatternVisitor.py @@ -13,7 +13,8 @@ from stix2.patterns import _BooleanExpression from antlr4 import CommonTokenStream, InputStream # need to import all classes because we need to access them via globals() -from .patterns import * +from .patterns import * # noqa + def collapse_lists(lists): result = [] @@ -33,7 +34,6 @@ def values_only(things): return values - # This class defines a complete generic visitor for a parse tree produced by STIXPatternParser. From 3ea8035fcb97bdff748240335cb678e7c7b4cc80 Mon Sep 17 00:00:00 2001 From: Richard Piazza Date: Fri, 7 Dec 2018 13:47:44 -0500 Subject: [PATCH 03/18] flakey2 --- stix2/STIXPatternVisitor.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/stix2/STIXPatternVisitor.py b/stix2/STIXPatternVisitor.py index a94aa40..43be107 100644 --- a/stix2/STIXPatternVisitor.py +++ b/stix2/STIXPatternVisitor.py @@ -14,6 +14,9 @@ from antlr4 import CommonTokenStream, InputStream # need to import all classes because we need to access them via globals() from .patterns import * # noqa +from .patterns import (FollowedByObservationExpression, StartStopQualifier, WithinQualifier, + RepeatQualifier, StringConstant, IntegerConstant, FloatConstant, HexConstant, + BinaryConstant, BooleanConstant, TimestampConstant) def collapse_lists(lists): From 52c7d2a7229fc12813bccac384bf0b821503393a Mon Sep 17 00:00:00 2001 From: Richard Piazza Date: Fri, 7 Dec 2018 13:59:57 -0500 Subject: [PATCH 04/18] flakey3 --- stix2/test/test_pattern_expressions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stix2/test/test_pattern_expressions.py b/stix2/test/test_pattern_expressions.py index c136eaa..777709c 100644 --- a/stix2/test/test_pattern_expressions.py +++ b/stix2/test/test_pattern_expressions.py @@ -2,10 +2,10 @@ import datetime import pytest -import stix2 - from stix2.STIXPatternVisitor import create_pattern_object +import stix2 + def test_create_comparison_expression(): exp = stix2.EqualityComparisonExpression( From 55ac3564bd4b9b6fc38c05f55d0a39c7a456e2e1 Mon Sep 17 00:00:00 2001 From: Richard Piazza Date: Fri, 7 Dec 2018 14:03:46 -0500 Subject: [PATCH 05/18] flakey4 --- stix2/__init__.py | 1 + stix2/test/test_pattern_expressions.py | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/stix2/__init__.py b/stix2/__init__.py index 449be68..dfffec0 100644 --- a/stix2/__init__.py +++ b/stix2/__init__.py @@ -53,6 +53,7 @@ from .patterns import (AndBooleanExpression, AndObservationExpression, from .utils import new_version, revoke from .v20 import * # This import will always be the latest STIX 2.X version from .version import __version__ +from .STIXPatternVisitor import create_pattern_object _collect_stix2_obj_maps() diff --git a/stix2/test/test_pattern_expressions.py b/stix2/test/test_pattern_expressions.py index 777709c..ec656c8 100644 --- a/stix2/test/test_pattern_expressions.py +++ b/stix2/test/test_pattern_expressions.py @@ -2,8 +2,6 @@ import datetime import pytest -from stix2.STIXPatternVisitor import create_pattern_object - import stix2 From f3526bbfa675e4b60e6ff2b5c518bf8218e0823a Mon Sep 17 00:00:00 2001 From: Richard Piazza Date: Fri, 7 Dec 2018 14:06:36 -0500 Subject: [PATCH 06/18] flakey5 --- stix2/__init__.py | 3 ++- stix2/test/test_pattern_expressions.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/stix2/__init__.py b/stix2/__init__.py index dfffec0..c5cf3f3 100644 --- a/stix2/__init__.py +++ b/stix2/__init__.py @@ -50,10 +50,11 @@ from .patterns import (AndBooleanExpression, AndObservationExpression, ReferenceObjectPathComponent, RepeatQualifier, StartStopQualifier, StringConstant, TimestampConstant, WithinQualifier) +from .STIXPatternVisitor import create_pattern_object from .utils import new_version, revoke from .v20 import * # This import will always be the latest STIX 2.X version from .version import __version__ -from .STIXPatternVisitor import create_pattern_object + _collect_stix2_obj_maps() diff --git a/stix2/test/test_pattern_expressions.py b/stix2/test/test_pattern_expressions.py index ec656c8..91aee5a 100644 --- a/stix2/test/test_pattern_expressions.py +++ b/stix2/test/test_pattern_expressions.py @@ -506,12 +506,12 @@ def test_make_constant_already_a_constant(): def test_parsing_comparison_expression(): - patt_obj = create_pattern_object("[file:hashes.'SHA-256' = 'aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f']") + patt_obj = stix2.create_pattern_object("[file:hashes.'SHA-256' = 'aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f']") assert str(patt_obj) == "[file:hashes.'SHA-256' = 'aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f']" def test_parsing_qualified_expression(): - patt_obj = create_pattern_object( + patt_obj = stix2.create_pattern_object( "[network-traffic:dst_ref.type = 'domain-name' AND network-traffic:dst_ref.value = 'example.com'] REPEATS 5 TIMES WITHIN 1800 SECONDS") assert str( patt_obj) == "[network-traffic:dst_ref.type = 'domain-name' AND network-traffic:dst_ref.value = 'example.com'] REPEATS 5 TIMES WITHIN 1800 SECONDS" From 99276c92fc5beea5278742699249248f76c1feb7 Mon Sep 17 00:00:00 2001 From: Richard Piazza Date: Sun, 9 Dec 2018 21:42:24 -0500 Subject: [PATCH 07/18] flake-y again --- stix2/__init__.py | 1 - stix2/patterns.py | 3 ++- stix2/test/test_pattern_expressions.py | 6 ++++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/stix2/__init__.py b/stix2/__init__.py index c5cf3f3..3d70001 100644 --- a/stix2/__init__.py +++ b/stix2/__init__.py @@ -50,7 +50,6 @@ from .patterns import (AndBooleanExpression, AndObservationExpression, ReferenceObjectPathComponent, RepeatQualifier, StartStopQualifier, StringConstant, TimestampConstant, WithinQualifier) -from .STIXPatternVisitor import create_pattern_object from .utils import new_version, revoke from .v20 import * # This import will always be the latest STIX 2.X version from .version import __version__ diff --git a/stix2/patterns.py b/stix2/patterns.py index 4828b07..fec060e 100644 --- a/stix2/patterns.py +++ b/stix2/patterns.py @@ -5,6 +5,7 @@ import base64 import binascii import datetime import re +import six from .utils import parse_into_datetime @@ -14,7 +15,7 @@ def escape_quotes_and_backslashes(s): def quote_if_needed(x): - if isinstance(x, str): + if isinstance(x, six.string_types): if x.find("-") != -1: if not x.startswith("'"): return "'" + x + "'" diff --git a/stix2/test/test_pattern_expressions.py b/stix2/test/test_pattern_expressions.py index 91aee5a..c136eaa 100644 --- a/stix2/test/test_pattern_expressions.py +++ b/stix2/test/test_pattern_expressions.py @@ -4,6 +4,8 @@ import pytest import stix2 +from stix2.STIXPatternVisitor import create_pattern_object + def test_create_comparison_expression(): exp = stix2.EqualityComparisonExpression( @@ -506,12 +508,12 @@ def test_make_constant_already_a_constant(): def test_parsing_comparison_expression(): - patt_obj = stix2.create_pattern_object("[file:hashes.'SHA-256' = 'aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f']") + patt_obj = create_pattern_object("[file:hashes.'SHA-256' = 'aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f']") assert str(patt_obj) == "[file:hashes.'SHA-256' = 'aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f']" def test_parsing_qualified_expression(): - patt_obj = stix2.create_pattern_object( + patt_obj = create_pattern_object( "[network-traffic:dst_ref.type = 'domain-name' AND network-traffic:dst_ref.value = 'example.com'] REPEATS 5 TIMES WITHIN 1800 SECONDS") assert str( patt_obj) == "[network-traffic:dst_ref.type = 'domain-name' AND network-traffic:dst_ref.value = 'example.com'] REPEATS 5 TIMES WITHIN 1800 SECONDS" From a5eca9916c51063c72902e98296abba4116e5be1 Mon Sep 17 00:00:00 2001 From: Richard Piazza Date: Sun, 9 Dec 2018 21:48:24 -0500 Subject: [PATCH 08/18] last flake-y --- stix2/STIXPatternVisitor.py | 14 +++++++------- stix2/__init__.py | 1 - stix2/patterns.py | 1 + stix2/test/test_pattern_expressions.py | 1 - 4 files changed, 8 insertions(+), 9 deletions(-) diff --git a/stix2/STIXPatternVisitor.py b/stix2/STIXPatternVisitor.py index 43be107..89b79da 100644 --- a/stix2/STIXPatternVisitor.py +++ b/stix2/STIXPatternVisitor.py @@ -3,20 +3,20 @@ import inspect import six from stix2patterns.grammars.STIXPatternLexer import STIXPatternLexer -from stix2patterns.grammars.STIXPatternParser import ( - STIXPatternParser, TerminalNode, -) +from stix2patterns.grammars.STIXPatternParser import (STIXPatternParser, + TerminalNode) from stix2patterns.grammars.STIXPatternVisitor import STIXPatternVisitor from stix2patterns.validator import STIXPatternErrorListener -from stix2.patterns import _BooleanExpression from antlr4 import CommonTokenStream, InputStream +from stix2.patterns import _BooleanExpression # need to import all classes because we need to access them via globals() +from .patterns import (BinaryConstant, BooleanConstant, FloatConstant, + FollowedByObservationExpression, HexConstant, + IntegerConstant, RepeatQualifier, StartStopQualifier, + StringConstant, TimestampConstant, WithinQualifier) from .patterns import * # noqa -from .patterns import (FollowedByObservationExpression, StartStopQualifier, WithinQualifier, - RepeatQualifier, StringConstant, IntegerConstant, FloatConstant, HexConstant, - BinaryConstant, BooleanConstant, TimestampConstant) def collapse_lists(lists): diff --git a/stix2/__init__.py b/stix2/__init__.py index 3d70001..449be68 100644 --- a/stix2/__init__.py +++ b/stix2/__init__.py @@ -54,7 +54,6 @@ from .utils import new_version, revoke from .v20 import * # This import will always be the latest STIX 2.X version from .version import __version__ - _collect_stix2_obj_maps() DEFAULT_VERSION = "2.0" # Default version will always be the latest STIX 2.X version diff --git a/stix2/patterns.py b/stix2/patterns.py index fec060e..1e0e03f 100644 --- a/stix2/patterns.py +++ b/stix2/patterns.py @@ -5,6 +5,7 @@ import base64 import binascii import datetime import re + import six from .utils import parse_into_datetime diff --git a/stix2/test/test_pattern_expressions.py b/stix2/test/test_pattern_expressions.py index c136eaa..0aff872 100644 --- a/stix2/test/test_pattern_expressions.py +++ b/stix2/test/test_pattern_expressions.py @@ -3,7 +3,6 @@ import datetime import pytest import stix2 - from stix2.STIXPatternVisitor import create_pattern_object From 05964ee0c72af9ca7c9ab71b676374779ce95c45 Mon Sep 17 00:00:00 2001 From: Richard Piazza Date: Mon, 10 Dec 2018 12:14:31 -0500 Subject: [PATCH 09/18] import experiment --- stix2/STIXPatternVisitor.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/stix2/STIXPatternVisitor.py b/stix2/STIXPatternVisitor.py index 89b79da..902541e 100644 --- a/stix2/STIXPatternVisitor.py +++ b/stix2/STIXPatternVisitor.py @@ -12,10 +12,10 @@ from antlr4 import CommonTokenStream, InputStream from stix2.patterns import _BooleanExpression # need to import all classes because we need to access them via globals() -from .patterns import (BinaryConstant, BooleanConstant, FloatConstant, - FollowedByObservationExpression, HexConstant, - IntegerConstant, RepeatQualifier, StartStopQualifier, - StringConstant, TimestampConstant, WithinQualifier) +# from .patterns import (BinaryConstant, BooleanConstant, FloatConstant, +# FollowedByObservationExpression, HexConstant, +# IntegerConstant, RepeatQualifier, StartStopQualifier, +# StringConstant, TimestampConstant, WithinQualifier) from .patterns import * # noqa @@ -29,9 +29,9 @@ def collapse_lists(lists): return result -def values_only(things): +def remove_terminal_nodes(parse_tree_nodes): values = [] - for x in things: + for x in parse_tree_nodes: if not isinstance(x, TerminalNode): values.append(x) return values @@ -152,7 +152,7 @@ class STIXPatternVisitorForSTIX2(STIXPatternVisitor): def visitPropTestEqual(self, ctx): children = self.visitChildren(ctx) operator = children[1].symbol.type - negated = negated = operator != STIXPatternParser.EQ + negated = operator != STIXPatternParser.EQ return self.instantiate("EqualityComparisonExpression", children[0], children[3 if len(children) > 3 else 2], negated) @@ -274,7 +274,7 @@ class STIXPatternVisitorForSTIX2(STIXPatternVisitor): # Visit a parse tree produced by STIXPatternParser#setLiteral. def visitSetLiteral(self, ctx): children = self.visitChildren(ctx) - return self.instantiate("ListConstant", values_only(children)) + return self.instantiate("ListConstant", remove_terminal_nodes(children)) # Visit a parse tree produced by STIXPatternParser#primitiveLiteral. def visitPrimitiveLiteral(self, ctx): From 1bb3aa12f00059f47bbae321a433f2b92cd01b48 Mon Sep 17 00:00:00 2001 From: Richard Piazza Date: Mon, 10 Dec 2018 12:16:39 -0500 Subject: [PATCH 10/18] import experiment 2 --- stix2/STIXPatternVisitor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stix2/STIXPatternVisitor.py b/stix2/STIXPatternVisitor.py index 902541e..523038f 100644 --- a/stix2/STIXPatternVisitor.py +++ b/stix2/STIXPatternVisitor.py @@ -16,7 +16,7 @@ from stix2.patterns import _BooleanExpression # FollowedByObservationExpression, HexConstant, # IntegerConstant, RepeatQualifier, StartStopQualifier, # StringConstant, TimestampConstant, WithinQualifier) -from .patterns import * # noqa +from .patterns import * # noqa: F405 def collapse_lists(lists): From 7bd330dfae9c2e5e1cc1473a8ab88daa098f0689 Mon Sep 17 00:00:00 2001 From: Richard Piazza Date: Mon, 10 Dec 2018 12:25:59 -0500 Subject: [PATCH 11/18] import experiment 3 --- stix2/STIXPatternVisitor.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/stix2/STIXPatternVisitor.py b/stix2/STIXPatternVisitor.py index 523038f..c7c5a3e 100644 --- a/stix2/STIXPatternVisitor.py +++ b/stix2/STIXPatternVisitor.py @@ -11,11 +11,6 @@ from stix2patterns.validator import STIXPatternErrorListener from antlr4 import CommonTokenStream, InputStream from stix2.patterns import _BooleanExpression -# need to import all classes because we need to access them via globals() -# from .patterns import (BinaryConstant, BooleanConstant, FloatConstant, -# FollowedByObservationExpression, HexConstant, -# IntegerConstant, RepeatQualifier, StartStopQualifier, -# StringConstant, TimestampConstant, WithinQualifier) from .patterns import * # noqa: F405 From fcea810ea104880233ec7584ba008cb9936e65bc Mon Sep 17 00:00:00 2001 From: Richard Piazza Date: Mon, 10 Dec 2018 12:54:58 -0500 Subject: [PATCH 12/18] added test for ListConstant --- stix2/test/test_pattern_expressions.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/stix2/test/test_pattern_expressions.py b/stix2/test/test_pattern_expressions.py index 0aff872..2d59cde 100644 --- a/stix2/test/test_pattern_expressions.py +++ b/stix2/test/test_pattern_expressions.py @@ -516,3 +516,8 @@ def test_parsing_qualified_expression(): "[network-traffic:dst_ref.type = 'domain-name' AND network-traffic:dst_ref.value = 'example.com'] REPEATS 5 TIMES WITHIN 1800 SECONDS") assert str( patt_obj) == "[network-traffic:dst_ref.type = 'domain-name' AND network-traffic:dst_ref.value = 'example.com'] REPEATS 5 TIMES WITHIN 1800 SECONDS" + + +def test_list_constant(): + patt_obj = create_pattern_object("[network-traffic:src_ref.value IN ('10.0.0.0', '10.0.0.1', '10.0.0.2')]") + assert str(patt_obj) == "[network-traffic:src_ref.value IN ('10.0.0.0', '10.0.0.1', '10.0.0.2')]" \ No newline at end of file From b3f69bf942256c8d890440f67f0e3ab3312781b0 Mon Sep 17 00:00:00 2001 From: Richard Piazza Date: Mon, 10 Dec 2018 13:42:05 -0500 Subject: [PATCH 13/18] imports again --- stix2/STIXPatternVisitor.py | 9 +++++++-- stix2/test/test_pattern_expressions.py | 2 +- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/stix2/STIXPatternVisitor.py b/stix2/STIXPatternVisitor.py index c7c5a3e..6c6690f 100644 --- a/stix2/STIXPatternVisitor.py +++ b/stix2/STIXPatternVisitor.py @@ -9,9 +9,14 @@ from stix2patterns.grammars.STIXPatternVisitor import STIXPatternVisitor from stix2patterns.validator import STIXPatternErrorListener from antlr4 import CommonTokenStream, InputStream -from stix2.patterns import _BooleanExpression +from stix2.patterns import (BinaryConstant, BooleanConstant, FloatConstant, + FollowedByObservationExpression, HexConstant, + IntegerConstant, RepeatQualifier, + StartStopQualifier, StringConstant, + TimestampConstant, WithinQualifier) +from stix2.patterns import _BooleanExpression # noqa -from .patterns import * # noqa: F405 +from .patterns import * # noqa: F403, F401 def collapse_lists(lists): diff --git a/stix2/test/test_pattern_expressions.py b/stix2/test/test_pattern_expressions.py index 2d59cde..62dc242 100644 --- a/stix2/test/test_pattern_expressions.py +++ b/stix2/test/test_pattern_expressions.py @@ -520,4 +520,4 @@ def test_parsing_qualified_expression(): def test_list_constant(): patt_obj = create_pattern_object("[network-traffic:src_ref.value IN ('10.0.0.0', '10.0.0.1', '10.0.0.2')]") - assert str(patt_obj) == "[network-traffic:src_ref.value IN ('10.0.0.0', '10.0.0.1', '10.0.0.2')]" \ No newline at end of file + assert str(patt_obj) == "[network-traffic:src_ref.value IN ('10.0.0.0', '10.0.0.1', '10.0.0.2')]" From fc0069ed600b33a719c95867da6daa8a68718e9c Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Mon, 10 Dec 2018 14:29:31 -0500 Subject: [PATCH 14/18] re-order imports, add entry to isort file --- .isort.cfg | 1 + stix2/STIXPatternVisitor.py | 17 +++++++---------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/.isort.cfg b/.isort.cfg index 0780dcd..84bc4c7 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -2,6 +2,7 @@ skip = workbench.py not_skip = __init__.py known_third_party = + antlr4, dateutil, medallion, pytest, diff --git a/stix2/STIXPatternVisitor.py b/stix2/STIXPatternVisitor.py index 6c6690f..0857f02 100644 --- a/stix2/STIXPatternVisitor.py +++ b/stix2/STIXPatternVisitor.py @@ -1,6 +1,7 @@ import importlib import inspect +from antlr4 import CommonTokenStream, InputStream import six from stix2patterns.grammars.STIXPatternLexer import STIXPatternLexer from stix2patterns.grammars.STIXPatternParser import (STIXPatternParser, @@ -8,15 +9,11 @@ from stix2patterns.grammars.STIXPatternParser import (STIXPatternParser, from stix2patterns.grammars.STIXPatternVisitor import STIXPatternVisitor from stix2patterns.validator import STIXPatternErrorListener -from antlr4 import CommonTokenStream, InputStream -from stix2.patterns import (BinaryConstant, BooleanConstant, FloatConstant, - FollowedByObservationExpression, HexConstant, - IntegerConstant, RepeatQualifier, - StartStopQualifier, StringConstant, - TimestampConstant, WithinQualifier) -from stix2.patterns import _BooleanExpression # noqa - -from .patterns import * # noqa: F403, F401 +from .patterns import (BinaryConstant, BooleanConstant, FloatConstant, + FollowedByObservationExpression, HexConstant, + IntegerConstant, RepeatQualifier, StartStopQualifier, + StringConstant, TimestampConstant, WithinQualifier, + _BooleanExpression) def collapse_lists(lists): @@ -46,7 +43,7 @@ class STIXPatternVisitorForSTIX2(STIXPatternVisitor): def __init__(self, module_suffix, module_name): if module_suffix and module_name: self.module_suffix = module_suffix - if STIXPatternVisitorForSTIX2.classes == {}: + if not STIXPatternVisitorForSTIX2.classes: module = importlib.import_module(module_name) for k, c in inspect.getmembers(module, inspect.isclass): STIXPatternVisitorForSTIX2.classes[k] = c From 50a2191805acc6d66adfd6fab90b723728b09b32 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Mon, 10 Dec 2018 14:44:44 -0500 Subject: [PATCH 15/18] Favor star import, disable messages for undefined methods F405 --- stix2/STIXPatternVisitor.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/stix2/STIXPatternVisitor.py b/stix2/STIXPatternVisitor.py index 0857f02..456c96f 100644 --- a/stix2/STIXPatternVisitor.py +++ b/stix2/STIXPatternVisitor.py @@ -9,11 +9,10 @@ from stix2patterns.grammars.STIXPatternParser import (STIXPatternParser, from stix2patterns.grammars.STIXPatternVisitor import STIXPatternVisitor from stix2patterns.validator import STIXPatternErrorListener -from .patterns import (BinaryConstant, BooleanConstant, FloatConstant, - FollowedByObservationExpression, HexConstant, - IntegerConstant, RepeatQualifier, StartStopQualifier, - StringConstant, TimestampConstant, WithinQualifier, - _BooleanExpression) +from .patterns import * +from .patterns import _BooleanExpression + +# flake8: noqa F405 def collapse_lists(lists): From 7702d435ba0233dff1c83ba3237de66f13a60e6a Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Mon, 10 Dec 2018 15:07:38 -0500 Subject: [PATCH 16/18] update method to use docstrings --- stix2/STIXPatternVisitor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stix2/STIXPatternVisitor.py b/stix2/STIXPatternVisitor.py index 456c96f..94c8559 100644 --- a/stix2/STIXPatternVisitor.py +++ b/stix2/STIXPatternVisitor.py @@ -309,10 +309,10 @@ class STIXPatternVisitorForSTIX2(STIXPatternVisitor): def create_pattern_object(pattern, module_suffix="", module_name=""): - ''' + """ Validates a pattern against the STIX Pattern grammar. Error messages are returned in a list. The test passed if the returned list is empty. - ''' + """ start = '' if isinstance(pattern, six.string_types): From 9a69823d089fe362576b02432dc375543a94b304 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Mon, 10 Dec 2018 15:08:43 -0500 Subject: [PATCH 17/18] Revert unnecessary changes --- .../20170531213258226477.json | 60 +++++++++---------- .../20181101232448456000.json | 50 ++++++++-------- .../20181101232448457000.json | 50 ++++++++-------- 3 files changed, 80 insertions(+), 80 deletions(-) diff --git a/stix2/test/stix2_data/malware/malware--6b616fc1-1505-48e3-8b2c-0d19337bff38/20170531213258226477.json b/stix2/test/stix2_data/malware/malware--6b616fc1-1505-48e3-8b2c-0d19337bff38/20170531213258226477.json index 4c03007..c60200b 100644 --- a/stix2/test/stix2_data/malware/malware--6b616fc1-1505-48e3-8b2c-0d19337bff38/20170531213258226477.json +++ b/stix2/test/stix2_data/malware/malware--6b616fc1-1505-48e3-8b2c-0d19337bff38/20170531213258226477.json @@ -1,34 +1,34 @@ { - "id": "bundle--f64de948-7067-4534-8018-85f03d470625", - "objects": [ - { - "created": "2017-05-31T21:32:58.226477Z", - "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5", - "description": "Rover is malware suspected of being used for espionage purposes. It was used in 2015 in a targeted email sent to an Indian Ambassador to Afghanistan.[[Citation: Palo Alto Rover]]", - "external_references": [ + "id": "bundle--f64de948-7067-4534-8018-85f03d470625", + "objects": [ { - "external_id": "S0090", - "source_name": "mitre-attack", - "url": "https://attack.mitre.org/wiki/Software/S0090" - }, - { - "description": "Ray, V., Hayashi, K. (2016, February 29). New Malware \u2018Rover\u2019 Targets Indian Ambassador to Afghanistan. Retrieved February 29, 2016.", - "source_name": "Palo Alto Rover", - "url": "http://researchcenter.paloaltonetworks.com/2016/02/new-malware-rover-targets-indian-ambassador-to-afghanistan/" + "created": "2017-05-31T21:32:58.226477Z", + "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5", + "description": "Rover is malware suspected of being used for espionage purposes. It was used in 2015 in a targeted email sent to an Indian Ambassador to Afghanistan.[[Citation: Palo Alto Rover]]", + "external_references": [ + { + "external_id": "S0090", + "source_name": "mitre-attack", + "url": "https://attack.mitre.org/wiki/Software/S0090" + }, + { + "description": "Ray, V., Hayashi, K. (2016, February 29). New Malware \u2018Rover\u2019 Targets Indian Ambassador to Afghanistan. Retrieved February 29, 2016.", + "source_name": "Palo Alto Rover", + "url": "http://researchcenter.paloaltonetworks.com/2016/02/new-malware-rover-targets-indian-ambassador-to-afghanistan/" + } + ], + "id": "malware--6b616fc1-1505-48e3-8b2c-0d19337bff38", + "labels": [ + "malware" + ], + "modified": "2017-05-31T21:32:58.226477Z", + "name": "Rover", + "object_marking_refs": [ + "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168" + ], + "type": "malware" } - ], - "id": "malware--6b616fc1-1505-48e3-8b2c-0d19337bff38", - "labels": [ - "malware" - ], - "modified": "2017-05-31T21:32:58.226477Z", - "name": "Rover", - "object_marking_refs": [ - "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168" - ], - "type": "malware" - } - ], - "spec_version": "2.0", - "type": "bundle" + ], + "spec_version": "2.0", + "type": "bundle" } diff --git a/stix2/test/stix2_data/malware/malware--6b616fc1-1505-48e3-8b2c-0d19337bff38/20181101232448456000.json b/stix2/test/stix2_data/malware/malware--6b616fc1-1505-48e3-8b2c-0d19337bff38/20181101232448456000.json index 1016dec..af47f27 100644 --- a/stix2/test/stix2_data/malware/malware--6b616fc1-1505-48e3-8b2c-0d19337bff38/20181101232448456000.json +++ b/stix2/test/stix2_data/malware/malware--6b616fc1-1505-48e3-8b2c-0d19337bff38/20181101232448456000.json @@ -1,27 +1,27 @@ { - "type": "malware", - "id": "malware--6b616fc1-1505-48e3-8b2c-0d19337bff38", - "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5", - "created": "2017-05-31T21:32:58.226Z", - "modified": "2018-11-01T23:24:48.456Z", - "name": "Rover", - "description": "Rover is malware suspected of being used for espionage purposes. It was used in 2015 in a targeted email sent to an Indian Ambassador to Afghanistan.[[Citation: Palo Alto Rover]]", - "labels": [ - "version two" - ], - "external_references": [ - { - "source_name": "mitre-attack", - "url": "https://attack.mitre.org/wiki/Software/S0090", - "external_id": "S0090" - }, - { - "source_name": "Palo Alto Rover", - "description": "Ray, V., Hayashi, K. (2016, February 29). New Malware \u2018Rover\u2019 Targets Indian Ambassador to Afghanistan. Retrieved February 29, 2016.", - "url": "http://researchcenter.paloaltonetworks.com/2016/02/new-malware-rover-targets-indian-ambassador-to-afghanistan/" - } - ], - "object_marking_refs": [ - "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168" - ] + "type": "malware", + "id": "malware--6b616fc1-1505-48e3-8b2c-0d19337bff38", + "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5", + "created": "2017-05-31T21:32:58.226Z", + "modified": "2018-11-01T23:24:48.456Z", + "name": "Rover", + "description": "Rover is malware suspected of being used for espionage purposes. It was used in 2015 in a targeted email sent to an Indian Ambassador to Afghanistan.[[Citation: Palo Alto Rover]]", + "labels": [ + "version two" + ], + "external_references": [ + { + "source_name": "mitre-attack", + "url": "https://attack.mitre.org/wiki/Software/S0090", + "external_id": "S0090" + }, + { + "source_name": "Palo Alto Rover", + "description": "Ray, V., Hayashi, K. (2016, February 29). New Malware \u2018Rover\u2019 Targets Indian Ambassador to Afghanistan. Retrieved February 29, 2016.", + "url": "http://researchcenter.paloaltonetworks.com/2016/02/new-malware-rover-targets-indian-ambassador-to-afghanistan/" + } + ], + "object_marking_refs": [ + "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168" + ] } diff --git a/stix2/test/stix2_data/malware/malware--6b616fc1-1505-48e3-8b2c-0d19337bff38/20181101232448457000.json b/stix2/test/stix2_data/malware/malware--6b616fc1-1505-48e3-8b2c-0d19337bff38/20181101232448457000.json index e7328f8..446fb26 100644 --- a/stix2/test/stix2_data/malware/malware--6b616fc1-1505-48e3-8b2c-0d19337bff38/20181101232448457000.json +++ b/stix2/test/stix2_data/malware/malware--6b616fc1-1505-48e3-8b2c-0d19337bff38/20181101232448457000.json @@ -1,27 +1,27 @@ { - "type": "malware", - "id": "malware--6b616fc1-1505-48e3-8b2c-0d19337bff38", - "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5", - "created": "2017-05-31T21:32:58.226Z", - "modified": "2018-11-01T23:24:48.457Z", - "name": "Rover", - "description": "Rover is malware suspected of being used for espionage purposes. It was used in 2015 in a targeted email sent to an Indian Ambassador to Afghanistan.[[Citation: Palo Alto Rover]]", - "labels": [ - "version three" - ], - "external_references": [ - { - "source_name": "mitre-attack", - "url": "https://attack.mitre.org/wiki/Software/S0090", - "external_id": "S0090" - }, - { - "source_name": "Palo Alto Rover", - "description": "Ray, V., Hayashi, K. (2016, February 29). New Malware \u2018Rover\u2019 Targets Indian Ambassador to Afghanistan. Retrieved February 29, 2016.", - "url": "http://researchcenter.paloaltonetworks.com/2016/02/new-malware-rover-targets-indian-ambassador-to-afghanistan/" - } - ], - "object_marking_refs": [ - "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168" - ] + "type": "malware", + "id": "malware--6b616fc1-1505-48e3-8b2c-0d19337bff38", + "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5", + "created": "2017-05-31T21:32:58.226Z", + "modified": "2018-11-01T23:24:48.457Z", + "name": "Rover", + "description": "Rover is malware suspected of being used for espionage purposes. It was used in 2015 in a targeted email sent to an Indian Ambassador to Afghanistan.[[Citation: Palo Alto Rover]]", + "labels": [ + "version three" + ], + "external_references": [ + { + "source_name": "mitre-attack", + "url": "https://attack.mitre.org/wiki/Software/S0090", + "external_id": "S0090" + }, + { + "source_name": "Palo Alto Rover", + "description": "Ray, V., Hayashi, K. (2016, February 29). New Malware \u2018Rover\u2019 Targets Indian Ambassador to Afghanistan. Retrieved February 29, 2016.", + "url": "http://researchcenter.paloaltonetworks.com/2016/02/new-malware-rover-targets-indian-ambassador-to-afghanistan/" + } + ], + "object_marking_refs": [ + "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168" + ] } From f20ee915447396c730c198a06bc99bb5608c5105 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Mon, 10 Dec 2018 15:23:26 -0500 Subject: [PATCH 18/18] rename 'STIXPatternVisitor' to 'pattern_visitor' --- stix2/{STIXPatternVisitor.py => pattern_visitor.py} | 0 stix2/test/test_pattern_expressions.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename stix2/{STIXPatternVisitor.py => pattern_visitor.py} (100%) diff --git a/stix2/STIXPatternVisitor.py b/stix2/pattern_visitor.py similarity index 100% rename from stix2/STIXPatternVisitor.py rename to stix2/pattern_visitor.py diff --git a/stix2/test/test_pattern_expressions.py b/stix2/test/test_pattern_expressions.py index 62dc242..ac98489 100644 --- a/stix2/test/test_pattern_expressions.py +++ b/stix2/test/test_pattern_expressions.py @@ -3,7 +3,7 @@ import datetime import pytest import stix2 -from stix2.STIXPatternVisitor import create_pattern_object +from stix2.pattern_visitor import create_pattern_object def test_create_comparison_expression():