From a9de13107f20f15922759f6dc79b85539ff2f063 Mon Sep 17 00:00:00 2001 From: mbastian1135 Date: Fri, 24 Aug 2018 10:40:37 -0400 Subject: [PATCH 01/10] pattern guide --- stix2/patterns.py | 249 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 244 insertions(+), 5 deletions(-) diff --git a/stix2/patterns.py b/stix2/patterns.py index 3f9cbd9..2e963b4 100644 --- a/stix2/patterns.py +++ b/stix2/patterns.py @@ -18,6 +18,11 @@ class _Constant(object): class StringConstant(_Constant): + """Pattern string constant + + Args: + value (str): string value + """ def __init__(self, value): self.value = value @@ -26,6 +31,11 @@ class StringConstant(_Constant): class TimestampConstant(_Constant): + """Pattern timestamp constant + + Args: + value (datetime.datetime OR str): if string, must be a timestamp string + """ def __init__(self, value): try: self.value = parse_into_datetime(value) @@ -37,6 +47,12 @@ class TimestampConstant(_Constant): class IntegerConstant(_Constant): + """Pattern interger constant + + Args: + value (int): integer value + + """ def __init__(self, value): try: self.value = int(value) @@ -59,6 +75,13 @@ class FloatConstant(_Constant): class BooleanConstant(_Constant): + """Pattern boolean constant + + Args: + value (str OR int): + (str) 'true', 't' for True; 'false', 'f' for False + (int) 1 for True; 0 for False + """ def __init__(self, value): if isinstance(value, bool): self.value = value @@ -106,6 +129,15 @@ _HASH_REGEX = { class HashConstant(StringConstant): + """Pattern hash constant + + Args: + value (str): hash value + type (str): hash algorithm name + supported hash algorithms: "MD5", "MD6", RIPEMD160", "SHA1", + "SHA224", "SHA256", "SHA384", "SHA512", "SHA3224", "SHA3256", + "SHA3384", "SHA3512", "SSDEEP", "WHIRLPOOL" + """ def __init__(self, value, type): key = type.upper().replace('-', '') if key in _HASH_REGEX: @@ -116,7 +148,11 @@ class HashConstant(StringConstant): class BinaryConstant(_Constant): + """Pattern binary constant + Args: + value (str): base64 encoded string value + """ def __init__(self, value): try: base64.b64decode(value) @@ -129,6 +165,11 @@ class BinaryConstant(_Constant): class HexConstant(_Constant): + """Pattern hexadecimal constant + + Args: + value (str): hexadecimal value + """ def __init__(self, value): if not re.match('^([a-fA-F0-9]{2})+$', value): raise ValueError("must contain an even number of hexadecimal characters") @@ -139,6 +180,11 @@ class HexConstant(_Constant): class ListConstant(_Constant): + """Pattern list constant + + Args: + value (list): list of values + """ def __init__(self, values): self.value = values @@ -147,6 +193,12 @@ class ListConstant(_Constant): def make_constant(value): + """Convert value to Pattern constant, best effort attempt + at determining root value type and corresponding conversion + + Args: + value (): value to convert to Pattern constant + """ if isinstance(value, _Constant): return value @@ -182,6 +234,16 @@ class _ObjectPathComponent(object): class BasicObjectPathComponent(_ObjectPathComponent): + """Basic object path component (for a observation or expression) + + By "Basic", implies that the object path component is not a + list, object reference or futher referenced property, i.e. terminal + component + + Args: + property_name (str): object property name + is_key (): is dictionary key, default: False + """ def __init__(self, property_name, is_key=False): self.property_name = property_name # TODO: set is_key to True if this component is a dictionary key @@ -192,6 +254,13 @@ class BasicObjectPathComponent(_ObjectPathComponent): class ListObjectPathComponent(_ObjectPathComponent): + """List object path component (for an observation or expression) + + + Args: + property_name (): list object property name + index (): index of the list property's value that is specified + """ def __init__(self, property_name, index): self.property_name = property_name self.index = index @@ -201,6 +270,11 @@ class ListObjectPathComponent(_ObjectPathComponent): class ReferenceObjectPathComponent(_ObjectPathComponent): + """Reference object path component (for an observation or expression) + + Args: + reference_property_name (str): reference object property name + """ def __init__(self, reference_property_name): self.property_name = reference_property_name @@ -209,6 +283,12 @@ class ReferenceObjectPathComponent(_ObjectPathComponent): class ObjectPath(object): + """Pattern operand object (property) path + + Args: + object_type_name (str): name of object type for corresponding object path component + property_path (_ObjectPathComponent OR str): Object path + """ def __init__(self, object_type_name, property_path): self.object_type_name = object_type_name self.property_path = [x if isinstance(x, _ObjectPathComponent) else @@ -219,11 +299,17 @@ class ObjectPath(object): return "%s:%s" % (self.object_type_name, ".".join(["%s" % x for x in self.property_path])) def merge(self, other): + """Extend the object property with that of the supplied object property path""" self.property_path.extend(other.property_path) return self @staticmethod def make_object_path(lhs): + """Create ObjectPath from string encoded object path + + Args: + lhs (str): object path of left-hand-side component of expression + """ path_as_parts = lhs.split(":") return ObjectPath(path_as_parts[0], path_as_parts[1].split(".")) @@ -233,6 +319,14 @@ class _PatternExpression(object): class _ComparisonExpression(_PatternExpression): + """Pattern Comparison Expression + + Args: + operator (str): operator of comparison expression + lhs (ObjectPath OR str): object path of left-hand-side component of expression + rhs (ObjectPath OR str): object path of right-hand-side component of expression + negated (bool): comparison expression negated. Default: False + """ def __init__(self, operator, lhs, rhs, negated=False): if operator == "=" and isinstance(rhs, (ListConstant, list)): self.operator = "IN" @@ -257,58 +351,139 @@ class _ComparisonExpression(_PatternExpression): class EqualityComparisonExpression(_ComparisonExpression): + """Pattern Equality Comparison Expression + + Args: + lhs (ObjectPath OR str): object path of left-hand-side component of expression + rhs (ObjectPath OR str): object path of right-hand-side component of expression + negated (bool): comparison expression negated. Default: False + """ def __init__(self, lhs, rhs, negated=False): super(EqualityComparisonExpression, self).__init__("=", lhs, rhs, negated) class GreaterThanComparisonExpression(_ComparisonExpression): + """Pattern Greater-than Comparison Expression + + Args: + lhs (ObjectPath OR str): object path of left-hand-side component of expression + rhs (ObjectPath OR str): object path of right-hand-side component of expression + negated (bool): comparison expression negated. Default: False + """ def __init__(self, lhs, rhs, negated=False): super(GreaterThanComparisonExpression, self).__init__(">", lhs, rhs, negated) class LessThanComparisonExpression(_ComparisonExpression): + """Pattern Less-than Comparison Expression + + Args: + lhs (ObjectPath OR str): object path of left-hand-side component of expression + rhs (ObjectPath OR str): object path of right-hand-side component of expression + negated (bool): comparison expression negated. Default: False + """ def __init__(self, lhs, rhs, negated=False): super(LessThanComparisonExpression, self).__init__("<", lhs, rhs, negated) class GreaterThanEqualComparisonExpression(_ComparisonExpression): + """Pattern Greater-Than-or-Equal-to Comparison Expression + + Args: + lhs (ObjectPath OR str): object path of left-hand-side component of expression + rhs (ObjectPath OR str): object path of right-hand-side component of expression + negated (bool): comparison expression negated. Default: False + """ def __init__(self, lhs, rhs, negated=False): super(GreaterThanEqualComparisonExpression, self).__init__(">=", lhs, rhs, negated) class LessThanEqualComparisonExpression(_ComparisonExpression): + """Pattern Less-Than-or-Equal-to Comparison Expression + + Args: + lhs (ObjectPath OR str): object path of left-hand-side component of expression + rhs (ObjectPath OR str): object path of right-hand-side component of expression + negated (bool): comparison expression negated. Default: False + """ + def __init__(self, lhs, rhs, negated=False): super(LessThanEqualComparisonExpression, self).__init__("<=", lhs, rhs, negated) class InComparisonExpression(_ComparisonExpression): + """'in' Comparison Expression + + Args: + lhs (ObjectPath OR str): object path of left-hand-side component of expression + rhs (ObjectPath OR str): object path of right-hand-side component of expression + negated (bool): comparison expression negated. Default: False + """ def __init__(self, lhs, rhs, negated=False): super(InComparisonExpression, self).__init__("IN", lhs, rhs, negated) class LikeComparisonExpression(_ComparisonExpression): + """'in' Comparison Expression + + Args: + lhs (ObjectPath OR str): object path of left-hand-side component of expression + rhs (ObjectPath OR str): object path of right-hand-side component of expression + negated (bool): comparison expression negated. Default: False + """ + def __init__(self, lhs, rhs, negated=False): super(LikeComparisonExpression, self).__init__("LIKE", lhs, rhs, negated) class MatchesComparisonExpression(_ComparisonExpression): + """Matches Comparison Expression + + Args: + lhs (ObjectPath OR str): object path of left-hand-side component of expression + rhs (ObjectPath OR str): object path of right-hand-side component of expression + negated (bool): comparison expression negated. Default: False + """ + def __init__(self, lhs, rhs, negated=False): super(MatchesComparisonExpression, self).__init__("MATCHES", lhs, rhs, negated) class IsSubsetComparisonExpression(_ComparisonExpression): - def __init__(self, lhs, rhs, negated=False): - super(IsSubsetComparisonExpression, self).__init__("ISSUBSET", lhs, rhs, negated) + """ 'is subset' Comparison Expression + + Args: + lhs (ObjectPath OR str): object path of left-hand-side component of expression + rhs (ObjectPath OR str): object path of right-hand-side component of expression + negated (bool): comparison expression negated. Default: False + """ + + def __init__(self, lhs, rhs, negated=False): + super(IsSubsetComparisonExpression, self).__init__("ISSUBSET", lhs, rhs, negated) class IsSupersetComparisonExpression(_ComparisonExpression): - def __init__(self, lhs, rhs, negated=False): - super(IsSupersetComparisonExpression, self).__init__("ISSUPERSET", lhs, rhs, negated) + """ 'is super set' Comparison Expression + + Args: + lhs (ObjectPath OR str): object path of left-hand-side component of expression + rhs (ObjectPath OR str): object path of right-hand-side component of expression + negated (bool): comparison expression negated. Default: False + """ + + def __init__(self, lhs, rhs, negated=False): + super(IsSupersetComparisonExpression, self).__init__("ISSUPERSET", lhs, rhs, negated) class _BooleanExpression(_PatternExpression): + """Pattern Boolean Expression + + Args: + operator (str): boolean operator + operands (list): boolean operands + """ def __init__(self, operator, operands): - self.operator = operator + self.operator = operator self.operands = [] for arg in operands: if not hasattr(self, "root_type"): @@ -327,16 +502,31 @@ class _BooleanExpression(_PatternExpression): class AndBooleanExpression(_BooleanExpression): + """Pattern 'AND' Boolean Expression + + Args: + operands (list): AND operands + """ def __init__(self, operands): super(AndBooleanExpression, self).__init__("AND", operands) class OrBooleanExpression(_BooleanExpression): + """Pattern 'OR' Boolean Expression + + Args: + operands (list): OR operands + """ def __init__(self, operands): super(OrBooleanExpression, self).__init__("OR", operands) class ObservationExpression(_PatternExpression): + """Observation Expression + + Args: + operand (str): observation expression operand + """ def __init__(self, operand): self.operand = operand @@ -345,6 +535,12 @@ class ObservationExpression(_PatternExpression): class _CompoundObservationExpression(_PatternExpression): + """Compound Observation Expression + + Args: + operator (str): compound observation operator + operands (str): compound observation operands + """ def __init__(self, operator, operands): self.operator = operator self.operands = operands @@ -357,21 +553,42 @@ class _CompoundObservationExpression(_PatternExpression): class AndObservationExpression(_CompoundObservationExpression): + """Pattern 'AND' Compound Observation Expression + + Args: + operands (str): compound observation operands + """ + def __init__(self, operands): super(AndObservationExpression, self).__init__("AND", operands) class OrObservationExpression(_CompoundObservationExpression): + """Pattern 'OR' Compound Observation Expression + + Args: + operands (str): compound observation operands + """ def __init__(self, operands): super(OrObservationExpression, self).__init__("OR", operands) class FollowedByObservationExpression(_CompoundObservationExpression): + """Pattern 'Followed by' Compound Observation Expression + + Args: + operands (str): compound observation operands + """ def __init__(self, operands): super(FollowedByObservationExpression, self).__init__("FOLLOWEDBY", operands) class ParentheticalExpression(_PatternExpression): + """Pattern Parenthetical Observation Expression + + Args: + exp (str): observation expression + """ def __init__(self, exp): self.expression = exp if hasattr(exp, "root_type"): @@ -386,6 +603,11 @@ class _ExpressionQualifier(_PatternExpression): class RepeatQualifier(_ExpressionQualifier): + """Pattern Repeat Qualifier + + Args: + times_to_repeat (int): times the qualifiers is repeated + """ def __init__(self, times_to_repeat): if isinstance(times_to_repeat, IntegerConstant): self.times_to_repeat = times_to_repeat @@ -399,6 +621,11 @@ class RepeatQualifier(_ExpressionQualifier): class WithinQualifier(_ExpressionQualifier): + """Pattern 'Within' Qualifier + + Args: + number_of_seconds (int): seconds value for 'within' qualifier + """ def __init__(self, number_of_seconds): if isinstance(number_of_seconds, IntegerConstant): self.number_of_seconds = number_of_seconds @@ -412,6 +639,12 @@ class WithinQualifier(_ExpressionQualifier): class StartStopQualifier(_ExpressionQualifier): + """Pattern Start/Stop Qualifier + + Args: + start_time (TimestampConstant OR datetime.date): start timestamp for qualifier + stop_time (TimestampConstant OR datetime.date): stop timestamp for qualifier + """ def __init__(self, start_time, stop_time): if isinstance(start_time, TimestampConstant): self.start_time = start_time @@ -431,6 +664,12 @@ class StartStopQualifier(_ExpressionQualifier): class QualifiedObservationExpression(_PatternExpression): + """Pattern Qualified Observation Expression + + Args: + observation_expression (PatternExpression OR _CompoundObservationExpression OR ): pattern expression + qualifier (_ExpressionQualifier): pattern expression qualifier + """ def __init__(self, observation_expression, qualifier): self.observation_expression = observation_expression self.qualifier = qualifier From b5a301ff28b6a73054741d5cd9a90a0036a7f6e5 Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Mon, 27 Aug 2018 13:56:49 -0400 Subject: [PATCH 02/10] Update list of maintainers --- README.rst | 3 --- 1 file changed, 3 deletions(-) diff --git a/README.rst b/README.rst index 79b8e7e..6770ec6 100644 --- a/README.rst +++ b/README.rst @@ -170,9 +170,6 @@ repositories/maintainers-guide#additionalMaintainers>`__. **Current Maintainers of this TC Open Repository** -- `Greg Back `__; GitHub ID: - https://github.com/gtback/; WWW: `MITRE - Corporation `__ - `Chris Lenk `__; GitHub ID: https://github.com/clenk/; WWW: `MITRE Corporation `__ From 10149b9345904be9aefcfc9af583155c683db3f0 Mon Sep 17 00:00:00 2001 From: mbastian1135 Date: Mon, 27 Aug 2018 16:21:14 -0400 Subject: [PATCH 03/10] added 'to_pattern()' utility for pattern expressions; more information on patterning guide --- docs/guide/creating.ipynb | 2 +- stix2/patterns.py | 53 +++++++++++++++++++++++++-------------- 2 files changed, 35 insertions(+), 20 deletions(-) diff --git a/docs/guide/creating.ipynb b/docs/guide/creating.ipynb index 61bbe15..058aae3 100644 --- a/docs/guide/creating.ipynb +++ b/docs/guide/creating.ipynb @@ -881,7 +881,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.3" + "version": "3.6.5" } }, "nbformat": 4, diff --git a/stix2/patterns.py b/stix2/patterns.py index 2e963b4..881effb 100644 --- a/stix2/patterns.py +++ b/stix2/patterns.py @@ -51,7 +51,6 @@ class IntegerConstant(_Constant): Args: value (int): integer value - """ def __init__(self, value): try: @@ -133,10 +132,10 @@ class HashConstant(StringConstant): Args: value (str): hash value - type (str): hash algorithm name - supported hash algorithms: "MD5", "MD6", RIPEMD160", "SHA1", - "SHA224", "SHA256", "SHA384", "SHA512", "SHA3224", "SHA3256", - "SHA3384", "SHA3512", "SSDEEP", "WHIRLPOOL" + type (str): hash algorithm name. Supported hash algorithms: + "MD5", "MD6", RIPEMD160", "SHA1", "SHA224", "SHA256", + "SHA384", "SHA512", "SHA3224", "SHA3256", "SHA3384", + "SHA3512", "SSDEEP", "WHIRLPOOL" """ def __init__(self, value, type): key = type.upper().replace('-', '') @@ -197,7 +196,7 @@ def make_constant(value): at determining root value type and corresponding conversion Args: - value (): value to convert to Pattern constant + value: value to convert to Pattern constant """ if isinstance(value, _Constant): return value @@ -234,7 +233,7 @@ class _ObjectPathComponent(object): class BasicObjectPathComponent(_ObjectPathComponent): - """Basic object path component (for a observation or expression) + """Basic object path component (for an observation or expression) By "Basic", implies that the object path component is not a list, object reference or futher referenced property, i.e. terminal @@ -242,7 +241,7 @@ class BasicObjectPathComponent(_ObjectPathComponent): Args: property_name (str): object property name - is_key (): is dictionary key, default: False + is_key (bool): is dictionary key, default: False """ def __init__(self, property_name, is_key=False): self.property_name = property_name @@ -256,10 +255,9 @@ class BasicObjectPathComponent(_ObjectPathComponent): class ListObjectPathComponent(_ObjectPathComponent): """List object path component (for an observation or expression) - Args: - property_name (): list object property name - index (): index of the list property's value that is specified + property_name (str): list object property name + index (int): index of the list property's value that is specified """ def __init__(self, property_name, index): self.property_name = property_name @@ -287,7 +285,7 @@ class ObjectPath(object): Args: object_type_name (str): name of object type for corresponding object path component - property_path (_ObjectPathComponent OR str): Object path + property_path (_ObjectPathComponent OR str): object path """ def __init__(self, object_type_name, property_path): self.object_type_name = object_type_name @@ -315,8 +313,11 @@ class ObjectPath(object): class _PatternExpression(object): - pass + def to_pattern(self): + """return a properly formatted string of the pattern expression""" + return "[{}]".format(self.__str__()) + class _ComparisonExpression(_PatternExpression): """Pattern Comparison Expression @@ -424,7 +425,7 @@ class InComparisonExpression(_ComparisonExpression): class LikeComparisonExpression(_ComparisonExpression): - """'in' Comparison Expression + """'like' Comparison Expression Args: lhs (ObjectPath OR str): object path of left-hand-side component of expression @@ -437,7 +438,7 @@ class LikeComparisonExpression(_ComparisonExpression): class MatchesComparisonExpression(_ComparisonExpression): - """Matches Comparison Expression + """'Matches' Comparison Expression Args: lhs (ObjectPath OR str): object path of left-hand-side component of expression @@ -476,7 +477,7 @@ class IsSupersetComparisonExpression(_ComparisonExpression): class _BooleanExpression(_PatternExpression): - """Pattern Boolean Expression + """Boolean Pattern Expression Args: operator (str): boolean operator @@ -502,7 +503,7 @@ class _BooleanExpression(_PatternExpression): class AndBooleanExpression(_BooleanExpression): - """Pattern 'AND' Boolean Expression + """'AND' Boolean Pattern Expression Args: operands (list): AND operands @@ -512,7 +513,7 @@ class AndBooleanExpression(_BooleanExpression): class OrBooleanExpression(_BooleanExpression): - """Pattern 'OR' Boolean Expression + """'OR' Boolean Pattern Expression Args: operands (list): OR operands @@ -551,9 +552,14 @@ class _CompoundObservationExpression(_PatternExpression): sub_exprs.append("%s" % o) return (" " + self.operator + " ").join(sub_exprs) + def to_pattern(self): + return "{0} {1} {2}".format(self.operands[0].to_pattern(), + self.operator, + self.operands[1].to_pattern()) + class AndObservationExpression(_CompoundObservationExpression): - """Pattern 'AND' Compound Observation Expression + """'AND' Compound Observation Pattern Expression Args: operands (str): compound observation operands @@ -582,6 +588,9 @@ class FollowedByObservationExpression(_CompoundObservationExpression): def __init__(self, operands): super(FollowedByObservationExpression, self).__init__("FOLLOWEDBY", operands) + def to_pattern(self): + return "[{}] {} [{}]".format(self.operands[0], "FOLLOWEDBY", self.operands[1]) + class ParentheticalExpression(_PatternExpression): """Pattern Parenthetical Observation Expression @@ -597,6 +606,9 @@ class ParentheticalExpression(_PatternExpression): def __str__(self): return "(%s)" % self.expression + def to_pattern(self): + return "({})".format(self.expression.to_pattern()) + class _ExpressionQualifier(_PatternExpression): pass @@ -676,3 +688,6 @@ class QualifiedObservationExpression(_PatternExpression): def __str__(self): return "%s %s" % (self.observation_expression, self.qualifier) + + def to_pattern(self): + return "{} {}".format(self.observation_expression.to_pattern(), self.qualifier) From cd81d97beba72272dd74c2d3abde1baae4b7bad3 Mon Sep 17 00:00:00 2001 From: mbastian1135 Date: Mon, 27 Aug 2018 16:24:23 -0400 Subject: [PATCH 04/10] added 'to_pattern()' utility for pattern expressions; more information on patterning guide --- stix2/patterns.py | 64 +++++++++++++++++++++++------------------------ 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/stix2/patterns.py b/stix2/patterns.py index 881effb..59bd2bf 100644 --- a/stix2/patterns.py +++ b/stix2/patterns.py @@ -19,7 +19,7 @@ class _Constant(object): class StringConstant(_Constant): """Pattern string constant - + Args: value (str): string value """ @@ -40,7 +40,7 @@ class TimestampConstant(_Constant): try: self.value = parse_into_datetime(value) except Exception: - raise ValueError("must be a datetime object or timestamp string.") + raise ValueError("Must be a datetime object or timestamp string.") def __str__(self): return "t%s" % repr(self.value) @@ -77,10 +77,10 @@ class BooleanConstant(_Constant): """Pattern boolean constant Args: - value (str OR int): + value (str OR int): (str) 'true', 't' for True; 'false', 'f' for False (int) 1 for True; 0 for False - """ + """ def __init__(self, value): if isinstance(value, bool): self.value = value @@ -168,7 +168,7 @@ class HexConstant(_Constant): Args: value (str): hexadecimal value - """ + """ def __init__(self, value): if not re.match('^([a-fA-F0-9]{2})+$', value): raise ValueError("must contain an even number of hexadecimal characters") @@ -235,7 +235,7 @@ class _ObjectPathComponent(object): class BasicObjectPathComponent(_ObjectPathComponent): """Basic object path component (for an observation or expression) - By "Basic", implies that the object path component is not a + By "Basic", implies that the object path component is not a list, object reference or futher referenced property, i.e. terminal component @@ -254,10 +254,10 @@ class BasicObjectPathComponent(_ObjectPathComponent): class ListObjectPathComponent(_ObjectPathComponent): """List object path component (for an observation or expression) - + Args: property_name (str): list object property name - index (int): index of the list property's value that is specified + index (int): index of the list property's value that is specified """ def __init__(self, property_name, index): self.property_name = property_name @@ -285,8 +285,8 @@ class ObjectPath(object): Args: object_type_name (str): name of object type for corresponding object path component - property_path (_ObjectPathComponent OR str): object path - """ + property_path (_ObjectPathComponent OR str): object path + """ def __init__(self, object_type_name, property_path): self.object_type_name = object_type_name self.property_path = [x if isinstance(x, _ObjectPathComponent) else @@ -316,15 +316,15 @@ class _PatternExpression(object): def to_pattern(self): """return a properly formatted string of the pattern expression""" - return "[{}]".format(self.__str__()) - + return "[{}]".format(self.__str__()) + class _ComparisonExpression(_PatternExpression): """Pattern Comparison Expression Args: operator (str): operator of comparison expression - lhs (ObjectPath OR str): object path of left-hand-side component of expression + lhs (ObjectPath OR str): object path of left-hand-side component of expression rhs (ObjectPath OR str): object path of right-hand-side component of expression negated (bool): comparison expression negated. Default: False """ @@ -393,7 +393,7 @@ class GreaterThanEqualComparisonExpression(_ComparisonExpression): Args: lhs (ObjectPath OR str): object path of left-hand-side component of expression rhs (ObjectPath OR str): object path of right-hand-side component of expression - negated (bool): comparison expression negated. Default: False + negated (bool): comparison expression negated. Default: False """ def __init__(self, lhs, rhs, negated=False): super(GreaterThanEqualComparisonExpression, self).__init__(">=", lhs, rhs, negated) @@ -418,7 +418,7 @@ class InComparisonExpression(_ComparisonExpression): Args: lhs (ObjectPath OR str): object path of left-hand-side component of expression rhs (ObjectPath OR str): object path of right-hand-side component of expression - negated (bool): comparison expression negated. Default: False + negated (bool): comparison expression negated. Default: False """ def __init__(self, lhs, rhs, negated=False): super(InComparisonExpression, self).__init__("IN", lhs, rhs, negated) @@ -430,9 +430,9 @@ class LikeComparisonExpression(_ComparisonExpression): Args: lhs (ObjectPath OR str): object path of left-hand-side component of expression rhs (ObjectPath OR str): object path of right-hand-side component of expression - negated (bool): comparison expression negated. Default: False + negated (bool): comparison expression negated. Default: False """ - + def __init__(self, lhs, rhs, negated=False): super(LikeComparisonExpression, self).__init__("LIKE", lhs, rhs, negated) @@ -443,9 +443,9 @@ class MatchesComparisonExpression(_ComparisonExpression): Args: lhs (ObjectPath OR str): object path of left-hand-side component of expression rhs (ObjectPath OR str): object path of right-hand-side component of expression - negated (bool): comparison expression negated. Default: False + negated (bool): comparison expression negated. Default: False """ - + def __init__(self, lhs, rhs, negated=False): super(MatchesComparisonExpression, self).__init__("MATCHES", lhs, rhs, negated) @@ -456,9 +456,9 @@ class IsSubsetComparisonExpression(_ComparisonExpression): Args: lhs (ObjectPath OR str): object path of left-hand-side component of expression rhs (ObjectPath OR str): object path of right-hand-side component of expression - negated (bool): comparison expression negated. Default: False + negated (bool): comparison expression negated. Default: False """ - + def __init__(self, lhs, rhs, negated=False): super(IsSubsetComparisonExpression, self).__init__("ISSUBSET", lhs, rhs, negated) @@ -469,9 +469,9 @@ class IsSupersetComparisonExpression(_ComparisonExpression): Args: lhs (ObjectPath OR str): object path of left-hand-side component of expression rhs (ObjectPath OR str): object path of right-hand-side component of expression - negated (bool): comparison expression negated. Default: False + negated (bool): comparison expression negated. Default: False """ - + def __init__(self, lhs, rhs, negated=False): super(IsSupersetComparisonExpression, self).__init__("ISSUPERSET", lhs, rhs, negated) @@ -480,11 +480,11 @@ class _BooleanExpression(_PatternExpression): """Boolean Pattern Expression Args: - operator (str): boolean operator - operands (list): boolean operands + operator (str): boolean operator + operands (list): boolean operands """ def __init__(self, operator, operands): - self.operator = operator + self.operator = operator self.operands = [] for arg in operands: if not hasattr(self, "root_type"): @@ -564,7 +564,7 @@ class AndObservationExpression(_CompoundObservationExpression): Args: operands (str): compound observation operands """ - + def __init__(self, operands): super(AndObservationExpression, self).__init__("AND", operands) @@ -574,7 +574,7 @@ class OrObservationExpression(_CompoundObservationExpression): Args: operands (str): compound observation operands - """ + """ def __init__(self, operands): super(OrObservationExpression, self).__init__("OR", operands) @@ -584,7 +584,7 @@ class FollowedByObservationExpression(_CompoundObservationExpression): Args: operands (str): compound observation operands - """ + """ def __init__(self, operands): super(FollowedByObservationExpression, self).__init__("FOLLOWEDBY", operands) @@ -619,7 +619,7 @@ class RepeatQualifier(_ExpressionQualifier): Args: times_to_repeat (int): times the qualifiers is repeated - """ + """ def __init__(self, times_to_repeat): if isinstance(times_to_repeat, IntegerConstant): self.times_to_repeat = times_to_repeat @@ -656,7 +656,7 @@ class StartStopQualifier(_ExpressionQualifier): Args: start_time (TimestampConstant OR datetime.date): start timestamp for qualifier stop_time (TimestampConstant OR datetime.date): stop timestamp for qualifier - """ + """ def __init__(self, start_time, stop_time): if isinstance(start_time, TimestampConstant): self.start_time = start_time @@ -680,7 +680,7 @@ class QualifiedObservationExpression(_PatternExpression): Args: observation_expression (PatternExpression OR _CompoundObservationExpression OR ): pattern expression - qualifier (_ExpressionQualifier): pattern expression qualifier + qualifier (_ExpressionQualifier): pattern expression qualifier """ def __init__(self, observation_expression, qualifier): self.observation_expression = observation_expression From 0ef9060ed09288950843fe899ea59bc0558295fe Mon Sep 17 00:00:00 2001 From: mbastian1135 Date: Mon, 27 Aug 2018 16:27:56 -0400 Subject: [PATCH 05/10] added 'to_pattern()' utility for pattern expressions; more information on patterning guide --- docs/guide/patterns.ipynb | 332 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 332 insertions(+) create mode 100644 docs/guide/patterns.ipynb diff --git a/docs/guide/patterns.ipynb b/docs/guide/patterns.ipynb new file mode 100644 index 0000000..0f79f7f --- /dev/null +++ b/docs/guide/patterns.ipynb @@ -0,0 +1,332 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# STIX Patterns\n", + "\n", + "python-stix2 supports STIX2 patterning insofar that patterns may be used for the pattern property of Indicators, identical to the STIX2 specification. python-stix2 does not evaluate patterns against STIX2 content, for that functionality see [cti-pattern-matcher](https://github.com/oasis-open/cti-pattern-matcher)\n", + "\n", + "## Intro\n", + "\n", + "python-stix2 patterns are built compositely from the bottom up, creating components at sublevels before those at higher levels.\n", + "\n", + "## Examples\n", + "\n", + "### Comparison Expressions" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Equality Comparison Expressions:\n", + "\n", + "\t[domain-name:value = 'site.of.interest.zaz']\n", + "\n", + "\t[file:parent_directory_ref.path = 'C:\\\\Windows\\\\System32']\n", + "\n", + "\n", + "Greater-than Comparison Expression:\n", + "\n", + "\t[file:extensions.windows-pebinary-ext.sections[*].entropy > 7.0]\n", + "\n", + "\n", + "Is-Subset Comparison Expression:\n", + "\n", + "\t[network-traffic:dst_ref.value ISSUBSET '2001:0db8:dead:beef:0000:0000:0000:0000/64']\n", + "\n" + ] + } + ], + "source": [ + "from stix2 import DomainName, File, IPv4Address\n", + "from stix2 import (ObjectPath, EqualityComparisonExpression, GreaterThanComparisonExpression,\n", + " IsSubsetComparisonExpression, FloatConstant, StringConstant)\n", + "\n", + "# ---- Equality Comparison expressions\n", + "print(\"Equality Comparison Expressions:\\n\")\n", + "\n", + "lhs = ObjectPath(\"domain-name\", [\"value\"])\n", + "ece_1 = EqualityComparisonExpression(lhs, \"site.of.interest.zaz\")\n", + "print(\"\\t{}\\n\".format(ece_1.to_pattern()))\n", + "\n", + "lhs = ObjectPath(\"file\", [\"parent_directory_ref\",\"path\"])\n", + "ece_2 = EqualityComparisonExpression(lhs, \"C:\\\\Windows\\\\System32\")\n", + "print(\"\\t{}\\n\".format(ece_2.to_pattern()))\n", + "\n", + "# Greater-than Comparison expressions\n", + "print(\"\\nGreater-than Comparison Expression:\\n\")\n", + "\n", + "lhs = ObjectPath(\"file\", [\"extensions\", \"windows-pebinary-ext\", \"sections[*]\", \"entropy\"])\n", + "gte = GreaterThanComparisonExpression(lhs, FloatConstant(\"7.0\"))\n", + "print(\"\\t{}\\n\".format(gte.to_pattern()))\n", + "\n", + "# IsSubset Comparison expressions\n", + "print(\"\\nIs-Subset Comparison Expression:\\n\")\n", + "\n", + "lhs = ObjectPath(\"network-traffic\", [\"dst_ref\", \"value\"])\n", + "iss = IsSubsetComparisonExpression(lhs, StringConstant(\"2001:0db8:dead:beef:0000:0000:0000:0000/64\"))\n", + "print(\"\\t{}\\n\".format(iss.to_pattern()))\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Observation Expressions" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Observation Expressions:\n", + "\n", + "(AND)\n", + "[email-message:sender_ref.value = 'stark@example.com' AND email-message:subject = 'Conference Info']\n", + "\n", + "(OR)\n", + "[url:value = 'http://example.com/foo' OR url:value = 'http://example.com/bar']\n", + "\n", + "(OR,AND)\n", + "[(file:name = 'pdf.exe' OR file:size = 371712) AND file:created = 2014-01-13 07:03:17+00:00]\n", + "\n", + "(AND,OR,OR)\n", + "([file:name = 'foo.dll'] AND [win-registry-key:key = 'HKEY_LOCAL_MACHINE\\\\foo\\\\bar']) OR [process:name = 'fooproc' OR process:name = 'procfoo']\n", + "\n", + "(FollowedBy)\n", + "[file:hashes.MD5 = '79054025255fb1a26e4bc422aef54eb4'] FOLLOWEDBY [win-registry-key:key = 'HKEY_LOCAL_MACHINE\\\\foo\\\\bar']\n", + "\n" + ] + } + ], + "source": [ + "from stix2 import (IntegerConstant, HashConstant, ObjectPath,\n", + " EqualityComparisonExpression, AndBooleanExpression,\n", + " OrBooleanExpression, ParentheticalExpression,\n", + " AndObservationExpression, OrObservationExpression,\n", + " FollowedByObservationExpression)\n", + "\n", + "# ---- Observation expressions\n", + "print(\"Observation Expressions:\\n\")\n", + "\n", + "# AND boolean\n", + "ece3 = EqualityComparisonExpression(ObjectPath(\"email-message\", [\"sender_ref\", \"value\"]), \"stark@example.com\")\n", + "ece4 = EqualityComparisonExpression(ObjectPath(\"email-message\", [\"subject\"]), \"Conference Info\")\n", + "abe = AndBooleanExpression([ece3, ece4])\n", + "print(\"(AND)\\n{}\\n\".format(abe.to_pattern()))\n", + "\n", + "# OR boolean\n", + "ece5 = EqualityComparisonExpression(ObjectPath(\"url\", [\"value\"]), \"http://example.com/foo\")\n", + "ece6 = EqualityComparisonExpression(ObjectPath(\"url\", [\"value\"]), \"http://example.com/bar\")\n", + "obe = OrBooleanExpression([ece5, ece6])\n", + "print(\"(OR)\\n{}\\n\".format(obe.to_pattern()))\n", + "\n", + "# ( OR ) AND boolean\n", + "ece7 = EqualityComparisonExpression(ObjectPath(\"file\", [\"name\"]), \"pdf.exe\")\n", + "ece8 = EqualityComparisonExpression(ObjectPath(\"file\", [\"size\"]), IntegerConstant(\"371712\"))\n", + "ece9 = EqualityComparisonExpression(ObjectPath(\"file\", [\"created\"]), \"2014-01-13T07:03:17Z\")\n", + "obe1 = OrBooleanExpression([ece7, ece8])\n", + "pobe = ParentheticalExpression(obe1)\n", + "abe1 = AndBooleanExpression([pobe, ece9])\n", + "print(\"(OR,AND)\\n{}\\n\".format(abe1.to_pattern()))\n", + "\n", + "# ( AND ) OR ( OR ) observation\n", + "ece20 = EqualityComparisonExpression(ObjectPath(\"file\", [\"name\"]), \"foo.dll\")\n", + "ece21 = EqualityComparisonExpression(ObjectPath(\"win-registry-key\", [\"key\"]), \"HKEY_LOCAL_MACHINE\\\\foo\\\\bar\")\n", + "ece22 = EqualityComparisonExpression(ObjectPath(\"process\", [\"name\"]), \"fooproc\")\n", + "ece23 = EqualityComparisonExpression(ObjectPath(\"process\", [\"name\"]), \"procfoo\")\n", + "# NOTE: we need to use AND/OR observation expression instead of just boolean \n", + "# expressions as the operands are not on the same object-type\n", + "aoe = ParentheticalExpression(AndObservationExpression([ece20, ece21]))\n", + "obe2 = OrBooleanExpression([ece22, ece23])\n", + "ooe = OrObservationExpression([aoe, obe2])\n", + "print(\"(AND,OR,OR)\\n{}\\n\".format(ooe.to_pattern()))\n", + "\n", + "# FOLLOWED-BY\n", + "ece10 = EqualityComparisonExpression(ObjectPath(\"file\", [\"hashes\", \"MD5\"]), HashConstant(\"79054025255fb1a26e4bc422aef54eb4\", \"MD5\"))\n", + "ece11 = EqualityComparisonExpression(ObjectPath(\"win-registry-key\", [\"key\"]), \"HKEY_LOCAL_MACHINE\\\\foo\\\\bar\")\n", + "fbe = FollowedByObservationExpression([ece10, ece11])\n", + "print(\"(FollowedBy)\\n{}\\n\".format(fbe.to_pattern()))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__See [note](#BooleanExpressions-vs-CompoundObservationExpressions) on when to use BooleanExpressions vs CompoundObservationExpressions__" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Qualified Observation Expressions" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Qualified Observation Expressions:\n", + "\n", + "(WITHIN)\n", + "[file:hashes.MD5 = '79054025255fb1a26e4bc422aef54eb4'] FOLLOWEDBY [win-registry-key:key = 'HKEY_LOCAL_MACHINE\\\\foo\\\\bar'] WITHIN 300 SECONDS\n", + "\n", + "(REPEAT, WITHIN)\n", + "[network-traffic:dst_ref.type = 'domain-name' AND network-traffic:dst_ref.value = 'example.com'] REPEATS 5 TIMES WITHIN 180 SECONDS\n", + "\n", + "(START-STOP)\n", + "[file:name = 'foo.dll'] START t'2016-06-01T00:00:00Z' STOP t'2016-07-01T00:00:00Z'\n", + "\n" + ] + } + ], + "source": [ + "from stix2 import (TimestampConstant, HashConstant, ObjectPath, EqualityComparisonExpression,\n", + " AndBooleanExpression, WithinQualifier, RepeatQualifier, StartStopQualifier,\n", + " QualifiedObservationExpression, FollowedByObservationExpression)\n", + "\n", + "# Qualified Observation Expressions\n", + "print(\"Qualified Observation Expressions:\\n\")\n", + "\n", + "# WITHIN\n", + "ece10 = EqualityComparisonExpression(ObjectPath(\"file\", [\"hashes\", \"MD5\"]), HashConstant(\"79054025255fb1a26e4bc422aef54eb4\", \"MD5\"))\n", + "ece11 = EqualityComparisonExpression(ObjectPath(\"win-registry-key\", [\"key\"]), \"HKEY_LOCAL_MACHINE\\\\foo\\\\bar\")\n", + "fbe = FollowedByObservationExpression([ece10, ece11])\n", + "qoe = QualifiedObservationExpression(fbe, WithinQualifier(300))\n", + "print(\"(WITHIN)\\n{}\\n\".format(qoe.to_pattern()))\n", + "\n", + "# REPEATS, WITHIN\n", + "ece12 = EqualityComparisonExpression(ObjectPath(\"network-traffic\", [\"dst_ref\", \"type\"]), \"domain-name\")\n", + "ece13 = EqualityComparisonExpression(ObjectPath(\"network-traffic\", [\"dst_ref\", \"value\"]), \"example.com\")\n", + "abe2 = AndBooleanExpression([ece12, ece13])\n", + "qoe1 = QualifiedObservationExpression(QualifiedObservationExpression(abe2, RepeatQualifier(5)), WithinQualifier(180))\n", + "print(\"(REPEAT, WITHIN)\\n{}\\n\".format(qoe1.to_pattern()))\n", + "\n", + "# START, STOP\n", + "ece14 = EqualityComparisonExpression(ObjectPath(\"file\", [\"name\"]), \"foo.dll\")\n", + "ssq = StartStopQualifier(TimestampConstant('2016-06-01T00:00:00Z'), TimestampConstant('2016-07-01T00:00:00Z'))\n", + "qoe2 = QualifiedObservationExpression(ece14, ssq)\n", + "print(\"(START-STOP)\\n{}\\n\".format(qoe2.to_pattern()))\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Attaching patterns to STIX2 Domain objects\n", + "\n", + "As seen in the previous examples, the *__to_pattern()__* utility is used to display the pattern. However, the true purpose of this utility is to be able to seamlessly add a constructed pattern to an python-stix2 object.\n", + "\n", + "The *__to_pattern()__* utility constructs a string version of the pattern that adheres to the STIX2 patterning language specification.\n", + "\n", + "### Example" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\n", + " \"type\": \"indicator\",\n", + " \"id\": \"indicator--9f2d8014-3c2f-4127-bc2c-a209913404a5\",\n", + " \"created\": \"2018-08-27T18:57:00.762Z\",\n", + " \"modified\": \"2018-08-27T18:57:00.762Z\",\n", + " \"name\": \"Cryptotorch\",\n", + " \"pattern\": \"[file:name = '$$t00rzch$$.elf']\",\n", + " \"valid_from\": \"2018-08-27T18:57:00.762972Z\",\n", + " \"labels\": [\n", + " \"malware\",\n", + " \"ransomware\"\n", + " ]\n", + "}\n" + ] + } + ], + "source": [ + "from stix2 import Indicator, EqualityComparisonExpression\n", + "\n", + "ece14 = EqualityComparisonExpression(ObjectPath(\"file\", [\"name\"]), \"$$t00rzch$$.elf\")\n", + "ind = Indicator(name=\"Cryptotorch\", labels=[\"malware\", \"ransomware\"], pattern=\"[{}]\".format(ece14))\n", + "print(ind)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## BooleanExpressions vs CompoundObservationExpressions\n", + "\n", + "Be careful to note the difference between these two very similar pattern components. \n", + "\n", + "__BooleanExpressions__\n", + " - stix2.AndBooleanExpression\n", + " - stix2.booleanExpression\n", + " \n", + " __Usage__: When the boolean sub-expressions refer to the same root object \n", + "\n", + " __Example__:\n", + " ```[domain-name:value = \"www.5z8.info\" AND domain-name:resolvess_to_refs[*].value = \"'198.51.100.1/32'\"]```\n", + " \n", + " __Rendering__: when pattern is rendered, brackets or parenthesis will encapsulate boolean expression\n", + " \n", + "__CompoundObservationExpressions__\n", + " - stix2.AndObservationExpression\n", + " - stix2.OrObservationExpression\n", + " \n", + " __Usage__: When the boolean sub-expressions refer to different root objects\n", + "\n", + " __Example__:\n", + " ```[file:name=\"foo.dll\"] AND [process:name = \"procfoo\"]```\n", + " \n", + " __Rendering__: when pattern is rendered, brackets will encapsulate each boolean sub-expression\n", + " \n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.5" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From 27dbaa5a65da0bedd7ea8081e051642426175b81 Mon Sep 17 00:00:00 2001 From: mbastian1135 Date: Mon, 27 Aug 2018 16:51:02 -0400 Subject: [PATCH 06/10] guide edits --- docs/guide/patterns.ipynb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/guide/patterns.ipynb b/docs/guide/patterns.ipynb index 0f79f7f..87bcba7 100644 --- a/docs/guide/patterns.ipynb +++ b/docs/guide/patterns.ipynb @@ -6,11 +6,11 @@ "source": [ "# STIX Patterns\n", "\n", - "python-stix2 supports STIX2 patterning insofar that patterns may be used for the pattern property of Indicators, identical to the STIX2 specification. python-stix2 does not evaluate patterns against STIX2 content, for that functionality see [cti-pattern-matcher](https://github.com/oasis-open/cti-pattern-matcher)\n", + "Python-stix2 supports STIX2 patterning insofar that patterns may be used for the pattern property of Indicators, identical to the STIX2 specification. Python-stix2 does not evaluate patterns against STIX2 content, for that functionality see [cti-pattern-matcher].(https://github.com/oasis-open/cti-pattern-matcher)\n", "\n", "## Intro\n", "\n", - "python-stix2 patterns are built compositely from the bottom up, creating components at sublevels before those at higher levels.\n", + "Python-stix2 patterns are built compositely from the bottom up, creating components at sublevels before those at higher levels.\n", "\n", "## Examples\n", "\n", From 5648d2bc45ec7937f991207acdacef20728c2957 Mon Sep 17 00:00:00 2001 From: mbastian1135 Date: Thu, 30 Aug 2018 11:57:20 -0400 Subject: [PATCH 07/10] added to pattern guide; reverting pattern str conversion (as was already proper) --- docs/guide/patterns.ipynb | 533 +++++++++++++------------ stix2/patterns.py | 30 +- stix2/test/test_pattern_expressions.py | 21 +- 3 files changed, 282 insertions(+), 302 deletions(-) diff --git a/docs/guide/patterns.ipynb b/docs/guide/patterns.ipynb index 87bcba7..c963407 100644 --- a/docs/guide/patterns.ipynb +++ b/docs/guide/patterns.ipynb @@ -4,282 +4,28 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# STIX Patterns\n", + "# STIX2 Patterns\n", "\n", - "Python-stix2 supports STIX2 patterning insofar that patterns may be used for the pattern property of Indicators, identical to the STIX2 specification. Python-stix2 does not evaluate patterns against STIX2 content, for that functionality see [cti-pattern-matcher].(https://github.com/oasis-open/cti-pattern-matcher)\n", + "python-stix2 supports STIX2 patterning insofar that patterns may be used for the pattern property of Indicators, identical to the STIX2 specification. python-stix2 does not evaluate patterns against STIX2 content, for that functionality see [cti-pattern-matcher](https://github.com/oasis-open/cti-pattern-matcher).\n", "\n", - "## Intro\n", "\n", - "Python-stix2 patterns are built compositely from the bottom up, creating components at sublevels before those at higher levels.\n", + "python-stix2 patterns are built compositely from the bottom up, creating subcomponent expressions first before those at higher levels.\n", "\n", - "## Examples\n", + "## API Tips\n", "\n", - "### Comparison Expressions" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Equality Comparison Expressions:\n", - "\n", - "\t[domain-name:value = 'site.of.interest.zaz']\n", - "\n", - "\t[file:parent_directory_ref.path = 'C:\\\\Windows\\\\System32']\n", - "\n", - "\n", - "Greater-than Comparison Expression:\n", - "\n", - "\t[file:extensions.windows-pebinary-ext.sections[*].entropy > 7.0]\n", - "\n", - "\n", - "Is-Subset Comparison Expression:\n", - "\n", - "\t[network-traffic:dst_ref.value ISSUBSET '2001:0db8:dead:beef:0000:0000:0000:0000/64']\n", - "\n" - ] - } - ], - "source": [ - "from stix2 import DomainName, File, IPv4Address\n", - "from stix2 import (ObjectPath, EqualityComparisonExpression, GreaterThanComparisonExpression,\n", - " IsSubsetComparisonExpression, FloatConstant, StringConstant)\n", + "### ObservationExpression\n", "\n", - "# ---- Equality Comparison expressions\n", - "print(\"Equality Comparison Expressions:\\n\")\n", + "Within the STIX2 Patterning specification, Observation Expressions denote a complete expression to be evaluated against a discrete observation. In other words, an Observation Expression must be created to apply to a single Observation instance. This is further made clear by the visual brackets(```[]```) that encapsulate an Observation Expression. Thus, whatever sub expressions that are within the Observation Expression are meant to be matched against the same Observable instance.\n", "\n", - "lhs = ObjectPath(\"domain-name\", [\"value\"])\n", - "ece_1 = EqualityComparisonExpression(lhs, \"site.of.interest.zaz\")\n", - "print(\"\\t{}\\n\".format(ece_1.to_pattern()))\n", + "This requirement manifests itself within python-stix2 via ```ObservationExpression```. When creating STIX2 observation expressions, whenever the current expression is complete, wrap it with ```ObservationExpression()```. This allows the complete pattern expression - no matter complexity - to be rendered as a proper specification adhering string. *__Of which, when pattern expressions are added to Indicator objects, the expression objects are implicitly converted to string representations__* . While the extra step may seem tedious in the construction of simple pattern expressions, this explicit marking of observation expressions becomes vital when converting the pattern expressions to strings. \n", "\n", - "lhs = ObjectPath(\"file\", [\"parent_directory_ref\",\"path\"])\n", - "ece_2 = EqualityComparisonExpression(lhs, \"C:\\\\Windows\\\\System32\")\n", - "print(\"\\t{}\\n\".format(ece_2.to_pattern()))\n", + "In all the examples, you can observe how in the process of building pattern expressions, when an Observation Expression is completed, it is wrapped with ```ObservationExpression()```.\n", "\n", - "# Greater-than Comparison expressions\n", - "print(\"\\nGreater-than Comparison Expression:\\n\")\n", + "### ParentheticalExpression\n", "\n", - "lhs = ObjectPath(\"file\", [\"extensions\", \"windows-pebinary-ext\", \"sections[*]\", \"entropy\"])\n", - "gte = GreaterThanComparisonExpression(lhs, FloatConstant(\"7.0\"))\n", - "print(\"\\t{}\\n\".format(gte.to_pattern()))\n", + "Do not be confused by the ```ParentheticalExpression``` object. It is not a distinct expression type but is also used to properly craft pattern expressions by denoting order priority and grouping of expression components. Use it in a similar manner as ```ObservationExpression```, wrapping completed subcomponent expressions with ```ParentheticalExpression()``` if explicit ordering is required. For usage examples with ```ParentheticalExpression```'s, see [here](#Compound-Observation-Expressions).\n", "\n", - "# IsSubset Comparison expressions\n", - "print(\"\\nIs-Subset Comparison Expression:\\n\")\n", - "\n", - "lhs = ObjectPath(\"network-traffic\", [\"dst_ref\", \"value\"])\n", - "iss = IsSubsetComparisonExpression(lhs, StringConstant(\"2001:0db8:dead:beef:0000:0000:0000:0000/64\"))\n", - "print(\"\\t{}\\n\".format(iss.to_pattern()))\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Observation Expressions" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Observation Expressions:\n", - "\n", - "(AND)\n", - "[email-message:sender_ref.value = 'stark@example.com' AND email-message:subject = 'Conference Info']\n", - "\n", - "(OR)\n", - "[url:value = 'http://example.com/foo' OR url:value = 'http://example.com/bar']\n", - "\n", - "(OR,AND)\n", - "[(file:name = 'pdf.exe' OR file:size = 371712) AND file:created = 2014-01-13 07:03:17+00:00]\n", - "\n", - "(AND,OR,OR)\n", - "([file:name = 'foo.dll'] AND [win-registry-key:key = 'HKEY_LOCAL_MACHINE\\\\foo\\\\bar']) OR [process:name = 'fooproc' OR process:name = 'procfoo']\n", - "\n", - "(FollowedBy)\n", - "[file:hashes.MD5 = '79054025255fb1a26e4bc422aef54eb4'] FOLLOWEDBY [win-registry-key:key = 'HKEY_LOCAL_MACHINE\\\\foo\\\\bar']\n", - "\n" - ] - } - ], - "source": [ - "from stix2 import (IntegerConstant, HashConstant, ObjectPath,\n", - " EqualityComparisonExpression, AndBooleanExpression,\n", - " OrBooleanExpression, ParentheticalExpression,\n", - " AndObservationExpression, OrObservationExpression,\n", - " FollowedByObservationExpression)\n", - "\n", - "# ---- Observation expressions\n", - "print(\"Observation Expressions:\\n\")\n", - "\n", - "# AND boolean\n", - "ece3 = EqualityComparisonExpression(ObjectPath(\"email-message\", [\"sender_ref\", \"value\"]), \"stark@example.com\")\n", - "ece4 = EqualityComparisonExpression(ObjectPath(\"email-message\", [\"subject\"]), \"Conference Info\")\n", - "abe = AndBooleanExpression([ece3, ece4])\n", - "print(\"(AND)\\n{}\\n\".format(abe.to_pattern()))\n", - "\n", - "# OR boolean\n", - "ece5 = EqualityComparisonExpression(ObjectPath(\"url\", [\"value\"]), \"http://example.com/foo\")\n", - "ece6 = EqualityComparisonExpression(ObjectPath(\"url\", [\"value\"]), \"http://example.com/bar\")\n", - "obe = OrBooleanExpression([ece5, ece6])\n", - "print(\"(OR)\\n{}\\n\".format(obe.to_pattern()))\n", - "\n", - "# ( OR ) AND boolean\n", - "ece7 = EqualityComparisonExpression(ObjectPath(\"file\", [\"name\"]), \"pdf.exe\")\n", - "ece8 = EqualityComparisonExpression(ObjectPath(\"file\", [\"size\"]), IntegerConstant(\"371712\"))\n", - "ece9 = EqualityComparisonExpression(ObjectPath(\"file\", [\"created\"]), \"2014-01-13T07:03:17Z\")\n", - "obe1 = OrBooleanExpression([ece7, ece8])\n", - "pobe = ParentheticalExpression(obe1)\n", - "abe1 = AndBooleanExpression([pobe, ece9])\n", - "print(\"(OR,AND)\\n{}\\n\".format(abe1.to_pattern()))\n", - "\n", - "# ( AND ) OR ( OR ) observation\n", - "ece20 = EqualityComparisonExpression(ObjectPath(\"file\", [\"name\"]), \"foo.dll\")\n", - "ece21 = EqualityComparisonExpression(ObjectPath(\"win-registry-key\", [\"key\"]), \"HKEY_LOCAL_MACHINE\\\\foo\\\\bar\")\n", - "ece22 = EqualityComparisonExpression(ObjectPath(\"process\", [\"name\"]), \"fooproc\")\n", - "ece23 = EqualityComparisonExpression(ObjectPath(\"process\", [\"name\"]), \"procfoo\")\n", - "# NOTE: we need to use AND/OR observation expression instead of just boolean \n", - "# expressions as the operands are not on the same object-type\n", - "aoe = ParentheticalExpression(AndObservationExpression([ece20, ece21]))\n", - "obe2 = OrBooleanExpression([ece22, ece23])\n", - "ooe = OrObservationExpression([aoe, obe2])\n", - "print(\"(AND,OR,OR)\\n{}\\n\".format(ooe.to_pattern()))\n", - "\n", - "# FOLLOWED-BY\n", - "ece10 = EqualityComparisonExpression(ObjectPath(\"file\", [\"hashes\", \"MD5\"]), HashConstant(\"79054025255fb1a26e4bc422aef54eb4\", \"MD5\"))\n", - "ece11 = EqualityComparisonExpression(ObjectPath(\"win-registry-key\", [\"key\"]), \"HKEY_LOCAL_MACHINE\\\\foo\\\\bar\")\n", - "fbe = FollowedByObservationExpression([ece10, ece11])\n", - "print(\"(FollowedBy)\\n{}\\n\".format(fbe.to_pattern()))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "__See [note](#BooleanExpressions-vs-CompoundObservationExpressions) on when to use BooleanExpressions vs CompoundObservationExpressions__" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Qualified Observation Expressions" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Qualified Observation Expressions:\n", - "\n", - "(WITHIN)\n", - "[file:hashes.MD5 = '79054025255fb1a26e4bc422aef54eb4'] FOLLOWEDBY [win-registry-key:key = 'HKEY_LOCAL_MACHINE\\\\foo\\\\bar'] WITHIN 300 SECONDS\n", - "\n", - "(REPEAT, WITHIN)\n", - "[network-traffic:dst_ref.type = 'domain-name' AND network-traffic:dst_ref.value = 'example.com'] REPEATS 5 TIMES WITHIN 180 SECONDS\n", - "\n", - "(START-STOP)\n", - "[file:name = 'foo.dll'] START t'2016-06-01T00:00:00Z' STOP t'2016-07-01T00:00:00Z'\n", - "\n" - ] - } - ], - "source": [ - "from stix2 import (TimestampConstant, HashConstant, ObjectPath, EqualityComparisonExpression,\n", - " AndBooleanExpression, WithinQualifier, RepeatQualifier, StartStopQualifier,\n", - " QualifiedObservationExpression, FollowedByObservationExpression)\n", - "\n", - "# Qualified Observation Expressions\n", - "print(\"Qualified Observation Expressions:\\n\")\n", - "\n", - "# WITHIN\n", - "ece10 = EqualityComparisonExpression(ObjectPath(\"file\", [\"hashes\", \"MD5\"]), HashConstant(\"79054025255fb1a26e4bc422aef54eb4\", \"MD5\"))\n", - "ece11 = EqualityComparisonExpression(ObjectPath(\"win-registry-key\", [\"key\"]), \"HKEY_LOCAL_MACHINE\\\\foo\\\\bar\")\n", - "fbe = FollowedByObservationExpression([ece10, ece11])\n", - "qoe = QualifiedObservationExpression(fbe, WithinQualifier(300))\n", - "print(\"(WITHIN)\\n{}\\n\".format(qoe.to_pattern()))\n", - "\n", - "# REPEATS, WITHIN\n", - "ece12 = EqualityComparisonExpression(ObjectPath(\"network-traffic\", [\"dst_ref\", \"type\"]), \"domain-name\")\n", - "ece13 = EqualityComparisonExpression(ObjectPath(\"network-traffic\", [\"dst_ref\", \"value\"]), \"example.com\")\n", - "abe2 = AndBooleanExpression([ece12, ece13])\n", - "qoe1 = QualifiedObservationExpression(QualifiedObservationExpression(abe2, RepeatQualifier(5)), WithinQualifier(180))\n", - "print(\"(REPEAT, WITHIN)\\n{}\\n\".format(qoe1.to_pattern()))\n", - "\n", - "# START, STOP\n", - "ece14 = EqualityComparisonExpression(ObjectPath(\"file\", [\"name\"]), \"foo.dll\")\n", - "ssq = StartStopQualifier(TimestampConstant('2016-06-01T00:00:00Z'), TimestampConstant('2016-07-01T00:00:00Z'))\n", - "qoe2 = QualifiedObservationExpression(ece14, ssq)\n", - "print(\"(START-STOP)\\n{}\\n\".format(qoe2.to_pattern()))\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Attaching patterns to STIX2 Domain objects\n", - "\n", - "As seen in the previous examples, the *__to_pattern()__* utility is used to display the pattern. However, the true purpose of this utility is to be able to seamlessly add a constructed pattern to an python-stix2 object.\n", - "\n", - "The *__to_pattern()__* utility constructs a string version of the pattern that adheres to the STIX2 patterning language specification.\n", - "\n", - "### Example" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--9f2d8014-3c2f-4127-bc2c-a209913404a5\",\n", - " \"created\": \"2018-08-27T18:57:00.762Z\",\n", - " \"modified\": \"2018-08-27T18:57:00.762Z\",\n", - " \"name\": \"Cryptotorch\",\n", - " \"pattern\": \"[file:name = '$$t00rzch$$.elf']\",\n", - " \"valid_from\": \"2018-08-27T18:57:00.762972Z\",\n", - " \"labels\": [\n", - " \"malware\",\n", - " \"ransomware\"\n", - " ]\n", - "}\n" - ] - } - ], - "source": [ - "from stix2 import Indicator, EqualityComparisonExpression\n", - "\n", - "ece14 = EqualityComparisonExpression(ObjectPath(\"file\", [\"name\"]), \"$$t00rzch$$.elf\")\n", - "ind = Indicator(name=\"Cryptotorch\", labels=[\"malware\", \"ransomware\"], pattern=\"[{}]\".format(ece14))\n", - "print(ind)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## BooleanExpressions vs CompoundObservationExpressions\n", + "### BooleanExpressions vs CompoundObservationExpressions\n", "\n", "Be careful to note the difference between these two very similar pattern components. \n", "\n", @@ -304,7 +50,262 @@ " ```[file:name=\"foo.dll\"] AND [process:name = \"procfoo\"]```\n", " \n", " __Rendering__: when pattern is rendered, brackets will encapsulate each boolean sub-expression\n", - " \n" + "\n", + "\n", + "\n", + "## Examples\n", + "\n", + "### Comparison Expressions" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "--- Equality Comparison Expressions\n", + "\n", + "\t[domain-name:value = 'site.of.interest.zaz']\n", + "\n", + "\t[file:parent_directory_ref.path = 'C:\\\\Windows\\\\System32']\n", + "\n", + "\n", + "--- Greater-than Comparison Expression\n", + "\n", + "\t[file:extensions.windows-pebinary-ext.sections[*].entropy > 7.0]\n", + "\n", + "\n", + "--- Is-Subset Comparison Expression\n", + "\n", + "\t[network-traffic:dst_ref.value ISSUBSET '2001:0db8:dead:beef:0000:0000:0000:0000/64']\n", + "\n" + ] + } + ], + "source": [ + "from stix2 import DomainName, File, IPv4Address\n", + "from stix2 import (ObjectPath, EqualityComparisonExpression, ObservationExpression,\n", + " GreaterThanComparisonExpression, IsSubsetComparisonExpression,\n", + " FloatConstant, StringConstant)\n", + "\n", + "# ---- Equality Comparison expressions\n", + "print(\"--- Equality Comparison Expressions\\n\")\n", + "\n", + "lhs = ObjectPath(\"domain-name\", [\"value\"])\n", + "ece_1 = ObservationExpression(EqualityComparisonExpression(lhs, \"site.of.interest.zaz\"))\n", + "print(\"\\t{}\\n\".format(ece_1))\n", + "\n", + "lhs = ObjectPath(\"file\", [\"parent_directory_ref\",\"path\"])\n", + "ece_2 = ObservationExpression(EqualityComparisonExpression(lhs, \"C:\\\\Windows\\\\System32\"))\n", + "print(\"\\t{}\\n\".format(ece_2))\n", + "\n", + "# Greater-than Comparison expressions\n", + "print(\"\\n--- Greater-than Comparison Expression\\n\")\n", + "\n", + "lhs = ObjectPath(\"file\", [\"extensions\", \"windows-pebinary-ext\", \"sections[*]\", \"entropy\"])\n", + "gte = ObservationExpression(GreaterThanComparisonExpression(lhs, FloatConstant(\"7.0\")))\n", + "print(\"\\t{}\\n\".format(gte))\n", + "\n", + "# IsSubset Comparison expressions\n", + "print(\"\\n--- Is-Subset Comparison Expression\\n\")\n", + "\n", + "lhs = ObjectPath(\"network-traffic\", [\"dst_ref\", \"value\"])\n", + "iss = ObservationExpression(IsSubsetComparisonExpression(lhs, StringConstant(\"2001:0db8:dead:beef:0000:0000:0000:0000/64\")))\n", + "print(\"\\t{}\\n\".format(iss))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Compound Observation Expressions" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "--- Compound Observation Expressions\n", + "\n", + "(AND)\n", + "[email-message:sender_ref.value = 'stark@example.com' AND email-message:subject = 'Conference Info']\n", + "\n", + "(OR)\n", + "[url:value = 'http://example.com/foo' OR url:value = 'http://example.com/bar']\n", + "\n", + "(OR,AND)\n", + "[(file:name = 'pdf.exe' OR file:size = 371712) AND file:created = 2014-01-13 07:03:17+00:00]\n", + "\n", + "(AND,OR,OR)\n", + "([file:name = 'foo.dll'] AND [win-registry-key:key = 'HKEY_LOCAL_MACHINE\\\\foo\\\\bar']) OR [process:name = 'fooproc' OR process:name = 'procfoo']\n", + "\n", + "(FollowedBy)\n", + "[file:hashes.MD5 = '79054025255fb1a26e4bc422aef54eb4'] FOLLOWEDBY [win-registry-key:key = 'HKEY_LOCAL_MACHINE\\\\foo\\\\bar']\n", + "\n" + ] + } + ], + "source": [ + "from stix2 import (IntegerConstant, HashConstant, ObjectPath,\n", + " EqualityComparisonExpression, AndBooleanExpression,\n", + " OrBooleanExpression, ParentheticalExpression,\n", + " AndObservationExpression, OrObservationExpression,\n", + " FollowedByObservationExpression, ObservationExpression)\n", + "\n", + "# ---- Observation expressions\n", + "print(\"--- Compound Observation Expressions\\n\")\n", + "\n", + "# AND boolean\n", + "ece3 = EqualityComparisonExpression(ObjectPath(\"email-message\", [\"sender_ref\", \"value\"]), \"stark@example.com\")\n", + "ece4 = EqualityComparisonExpression(ObjectPath(\"email-message\", [\"subject\"]), \"Conference Info\")\n", + "abe = ObservationExpression(AndBooleanExpression([ece3, ece4]))\n", + "print(\"(AND)\\n{}\\n\".format(abe))\n", + "\n", + "# OR boolean\n", + "ece5 = EqualityComparisonExpression(ObjectPath(\"url\", [\"value\"]), \"http://example.com/foo\")\n", + "ece6 = EqualityComparisonExpression(ObjectPath(\"url\", [\"value\"]), \"http://example.com/bar\")\n", + "obe = ObservationExpression(OrBooleanExpression([ece5, ece6]))\n", + "print(\"(OR)\\n{}\\n\".format(obe))\n", + "\n", + "# ( OR ) AND boolean\n", + "ece7 = EqualityComparisonExpression(ObjectPath(\"file\", [\"name\"]), \"pdf.exe\")\n", + "ece8 = EqualityComparisonExpression(ObjectPath(\"file\", [\"size\"]), IntegerConstant(\"371712\"))\n", + "ece9 = EqualityComparisonExpression(ObjectPath(\"file\", [\"created\"]), \"2014-01-13T07:03:17Z\")\n", + "obe1 = OrBooleanExpression([ece7, ece8])\n", + "pobe = ParentheticalExpression(obe1)\n", + "abe1 = ObservationExpression(AndBooleanExpression([pobe, ece9]))\n", + "print(\"(OR,AND)\\n{}\\n\".format(abe1))\n", + "\n", + "# ( AND ) OR ( OR ) observation\n", + "ece20 = ObservationExpression(EqualityComparisonExpression(ObjectPath(\"file\", [\"name\"]), \"foo.dll\"))\n", + "ece21 = ObservationExpression(EqualityComparisonExpression(ObjectPath(\"win-registry-key\", [\"key\"]), \"HKEY_LOCAL_MACHINE\\\\foo\\\\bar\"))\n", + "ece22 = EqualityComparisonExpression(ObjectPath(\"process\", [\"name\"]), \"fooproc\")\n", + "ece23 = EqualityComparisonExpression(ObjectPath(\"process\", [\"name\"]), \"procfoo\")\n", + "# NOTE: we need to use AND/OR observation expression instead of just boolean \n", + "# expressions as the operands are not on the same object-type\n", + "aoe = ParentheticalExpression(AndObservationExpression([ece20, ece21]))\n", + "obe2 = ObservationExpression(OrBooleanExpression([ece22, ece23]))\n", + "ooe = OrObservationExpression([aoe, obe2])\n", + "print(\"(AND,OR,OR)\\n{}\\n\".format(ooe))\n", + "\n", + "# FOLLOWED-BY\n", + "ece10 = ObservationExpression(EqualityComparisonExpression(ObjectPath(\"file\", [\"hashes\", \"MD5\"]), HashConstant(\"79054025255fb1a26e4bc422aef54eb4\", \"MD5\")))\n", + "ece11 = ObservationExpression(EqualityComparisonExpression(ObjectPath(\"win-registry-key\", [\"key\"]), \"HKEY_LOCAL_MACHINE\\\\foo\\\\bar\"))\n", + "fbe = FollowedByObservationExpression([ece10, ece11])\n", + "print(\"(FollowedBy)\\n{}\\n\".format(fbe))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Qualified Observation Expressions" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "--- Qualified Observation Expressions\n", + "\n", + "(WITHIN)\n", + "([file:hashes.MD5 = '79054025255fb1a26e4bc422aef54eb4'] FOLLOWEDBY [win-registry-key:key = 'HKEY_LOCAL_MACHINE\\\\foo\\\\bar']) WITHIN 300 SECONDS\n", + "\n", + "(REPEAT, WITHIN)\n", + "[network-traffic:dst_ref.type = 'domain-name' AND network-traffic:dst_ref.value = 'example.com'] REPEATS 5 TIMES WITHIN 180 SECONDS\n", + "\n", + "(START-STOP)\n", + "[file:name = 'foo.dll'] START t'2016-06-01T00:00:00Z' STOP t'2016-07-01T00:00:00Z'\n", + "\n" + ] + } + ], + "source": [ + "from stix2 import (TimestampConstant, HashConstant, ObjectPath, EqualityComparisonExpression,\n", + " AndBooleanExpression, WithinQualifier, RepeatQualifier, StartStopQualifier,\n", + " QualifiedObservationExpression, FollowedByObservationExpression,\n", + " ParentheticalExpression, ObservationExpression)\n", + "\n", + "# Qualified Observation Expressions\n", + "print(\"--- Qualified Observation Expressions\\n\")\n", + "\n", + "# WITHIN\n", + "ece10 = ObservationExpression(EqualityComparisonExpression(ObjectPath(\"file\", [\"hashes\", \"MD5\"]), HashConstant(\"79054025255fb1a26e4bc422aef54eb4\", \"MD5\")))\n", + "ece11 = ObservationExpression(EqualityComparisonExpression(ObjectPath(\"win-registry-key\", [\"key\"]), \"HKEY_LOCAL_MACHINE\\\\foo\\\\bar\"))\n", + "fbe = FollowedByObservationExpression([ece10, ece11])\n", + "par = ParentheticalExpression(fbe)\n", + "qoe = QualifiedObservationExpression(par, WithinQualifier(300))\n", + "print(\"(WITHIN)\\n{}\\n\".format(qoe))\n", + "\n", + "# REPEATS, WITHIN\n", + "ece12 = EqualityComparisonExpression(ObjectPath(\"network-traffic\", [\"dst_ref\", \"type\"]), \"domain-name\")\n", + "ece13 = EqualityComparisonExpression(ObjectPath(\"network-traffic\", [\"dst_ref\", \"value\"]), \"example.com\")\n", + "abe2 = ObservationExpression(AndBooleanExpression([ece12, ece13]))\n", + "qoe1 = QualifiedObservationExpression(QualifiedObservationExpression(abe2, RepeatQualifier(5)), WithinQualifier(180))\n", + "print(\"(REPEAT, WITHIN)\\n{}\\n\".format(qoe1))\n", + "\n", + "# START, STOP\n", + "ece14 = ObservationExpression(EqualityComparisonExpression(ObjectPath(\"file\", [\"name\"]), \"foo.dll\"))\n", + "ssq = StartStopQualifier(TimestampConstant('2016-06-01T00:00:00Z'), TimestampConstant('2016-07-01T00:00:00Z'))\n", + "qoe2 = QualifiedObservationExpression(ece14, ssq)\n", + "print(\"(START-STOP)\\n{}\\n\".format(qoe2))\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Attaching patterns to STIX2 Domain objects\n", + "\n", + "\n", + "### Example" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\n", + " \"type\": \"indicator\",\n", + " \"id\": \"indicator--219bc5fc-fdbf-4b54-a2fc-921be7ab3acb\",\n", + " \"created\": \"2018-08-29T23:58:00.548Z\",\n", + " \"modified\": \"2018-08-29T23:58:00.548Z\",\n", + " \"name\": \"Cryptotorch\",\n", + " \"pattern\": \"[file:name = '$$t00rzch$$.elf']\",\n", + " \"valid_from\": \"2018-08-29T23:58:00.548391Z\",\n", + " \"labels\": [\n", + " \"malware\",\n", + " \"ransomware\"\n", + " ]\n", + "}\n" + ] + } + ], + "source": [ + "from stix2 import Indicator, EqualityComparisonExpression, ObservationExpression\n", + "\n", + "ece14 = ObservationExpression(EqualityComparisonExpression(ObjectPath(\"file\", [\"name\"]), \"$$t00rzch$$.elf\"))\n", + "ind = Indicator(name=\"Cryptotorch\", labels=[\"malware\", \"ransomware\"], pattern=ece14)\n", + "print(ind)" ] } ], diff --git a/stix2/patterns.py b/stix2/patterns.py index 59bd2bf..146ec04 100644 --- a/stix2/patterns.py +++ b/stix2/patterns.py @@ -313,10 +313,7 @@ class ObjectPath(object): class _PatternExpression(object): - - def to_pattern(self): - """return a properly formatted string of the pattern expression""" - return "[{}]".format(self.__str__()) + pass class _ComparisonExpression(_PatternExpression): @@ -445,7 +442,6 @@ class MatchesComparisonExpression(_ComparisonExpression): rhs (ObjectPath OR str): object path of right-hand-side component of expression negated (bool): comparison expression negated. Default: False """ - def __init__(self, lhs, rhs, negated=False): super(MatchesComparisonExpression, self).__init__("MATCHES", lhs, rhs, negated) @@ -458,7 +454,6 @@ class IsSubsetComparisonExpression(_ComparisonExpression): rhs (ObjectPath OR str): object path of right-hand-side component of expression negated (bool): comparison expression negated. Default: False """ - def __init__(self, lhs, rhs, negated=False): super(IsSubsetComparisonExpression, self).__init__("ISSUBSET", lhs, rhs, negated) @@ -471,7 +466,6 @@ class IsSupersetComparisonExpression(_ComparisonExpression): rhs (ObjectPath OR str): object path of right-hand-side component of expression negated (bool): comparison expression negated. Default: False """ - def __init__(self, lhs, rhs, negated=False): super(IsSupersetComparisonExpression, self).__init__("ISSUPERSET", lhs, rhs, negated) @@ -498,12 +492,13 @@ class _BooleanExpression(_PatternExpression): def __str__(self): sub_exprs = [] for o in self.operands: - sub_exprs.append("%s" % o) + sub_exprs.append(str(o)) return (" " + self.operator + " ").join(sub_exprs) class AndBooleanExpression(_BooleanExpression): - """'AND' Boolean Pattern Expression + """'AND' Boolean Pattern Expression. Only use if both operands are of + the same root object. Args: operands (list): AND operands @@ -513,7 +508,7 @@ class AndBooleanExpression(_BooleanExpression): class OrBooleanExpression(_BooleanExpression): - """'OR' Boolean Pattern Expression + """'OR' Boolean Pattern Expression. Only use if both operands are of the same root object Args: operands (list): OR operands @@ -552,11 +547,6 @@ class _CompoundObservationExpression(_PatternExpression): sub_exprs.append("%s" % o) return (" " + self.operator + " ").join(sub_exprs) - def to_pattern(self): - return "{0} {1} {2}".format(self.operands[0].to_pattern(), - self.operator, - self.operands[1].to_pattern()) - class AndObservationExpression(_CompoundObservationExpression): """'AND' Compound Observation Pattern Expression @@ -564,7 +554,6 @@ class AndObservationExpression(_CompoundObservationExpression): Args: operands (str): compound observation operands """ - def __init__(self, operands): super(AndObservationExpression, self).__init__("AND", operands) @@ -588,9 +577,6 @@ class FollowedByObservationExpression(_CompoundObservationExpression): def __init__(self, operands): super(FollowedByObservationExpression, self).__init__("FOLLOWEDBY", operands) - def to_pattern(self): - return "[{}] {} [{}]".format(self.operands[0], "FOLLOWEDBY", self.operands[1]) - class ParentheticalExpression(_PatternExpression): """Pattern Parenthetical Observation Expression @@ -606,9 +592,6 @@ class ParentheticalExpression(_PatternExpression): def __str__(self): return "(%s)" % self.expression - def to_pattern(self): - return "({})".format(self.expression.to_pattern()) - class _ExpressionQualifier(_PatternExpression): pass @@ -688,6 +671,3 @@ class QualifiedObservationExpression(_PatternExpression): def __str__(self): return "%s %s" % (self.observation_expression, self.qualifier) - - def to_pattern(self): - return "{} {}".format(self.observation_expression.to_pattern(), self.qualifier) diff --git a/stix2/test/test_pattern_expressions.py b/stix2/test/test_pattern_expressions.py index 14e3774..a4d0a5c 100644 --- a/stix2/test/test_pattern_expressions.py +++ b/stix2/test/test_pattern_expressions.py @@ -9,6 +9,7 @@ def test_create_comparison_expression(): exp = stix2.EqualityComparisonExpression("file:hashes.'SHA-256'", stix2.HashConstant("aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f", "SHA-256")) # noqa + assert str(exp) == "file:hashes.'SHA-256' = 'aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f'" @@ -18,6 +19,7 @@ def test_boolean_expression(): exp2 = stix2.MatchesComparisonExpression("email-message:body_multipart[*].body_raw_ref.name", stix2.StringConstant("^Final Report.+\\.exe$")) exp = stix2.AndBooleanExpression([exp1, exp2]) + assert str(exp) == "email-message:from_ref.value MATCHES '.+\\\\@example\\\\.com$' AND email-message:body_multipart[*].body_raw_ref.name MATCHES '^Final Report.+\\\\.exe$'" # noqa @@ -66,9 +68,8 @@ def test_file_observable_expression(): "aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f", 'SHA-256')) exp2 = stix2.EqualityComparisonExpression("file:mime_type", stix2.StringConstant("application/x-pdf")) - bool_exp = stix2.AndBooleanExpression([exp1, exp2]) - exp = stix2.ObservationExpression(bool_exp) - assert str(exp) == "[file:hashes.'SHA-256' = 'aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f' AND file:mime_type = 'application/x-pdf']" # noqa + bool_exp = stix2.ObservationExpression(stix2.AndBooleanExpression([exp1, exp2])) + assert str(bool_exp) == "[file:hashes.'SHA-256' = 'aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f' AND file:mime_type = 'application/x-pdf']" # noqa @pytest.mark.parametrize("observation_class, op", [ @@ -109,14 +110,12 @@ def test_artifact_payload(): "application/vnd.tcpdump.pcap") exp2 = stix2.MatchesComparisonExpression("artifact:payload_bin", stix2.StringConstant("\\xd4\\xc3\\xb2\\xa1\\x02\\x00\\x04\\x00")) - and_exp = stix2.AndBooleanExpression([exp1, exp2]) - exp = stix2.ObservationExpression(and_exp) - assert str(exp) == "[artifact:mime_type = 'application/vnd.tcpdump.pcap' AND artifact:payload_bin MATCHES '\\\\xd4\\\\xc3\\\\xb2\\\\xa1\\\\x02\\\\x00\\\\x04\\\\x00']" # noqa + and_exp = stix2.ObservationExpression(stix2.AndBooleanExpression([exp1, exp2])) + assert str(and_exp) == "[artifact:mime_type = 'application/vnd.tcpdump.pcap' AND artifact:payload_bin MATCHES '\\\\xd4\\\\xc3\\\\xb2\\\\xa1\\\\x02\\\\x00\\\\x04\\\\x00']" # noqa def test_greater_than_python_constant(): - exp1 = stix2.GreaterThanComparisonExpression("file:extensions.windows-pebinary-ext.sections[*].entropy", - 7.0) + exp1 = stix2.GreaterThanComparisonExpression("file:extensions.windows-pebinary-ext.sections[*].entropy", 7.0) exp = stix2.ObservationExpression(exp1) assert str(exp) == "[file:extensions.windows-pebinary-ext.sections[*].entropy > 7.0]" @@ -129,14 +128,14 @@ def test_greater_than(): def test_less_than(): - exp = stix2.LessThanComparisonExpression("file:size", - 1024) + exp = stix2.LessThanComparisonExpression("file:size", 1024) assert str(exp) == "file:size < 1024" def test_greater_than_or_equal(): exp = stix2.GreaterThanEqualComparisonExpression("file:size", 1024) + assert str(exp) == "file:size >= 1024" @@ -261,7 +260,7 @@ def test_invalid_integer_constant(): def test_invalid_timestamp_constant(): with pytest.raises(ValueError) as excinfo: stix2.TimestampConstant('foo') - assert 'must be a datetime object or timestamp string' in str(excinfo) + assert 'Must be a datetime object or timestamp string' in str(excinfo) def test_invalid_float_constant(): From b70aed0ef70918065fb3b2afa895e996fa15a5bb Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Mon, 10 Sep 2018 21:29:08 -0400 Subject: [PATCH 08/10] Make a few word changes in patterning guide --- docs/guide/patterns.ipynb | 396 +++++++++++++++++++++++++++----------- 1 file changed, 286 insertions(+), 110 deletions(-) diff --git a/docs/guide/patterns.ipynb b/docs/guide/patterns.ipynb index c963407..ee06675 100644 --- a/docs/guide/patterns.ipynb +++ b/docs/guide/patterns.ipynb @@ -6,18 +6,17 @@ "source": [ "# STIX2 Patterns\n", "\n", - "python-stix2 supports STIX2 patterning insofar that patterns may be used for the pattern property of Indicators, identical to the STIX2 specification. python-stix2 does not evaluate patterns against STIX2 content, for that functionality see [cti-pattern-matcher](https://github.com/oasis-open/cti-pattern-matcher).\n", + "The Python ``stix2`` library supports STIX 2 patterning insofar that patterns may be used for the pattern property of Indicators, identical to the STIX 2 specification. ``stix2`` does not evaluate patterns against STIX 2 content; for that functionality see [cti-pattern-matcher](https://github.com/oasis-open/cti-pattern-matcher).\n", "\n", - "\n", - "python-stix2 patterns are built compositely from the bottom up, creating subcomponent expressions first before those at higher levels.\n", + "Patterns in the ``stix2`` library are built compositely from the bottom up, creating subcomponent expressions first before those at higher levels.\n", "\n", "## API Tips\n", "\n", "### ObservationExpression\n", "\n", - "Within the STIX2 Patterning specification, Observation Expressions denote a complete expression to be evaluated against a discrete observation. In other words, an Observation Expression must be created to apply to a single Observation instance. This is further made clear by the visual brackets(```[]```) that encapsulate an Observation Expression. Thus, whatever sub expressions that are within the Observation Expression are meant to be matched against the same Observable instance.\n", + "Within the STIX 2 Patterning specification, Observation Expressions denote a complete expression to be evaluated against a discrete observation. In other words, an Observation Expression must be created to apply to a single Observation instance. This is further made clear by the visual brackets(```[]```) that encapsulate an Observation Expression. Thus, whatever sub expressions that are within the Observation Expression are meant to be matched against the same Observable instance.\n", "\n", - "This requirement manifests itself within python-stix2 via ```ObservationExpression```. When creating STIX2 observation expressions, whenever the current expression is complete, wrap it with ```ObservationExpression()```. This allows the complete pattern expression - no matter complexity - to be rendered as a proper specification adhering string. *__Of which, when pattern expressions are added to Indicator objects, the expression objects are implicitly converted to string representations__* . While the extra step may seem tedious in the construction of simple pattern expressions, this explicit marking of observation expressions becomes vital when converting the pattern expressions to strings. \n", + "This requirement manifests itself within the ``stix2`` library via ```ObservationExpression```. When creating STIX 2 observation expressions, whenever the current expression is complete, wrap it with ```ObservationExpression()```. This allows the complete pattern expression - no matter its complexity - to be rendered as a proper specification-adhering string. __*Note: When pattern expressions are added to Indicator objects, the expression objects are implicitly converted to string representations*__. While the extra step may seem tedious in the construction of simple pattern expressions, this explicit marking of observation expressions becomes vital when converting the pattern expressions to strings. \n", "\n", "In all the examples, you can observe how in the process of building pattern expressions, when an Observation Expression is completed, it is wrapped with ```ObservationExpression()```.\n", "\n", @@ -30,10 +29,11 @@ "Be careful to note the difference between these two very similar pattern components. \n", "\n", "__BooleanExpressions__\n", - " - stix2.AndBooleanExpression\n", - " - stix2.booleanExpression\n", + "\n", + " - [AndBooleanExpression](../api/stix2.patterns.rst#stix2.patterns.AndBooleanExpression)\n", + " - [OrbooleanExpression](../api/stix2.patterns.rst#stix2.patterns.OrBooleanExpression)\n", " \n", - " __Usage__: When the boolean sub-expressions refer to the same root object \n", + " __Usage__: When the boolean sub-expressions refer to the *same* root object \n", "\n", " __Example__:\n", " ```[domain-name:value = \"www.5z8.info\" AND domain-name:resolvess_to_refs[*].value = \"'198.51.100.1/32'\"]```\n", @@ -41,10 +41,11 @@ " __Rendering__: when pattern is rendered, brackets or parenthesis will encapsulate boolean expression\n", " \n", "__CompoundObservationExpressions__\n", - " - stix2.AndObservationExpression\n", - " - stix2.OrObservationExpression\n", + "\n", + " - [AndObservationExpression](../api/stix2.patterns.rst#stix2.patterns.AndObservationExpression)\n", + " - [OrObservationExpression](../api/stix2.patterns.rst#stix2.patterns.OrObservationExpression)\n", " \n", - " __Usage__: When the boolean sub-expressions refer to different root objects\n", + " __Usage__: When the boolean sub-expressions refer to *different* root objects\n", "\n", " __Example__:\n", " ```[file:name=\"foo.dll\"] AND [process:name = \"procfoo\"]```\n", @@ -60,59 +61,98 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "from stix2 import DomainName, File, IPv4Address\n", + "from stix2 import (ObjectPath, EqualityComparisonExpression, ObservationExpression,\n", + " GreaterThanComparisonExpression, IsSubsetComparisonExpression,\n", + " FloatConstant, StringConstant)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Equality Comparison expressions" + ] + }, + { + "cell_type": "code", + "execution_count": 7, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "--- Equality Comparison Expressions\n", - "\n", "\t[domain-name:value = 'site.of.interest.zaz']\n", "\n", "\t[file:parent_directory_ref.path = 'C:\\\\Windows\\\\System32']\n", - "\n", - "\n", - "--- Greater-than Comparison Expression\n", - "\n", - "\t[file:extensions.windows-pebinary-ext.sections[*].entropy > 7.0]\n", - "\n", - "\n", - "--- Is-Subset Comparison Expression\n", - "\n", - "\t[network-traffic:dst_ref.value ISSUBSET '2001:0db8:dead:beef:0000:0000:0000:0000/64']\n", "\n" ] } ], "source": [ - "from stix2 import DomainName, File, IPv4Address\n", - "from stix2 import (ObjectPath, EqualityComparisonExpression, ObservationExpression,\n", - " GreaterThanComparisonExpression, IsSubsetComparisonExpression,\n", - " FloatConstant, StringConstant)\n", - "\n", - "# ---- Equality Comparison expressions\n", - "print(\"--- Equality Comparison Expressions\\n\")\n", - "\n", "lhs = ObjectPath(\"domain-name\", [\"value\"])\n", "ece_1 = ObservationExpression(EqualityComparisonExpression(lhs, \"site.of.interest.zaz\"))\n", "print(\"\\t{}\\n\".format(ece_1))\n", "\n", "lhs = ObjectPath(\"file\", [\"parent_directory_ref\",\"path\"])\n", "ece_2 = ObservationExpression(EqualityComparisonExpression(lhs, \"C:\\\\Windows\\\\System32\"))\n", - "print(\"\\t{}\\n\".format(ece_2))\n", - "\n", - "# Greater-than Comparison expressions\n", - "print(\"\\n--- Greater-than Comparison Expression\\n\")\n", - "\n", + "print(\"\\t{}\\n\".format(ece_2))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Greater-than Comparison expressions" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\t[file:extensions.windows-pebinary-ext.sections[*].entropy > 7.0]\n", + "\n" + ] + } + ], + "source": [ "lhs = ObjectPath(\"file\", [\"extensions\", \"windows-pebinary-ext\", \"sections[*]\", \"entropy\"])\n", "gte = ObservationExpression(GreaterThanComparisonExpression(lhs, FloatConstant(\"7.0\")))\n", - "print(\"\\t{}\\n\".format(gte))\n", - "\n", - "# IsSubset Comparison expressions\n", - "print(\"\\n--- Is-Subset Comparison Expression\\n\")\n", - "\n", + "print(\"\\t{}\\n\".format(gte))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### IsSubset Comparison expressions" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\t[network-traffic:dst_ref.value ISSUBSET '2001:0db8:dead:beef:0000:0000:0000:0000/64']\n", + "\n" + ] + } + ], + "source": [ "lhs = ObjectPath(\"network-traffic\", [\"dst_ref\", \"value\"])\n", "iss = ObservationExpression(IsSubsetComparisonExpression(lhs, StringConstant(\"2001:0db8:dead:beef:0000:0000:0000:0000/64\")))\n", "print(\"\\t{}\\n\".format(iss))" @@ -127,65 +167,130 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from stix2 import (IntegerConstant, HashConstant, ObjectPath,\n", + " EqualityComparisonExpression, AndBooleanExpression,\n", + " OrBooleanExpression, ParentheticalExpression,\n", + " AndObservationExpression, OrObservationExpression,\n", + " FollowedByObservationExpression, ObservationExpression)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### AND boolean" + ] + }, + { + "cell_type": "code", + "execution_count": 3, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "--- Compound Observation Expressions\n", - "\n", "(AND)\n", "[email-message:sender_ref.value = 'stark@example.com' AND email-message:subject = 'Conference Info']\n", - "\n", - "(OR)\n", - "[url:value = 'http://example.com/foo' OR url:value = 'http://example.com/bar']\n", - "\n", - "(OR,AND)\n", - "[(file:name = 'pdf.exe' OR file:size = 371712) AND file:created = 2014-01-13 07:03:17+00:00]\n", - "\n", - "(AND,OR,OR)\n", - "([file:name = 'foo.dll'] AND [win-registry-key:key = 'HKEY_LOCAL_MACHINE\\\\foo\\\\bar']) OR [process:name = 'fooproc' OR process:name = 'procfoo']\n", - "\n", - "(FollowedBy)\n", - "[file:hashes.MD5 = '79054025255fb1a26e4bc422aef54eb4'] FOLLOWEDBY [win-registry-key:key = 'HKEY_LOCAL_MACHINE\\\\foo\\\\bar']\n", "\n" ] } ], "source": [ - "from stix2 import (IntegerConstant, HashConstant, ObjectPath,\n", - " EqualityComparisonExpression, AndBooleanExpression,\n", - " OrBooleanExpression, ParentheticalExpression,\n", - " AndObservationExpression, OrObservationExpression,\n", - " FollowedByObservationExpression, ObservationExpression)\n", - "\n", - "# ---- Observation expressions\n", - "print(\"--- Compound Observation Expressions\\n\")\n", - "\n", - "# AND boolean\n", "ece3 = EqualityComparisonExpression(ObjectPath(\"email-message\", [\"sender_ref\", \"value\"]), \"stark@example.com\")\n", "ece4 = EqualityComparisonExpression(ObjectPath(\"email-message\", [\"subject\"]), \"Conference Info\")\n", "abe = ObservationExpression(AndBooleanExpression([ece3, ece4]))\n", - "print(\"(AND)\\n{}\\n\".format(abe))\n", - "\n", - "# OR boolean\n", + "print(\"(AND)\\n{}\\n\".format(abe))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### OR boolean" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "(OR)\n", + "[url:value = 'http://example.com/foo' OR url:value = 'http://example.com/bar']\n", + "\n" + ] + } + ], + "source": [ "ece5 = EqualityComparisonExpression(ObjectPath(\"url\", [\"value\"]), \"http://example.com/foo\")\n", "ece6 = EqualityComparisonExpression(ObjectPath(\"url\", [\"value\"]), \"http://example.com/bar\")\n", "obe = ObservationExpression(OrBooleanExpression([ece5, ece6]))\n", - "print(\"(OR)\\n{}\\n\".format(obe))\n", - "\n", - "# ( OR ) AND boolean\n", + "print(\"(OR)\\n{}\\n\".format(obe))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### ( OR ) AND boolean" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "(OR,AND)\n", + "[(file:name = 'pdf.exe' OR file:size = 371712) AND file:created = 2014-01-13 07:03:17+00:00]\n", + "\n" + ] + } + ], + "source": [ "ece7 = EqualityComparisonExpression(ObjectPath(\"file\", [\"name\"]), \"pdf.exe\")\n", "ece8 = EqualityComparisonExpression(ObjectPath(\"file\", [\"size\"]), IntegerConstant(\"371712\"))\n", "ece9 = EqualityComparisonExpression(ObjectPath(\"file\", [\"created\"]), \"2014-01-13T07:03:17Z\")\n", "obe1 = OrBooleanExpression([ece7, ece8])\n", "pobe = ParentheticalExpression(obe1)\n", "abe1 = ObservationExpression(AndBooleanExpression([pobe, ece9]))\n", - "print(\"(OR,AND)\\n{}\\n\".format(abe1))\n", - "\n", - "# ( AND ) OR ( OR ) observation\n", + "print(\"(OR,AND)\\n{}\\n\".format(abe1))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### ( AND ) OR ( OR ) observation" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "(AND,OR,OR)\n", + "([file:name = 'foo.dll'] AND [win-registry-key:key = 'HKEY_LOCAL_MACHINE\\\\foo\\\\bar']) OR [process:name = 'fooproc' OR process:name = 'procfoo']\n", + "\n" + ] + } + ], + "source": [ "ece20 = ObservationExpression(EqualityComparisonExpression(ObjectPath(\"file\", [\"name\"]), \"foo.dll\"))\n", "ece21 = ObservationExpression(EqualityComparisonExpression(ObjectPath(\"win-registry-key\", [\"key\"]), \"HKEY_LOCAL_MACHINE\\\\foo\\\\bar\"))\n", "ece22 = EqualityComparisonExpression(ObjectPath(\"process\", [\"name\"]), \"fooproc\")\n", @@ -195,9 +300,32 @@ "aoe = ParentheticalExpression(AndObservationExpression([ece20, ece21]))\n", "obe2 = ObservationExpression(OrBooleanExpression([ece22, ece23]))\n", "ooe = OrObservationExpression([aoe, obe2])\n", - "print(\"(AND,OR,OR)\\n{}\\n\".format(ooe))\n", - "\n", - "# FOLLOWED-BY\n", + "print(\"(AND,OR,OR)\\n{}\\n\".format(ooe))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### FOLLOWED-BY" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "(FollowedBy)\n", + "[file:hashes.MD5 = '79054025255fb1a26e4bc422aef54eb4'] FOLLOWEDBY [win-registry-key:key = 'HKEY_LOCAL_MACHINE\\\\foo\\\\bar']\n", + "\n" + ] + } + ], + "source": [ "ece10 = ObservationExpression(EqualityComparisonExpression(ObjectPath(\"file\", [\"hashes\", \"MD5\"]), HashConstant(\"79054025255fb1a26e4bc422aef54eb4\", \"MD5\")))\n", "ece11 = ObservationExpression(EqualityComparisonExpression(ObjectPath(\"win-registry-key\", [\"key\"]), \"HKEY_LOCAL_MACHINE\\\\foo\\\\bar\"))\n", "fbe = FollowedByObservationExpression([ece10, ece11])\n", @@ -213,21 +341,93 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "from stix2 import (TimestampConstant, HashConstant, ObjectPath, EqualityComparisonExpression,\n", + " AndBooleanExpression, WithinQualifier, RepeatQualifier, StartStopQualifier,\n", + " QualifiedObservationExpression, FollowedByObservationExpression,\n", + " ParentheticalExpression, ObservationExpression)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### WITHIN" + ] + }, + { + "cell_type": "code", + "execution_count": 9, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "--- Qualified Observation Expressions\n", - "\n", "(WITHIN)\n", "([file:hashes.MD5 = '79054025255fb1a26e4bc422aef54eb4'] FOLLOWEDBY [win-registry-key:key = 'HKEY_LOCAL_MACHINE\\\\foo\\\\bar']) WITHIN 300 SECONDS\n", - "\n", + "\n" + ] + } + ], + "source": [ + "ece10 = ObservationExpression(EqualityComparisonExpression(ObjectPath(\"file\", [\"hashes\", \"MD5\"]), HashConstant(\"79054025255fb1a26e4bc422aef54eb4\", \"MD5\")))\n", + "ece11 = ObservationExpression(EqualityComparisonExpression(ObjectPath(\"win-registry-key\", [\"key\"]), \"HKEY_LOCAL_MACHINE\\\\foo\\\\bar\"))\n", + "fbe = FollowedByObservationExpression([ece10, ece11])\n", + "par = ParentheticalExpression(fbe)\n", + "qoe = QualifiedObservationExpression(par, WithinQualifier(300))\n", + "print(\"(WITHIN)\\n{}\\n\".format(qoe))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### REPEATS, WITHIN" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ "(REPEAT, WITHIN)\n", "[network-traffic:dst_ref.type = 'domain-name' AND network-traffic:dst_ref.value = 'example.com'] REPEATS 5 TIMES WITHIN 180 SECONDS\n", - "\n", + "\n" + ] + } + ], + "source": [ + "ece12 = EqualityComparisonExpression(ObjectPath(\"network-traffic\", [\"dst_ref\", \"type\"]), \"domain-name\")\n", + "ece13 = EqualityComparisonExpression(ObjectPath(\"network-traffic\", [\"dst_ref\", \"value\"]), \"example.com\")\n", + "abe2 = ObservationExpression(AndBooleanExpression([ece12, ece13]))\n", + "qoe1 = QualifiedObservationExpression(QualifiedObservationExpression(abe2, RepeatQualifier(5)), WithinQualifier(180))\n", + "print(\"(REPEAT, WITHIN)\\n{}\\n\".format(qoe1))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### START, STOP" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ "(START-STOP)\n", "[file:name = 'foo.dll'] START t'2016-06-01T00:00:00Z' STOP t'2016-07-01T00:00:00Z'\n", "\n" @@ -235,34 +435,10 @@ } ], "source": [ - "from stix2 import (TimestampConstant, HashConstant, ObjectPath, EqualityComparisonExpression,\n", - " AndBooleanExpression, WithinQualifier, RepeatQualifier, StartStopQualifier,\n", - " QualifiedObservationExpression, FollowedByObservationExpression,\n", - " ParentheticalExpression, ObservationExpression)\n", - "\n", - "# Qualified Observation Expressions\n", - "print(\"--- Qualified Observation Expressions\\n\")\n", - "\n", - "# WITHIN\n", - "ece10 = ObservationExpression(EqualityComparisonExpression(ObjectPath(\"file\", [\"hashes\", \"MD5\"]), HashConstant(\"79054025255fb1a26e4bc422aef54eb4\", \"MD5\")))\n", - "ece11 = ObservationExpression(EqualityComparisonExpression(ObjectPath(\"win-registry-key\", [\"key\"]), \"HKEY_LOCAL_MACHINE\\\\foo\\\\bar\"))\n", - "fbe = FollowedByObservationExpression([ece10, ece11])\n", - "par = ParentheticalExpression(fbe)\n", - "qoe = QualifiedObservationExpression(par, WithinQualifier(300))\n", - "print(\"(WITHIN)\\n{}\\n\".format(qoe))\n", - "\n", - "# REPEATS, WITHIN\n", - "ece12 = EqualityComparisonExpression(ObjectPath(\"network-traffic\", [\"dst_ref\", \"type\"]), \"domain-name\")\n", - "ece13 = EqualityComparisonExpression(ObjectPath(\"network-traffic\", [\"dst_ref\", \"value\"]), \"example.com\")\n", - "abe2 = ObservationExpression(AndBooleanExpression([ece12, ece13]))\n", - "qoe1 = QualifiedObservationExpression(QualifiedObservationExpression(abe2, RepeatQualifier(5)), WithinQualifier(180))\n", - "print(\"(REPEAT, WITHIN)\\n{}\\n\".format(qoe1))\n", - "\n", - "# START, STOP\n", "ece14 = ObservationExpression(EqualityComparisonExpression(ObjectPath(\"file\", [\"name\"]), \"foo.dll\"))\n", "ssq = StartStopQualifier(TimestampConstant('2016-06-01T00:00:00Z'), TimestampConstant('2016-07-01T00:00:00Z'))\n", "qoe2 = QualifiedObservationExpression(ece14, ssq)\n", - "print(\"(START-STOP)\\n{}\\n\".format(qoe2))\n" + "print(\"(START-STOP)\\n{}\\n\".format(qoe2))" ] }, { @@ -325,7 +501,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.5" + "version": "3.6.3" } }, "nbformat": 4, From e80fb4f59adad16142d470ca5233f57bac4a4000 Mon Sep 17 00:00:00 2001 From: Maciej Urbanski Date: Wed, 3 Oct 2018 17:57:31 +0200 Subject: [PATCH 09/10] python 3.7 support --- .travis.yml | 1 + setup.py | 1 + stix2/base.py | 5 +++++ stix2/v20/common.py | 11 ++--------- stix2/v20/observables.py | 20 +++----------------- stix2/v20/sdo.py | 11 ++--------- tox.ini | 3 ++- 7 files changed, 16 insertions(+), 36 deletions(-) diff --git a/.travis.yml b/.travis.yml index 28728ff..062c1eb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,7 @@ python: - "3.4" - "3.5" - "3.6" + - "3.7" install: - pip install -U pip setuptools - pip install tox-travis pre-commit diff --git a/setup.py b/setup.py index 9700ec8..d701e60 100644 --- a/setup.py +++ b/setup.py @@ -43,6 +43,7 @@ setup( 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', ], keywords="stix stix2 json cti cyber threat intelligence", packages=find_packages(exclude=['*.test']), diff --git a/stix2/base.py b/stix2/base.py index c26f60a..0fd7858 100644 --- a/stix2/base.py +++ b/stix2/base.py @@ -331,3 +331,8 @@ class _Extension(_STIXBase): def _check_object_constraints(self): super(_Extension, self)._check_object_constraints() self._check_at_least_one_property() + + +def _cls_init(cls, obj, kwargs): + if getattr(cls, '__init__', object.__init__) is not object.__init__: + cls.__init__(obj, **kwargs) diff --git a/stix2/v20/common.py b/stix2/v20/common.py index 0bba3b1..c66b8f6 100644 --- a/stix2/v20/common.py +++ b/stix2/v20/common.py @@ -2,7 +2,7 @@ from collections import OrderedDict -from ..base import _STIXBase +from ..base import _cls_init, _STIXBase from ..markings import _MarkingsMixin from ..properties import (HashesProperty, IDProperty, ListProperty, Property, ReferenceProperty, SelectorProperty, StringProperty, @@ -169,14 +169,7 @@ def CustomMarking(type='x-custom-marking', properties=None): def __init__(self, **kwargs): _STIXBase.__init__(self, **kwargs) - try: - cls.__init__(self, **kwargs) - except (AttributeError, TypeError) as e: - # Don't accidentally catch errors raised in a custom __init__() - if ("has no attribute '__init__'" in str(e) or - str(e) == "object.__init__() takes no parameters"): - return - raise e + _cls_init(cls, self, kwargs) _register_marking(_Custom) return _Custom diff --git a/stix2/v20/observables.py b/stix2/v20/observables.py index 659929b..36def28 100644 --- a/stix2/v20/observables.py +++ b/stix2/v20/observables.py @@ -9,7 +9,7 @@ from collections import OrderedDict import copy import re -from ..base import _Extension, _Observable, _STIXBase +from ..base import _cls_init, _Extension, _Observable, _STIXBase from ..exceptions import (AtLeastOnePropertyError, CustomContentError, DependentPropertiesError, ParseError) from ..properties import (BinaryProperty, BooleanProperty, DictionaryProperty, @@ -1027,14 +1027,7 @@ def CustomObservable(type='x-custom-observable', properties=None): def __init__(self, **kwargs): _Observable.__init__(self, **kwargs) - try: - cls.__init__(self, **kwargs) - except (AttributeError, TypeError) as e: - # Don't accidentally catch errors raised in a custom __init__() - if ("has no attribute '__init__'" in str(e) or - str(e) == "object.__init__() takes no parameters"): - return - raise e + _cls_init(cls, self, kwargs) _register_observable(_Custom) return _Custom @@ -1090,14 +1083,7 @@ def CustomExtension(observable=None, type='x-custom-observable', properties=None def __init__(self, **kwargs): _Extension.__init__(self, **kwargs) - try: - cls.__init__(self, **kwargs) - except (AttributeError, TypeError) as e: - # Don't accidentally catch errors raised in a custom __init__() - if ("has no attribute '__init__'" in str(e) or - str(e) == "object.__init__() takes no parameters"): - return - raise e + _cls_init(cls, self, kwargs) _register_extension(observable, _Custom) return _Custom diff --git a/stix2/v20/sdo.py b/stix2/v20/sdo.py index abe7df4..1a688d1 100644 --- a/stix2/v20/sdo.py +++ b/stix2/v20/sdo.py @@ -6,7 +6,7 @@ import re import stix2 -from ..base import _STIXBase +from ..base import _cls_init, _STIXBase from ..markings import _MarkingsMixin from ..properties import (BooleanProperty, IDProperty, IntegerProperty, ListProperty, PatternProperty, ReferenceProperty, @@ -400,14 +400,7 @@ def CustomObject(type='x-custom-type', properties=None): def __init__(self, **kwargs): _STIXBase.__init__(self, **kwargs) - try: - cls.__init__(self, **kwargs) - except (AttributeError, TypeError) as e: - # Don't accidentally catch errors raised in a custom __init__() - if ("has no attribute '__init__'" in str(e) or - str(e) == "object.__init__() takes no parameters"): - return - raise e + _cls_init(cls, self, kwargs) stix2._register_type(_Custom, version="2.0") return _Custom diff --git a/tox.ini b/tox.ini index 14d379c..47fcce8 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27,py34,py35,py36,style,isort-check,packaging +envlist = py27,py34,py35,py36,py37,style,isort-check,packaging [testenv] deps = @@ -43,3 +43,4 @@ python = 3.4: py34, style 3.5: py35, style 3.6: py36, style, packaging + 3.7: py37, style From 8eef356fb62f9ddeca863fef6254f111917ebd14 Mon Sep 17 00:00:00 2001 From: Maciej Urbanski Date: Wed, 3 Oct 2018 18:11:19 +0200 Subject: [PATCH 10/10] workaround for running python 3.7 on travis --- .travis.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 062c1eb..261f125 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,11 @@ python: - "3.4" - "3.5" - "3.6" - - "3.7" +matrix: + include: + - python: 3.7 # https://github.com/travis-ci/travis-ci/issues/9069#issuecomment-425720905 + dist: xenial + sudo: true install: - pip install -U pip setuptools - pip install tox-travis pre-commit