From 97dfe092f9025b05a3b681e8c2cd59a68b18484b Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Mon, 19 Oct 2020 19:23:30 -0400 Subject: [PATCH 01/56] re-order properties --- stix2/v21/common.py | 10 +++--- stix2/v21/observables.py | 72 ++++++++++++++++++++-------------------- stix2/v21/sdo.py | 40 +++++++++++----------- 3 files changed, 61 insertions(+), 61 deletions(-) diff --git a/stix2/v21/common.py b/stix2/v21/common.py index 1f228af..7c6b388 100644 --- a/stix2/v21/common.py +++ b/stix2/v21/common.py @@ -94,7 +94,7 @@ class LanguageContent(_STIXBase21): ('object_modified', TimestampProperty(precision='millisecond')), # TODO: 'contents' https://docs.google.com/document/d/1ShNq4c3e1CkfANmD9O--mdZ5H0O_GLnjN28a_yrEaco/edit#heading=h.cfz5hcantmvx ('contents', DictionaryProperty(spec_version='2.1', required=True)), - ('revoked', BooleanProperty()), + ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('confidence', IntegerProperty()), ('external_references', ListProperty(ExternalReference)), @@ -153,15 +153,15 @@ class MarkingDefinition(_STIXBase21, _MarkingsMixin): _properties = OrderedDict([ ('type', TypeProperty(_type, spec_version='2.1')), ('spec_version', StringProperty(fixed='2.1')), - ('id', IDProperty(_type)), + ('id', IDProperty(_type, spec_version='2.1')), ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond', precision_constraint='min')), - ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), - ('granular_markings', ListProperty(GranularMarking)), ('definition_type', StringProperty(required=True)), ('name', StringProperty()), ('definition', MarkingProperty(required=True)), + ('external_references', ListProperty(ExternalReference)), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), + ('granular_markings', ListProperty(GranularMarking)), ]) def __init__(self, **kwargs): diff --git a/stix2/v21/observables.py b/stix2/v21/observables.py index 55224cd..7919383 100644 --- a/stix2/v21/observables.py +++ b/stix2/v21/observables.py @@ -28,6 +28,7 @@ class Artifact(_Observable): _type = 'artifact' _properties = OrderedDict([ ('type', TypeProperty(_type, spec_version='2.1')), + ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('mime_type', StringProperty()), ('payload_bin', BinaryProperty()), @@ -35,11 +36,10 @@ class Artifact(_Observable): ('hashes', HashesProperty(spec_version='2.1')), ('encryption_algorithm', StringProperty()), ('decryption_key', StringProperty()), - ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), - ('spec_version', StringProperty(fixed='2.1')), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ('defanged', BooleanProperty(default=lambda: False)), + ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) _id_contributing_properties = ["hashes", "payload_bin"] @@ -57,15 +57,15 @@ class AutonomousSystem(_Observable): _type = 'autonomous-system' _properties = OrderedDict([ ('type', TypeProperty(_type, spec_version='2.1')), + ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('number', IntegerProperty(required=True)), ('name', StringProperty()), ('rir', StringProperty()), - ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), - ('spec_version', StringProperty(fixed='2.1')), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ('defanged', BooleanProperty(default=lambda: False)), + ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) _id_contributing_properties = ["number"] @@ -78,6 +78,7 @@ class Directory(_Observable): _type = 'directory' _properties = OrderedDict([ ('type', TypeProperty(_type, spec_version='2.1')), + ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('path', StringProperty(required=True)), ('path_enc', StringProperty()), @@ -86,11 +87,10 @@ class Directory(_Observable): ('mtime', TimestampProperty()), ('atime', TimestampProperty()), ('contains_refs', ListProperty(ReferenceProperty(valid_types=['file', 'directory'], spec_version='2.1'))), - ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), - ('spec_version', StringProperty(fixed='2.1')), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ('defanged', BooleanProperty(default=lambda: False)), + ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) _id_contributing_properties = ["path"] @@ -103,14 +103,14 @@ class DomainName(_Observable): _type = 'domain-name' _properties = OrderedDict([ ('type', TypeProperty(_type, spec_version='2.1')), + ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('value', StringProperty(required=True)), ('resolves_to_refs', ListProperty(ReferenceProperty(valid_types=['ipv4-addr', 'ipv6-addr', 'domain-name'], spec_version='2.1'))), - ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), - ('spec_version', StringProperty(fixed='2.1')), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ('defanged', BooleanProperty(default=lambda: False)), + ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) _id_contributing_properties = ["value"] @@ -123,15 +123,15 @@ class EmailAddress(_Observable): _type = 'email-addr' _properties = OrderedDict([ ('type', TypeProperty(_type, spec_version='2.1')), + ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('value', StringProperty(required=True)), ('display_name', StringProperty()), ('belongs_to_ref', ReferenceProperty(valid_types='user-account', spec_version='2.1')), - ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), - ('spec_version', StringProperty(fixed='2.1')), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ('defanged', BooleanProperty(default=lambda: False)), + ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) _id_contributing_properties = ["value"] @@ -161,6 +161,7 @@ class EmailMessage(_Observable): _type = 'email-message' _properties = OrderedDict([ ('type', TypeProperty(_type, spec_version='2.1')), + ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('is_multipart', BooleanProperty(required=True)), ('date', TimestampProperty()), @@ -177,11 +178,10 @@ class EmailMessage(_Observable): ('body', StringProperty()), ('body_multipart', ListProperty(EmbeddedObjectProperty(type=EmailMIMEComponent))), ('raw_email_ref', ReferenceProperty(valid_types='artifact', spec_version='2.1')), - ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), - ('spec_version', StringProperty(fixed='2.1')), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ('defanged', BooleanProperty(default=lambda: False)), + ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) _id_contributing_properties = ["from_ref", "subject", "body"] @@ -345,6 +345,7 @@ class File(_Observable): _type = 'file' _properties = OrderedDict([ ('type', TypeProperty(_type, spec_version='2.1')), + ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('hashes', HashesProperty(spec_version='2.1')), ('size', IntegerProperty(min=0)), @@ -358,11 +359,10 @@ class File(_Observable): ('parent_directory_ref', ReferenceProperty(valid_types='directory', spec_version='2.1')), ('contains_refs', ListProperty(ReferenceProperty(valid_types=["SCO"], spec_version='2.1'))), ('content_ref', ReferenceProperty(valid_types='artifact', spec_version='2.1')), - ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), - ('spec_version', StringProperty(fixed='2.1')), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ('defanged', BooleanProperty(default=lambda: False)), + ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) _id_contributing_properties = ["hashes", "name", "parent_directory_ref", "extensions"] @@ -379,15 +379,15 @@ class IPv4Address(_Observable): _type = 'ipv4-addr' _properties = OrderedDict([ ('type', TypeProperty(_type, spec_version='2.1')), + ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('value', StringProperty(required=True)), ('resolves_to_refs', ListProperty(ReferenceProperty(valid_types='mac-addr', spec_version='2.1'))), ('belongs_to_refs', ListProperty(ReferenceProperty(valid_types='autonomous-system', spec_version='2.1'))), - ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), - ('spec_version', StringProperty(fixed='2.1')), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ('defanged', BooleanProperty(default=lambda: False)), + ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) _id_contributing_properties = ["value"] @@ -400,15 +400,15 @@ class IPv6Address(_Observable): _type = 'ipv6-addr' _properties = OrderedDict([ ('type', TypeProperty(_type, spec_version='2.1')), + ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('value', StringProperty(required=True)), ('resolves_to_refs', ListProperty(ReferenceProperty(valid_types='mac-addr', spec_version='2.1'))), ('belongs_to_refs', ListProperty(ReferenceProperty(valid_types='autonomous-system', spec_version='2.1'))), - ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), - ('spec_version', StringProperty(fixed='2.1')), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ('defanged', BooleanProperty(default=lambda: False)), + ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) _id_contributing_properties = ["value"] @@ -421,13 +421,13 @@ class MACAddress(_Observable): _type = 'mac-addr' _properties = OrderedDict([ ('type', TypeProperty(_type, spec_version='2.1')), + ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('value', StringProperty(required=True)), - ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), - ('spec_version', StringProperty(fixed='2.1')), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ('defanged', BooleanProperty(default=lambda: False)), + ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) _id_contributing_properties = ["value"] @@ -440,13 +440,13 @@ class Mutex(_Observable): _type = 'mutex' _properties = OrderedDict([ ('type', TypeProperty(_type, spec_version='2.1')), + ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('name', StringProperty(required=True)), - ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), - ('spec_version', StringProperty(fixed='2.1')), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ('defanged', BooleanProperty(default=lambda: False)), + ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) _id_contributing_properties = ["name"] @@ -551,6 +551,7 @@ class NetworkTraffic(_Observable): _type = 'network-traffic' _properties = OrderedDict([ ('type', TypeProperty(_type, spec_version='2.1')), + ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('start', TimestampProperty()), ('end', TimestampProperty()), @@ -569,11 +570,10 @@ class NetworkTraffic(_Observable): ('dst_payload_ref', ReferenceProperty(valid_types='artifact', spec_version='2.1')), ('encapsulates_refs', ListProperty(ReferenceProperty(valid_types='network-traffic', spec_version='2.1'))), ('encapsulated_by_ref', ReferenceProperty(valid_types='network-traffic', spec_version='2.1')), - ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), - ('spec_version', StringProperty(fixed='2.1')), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ('defanged', BooleanProperty(default=lambda: False)), + ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) _id_contributing_properties = ["start", "src_ref", "dst_ref", "src_port", "dst_port", "protocols"] @@ -673,6 +673,7 @@ class Process(_Observable): _type = 'process' _properties = OrderedDict([ ('type', TypeProperty(_type, spec_version='2.1')), + ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('is_hidden', BooleanProperty()), ('pid', IntegerProperty()), @@ -686,11 +687,10 @@ class Process(_Observable): ('image_ref', ReferenceProperty(valid_types='file', spec_version='2.1')), ('parent_ref', ReferenceProperty(valid_types='process', spec_version='2.1')), ('child_refs', ListProperty(ReferenceProperty(valid_types='process', spec_version='2.1'))), - ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), - ('spec_version', StringProperty(fixed='2.1')), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ('defanged', BooleanProperty(default=lambda: False)), + ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) _id_contributing_properties = [] @@ -717,6 +717,7 @@ class Software(_Observable): _type = 'software' _properties = OrderedDict([ ('type', TypeProperty(_type, spec_version='2.1')), + ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('name', StringProperty(required=True)), ('cpe', StringProperty()), @@ -724,11 +725,10 @@ class Software(_Observable): ('languages', ListProperty(StringProperty)), ('vendor', StringProperty()), ('version', StringProperty()), - ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), - ('spec_version', StringProperty(fixed='2.1')), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ('defanged', BooleanProperty(default=lambda: False)), + ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) _id_contributing_properties = ["name", "cpe", "swid", "vendor", "version"] @@ -741,13 +741,13 @@ class URL(_Observable): _type = 'url' _properties = OrderedDict([ ('type', TypeProperty(_type, spec_version='2.1')), + ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('value', StringProperty(required=True)), - ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), - ('spec_version', StringProperty(fixed='2.1')), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ('defanged', BooleanProperty(default=lambda: False)), + ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) _id_contributing_properties = ["value"] @@ -774,6 +774,7 @@ class UserAccount(_Observable): _type = 'user-account' _properties = OrderedDict([ ('type', TypeProperty(_type, spec_version='2.1')), + ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('user_id', StringProperty()), ('credential', StringProperty()), @@ -789,11 +790,10 @@ class UserAccount(_Observable): ('credential_last_changed', TimestampProperty()), ('account_first_login', TimestampProperty()), ('account_last_login', TimestampProperty()), - ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), - ('spec_version', StringProperty(fixed='2.1')), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ('defanged', BooleanProperty(default=lambda: False)), + ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) _id_contributing_properties = ["account_type", "user_id", "account_login"] @@ -835,6 +835,7 @@ class WindowsRegistryKey(_Observable): _type = 'windows-registry-key' _properties = OrderedDict([ ('type', TypeProperty(_type, spec_version='2.1')), + ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('key', StringProperty()), ('values', ListProperty(EmbeddedObjectProperty(type=WindowsRegistryValueType))), @@ -842,11 +843,10 @@ class WindowsRegistryKey(_Observable): ('modified_time', TimestampProperty()), ('creator_user_ref', ReferenceProperty(valid_types='user-account', spec_version='2.1')), ('number_of_subkeys', IntegerProperty()), - ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), - ('spec_version', StringProperty(fixed='2.1')), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ('defanged', BooleanProperty(default=lambda: False)), + ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) _id_contributing_properties = ["key", "values"] @@ -885,6 +885,7 @@ class X509Certificate(_Observable): _type = 'x509-certificate' _properties = OrderedDict([ ('type', TypeProperty(_type, spec_version='2.1')), + ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('is_self_signed', BooleanProperty()), ('hashes', HashesProperty(spec_version='2.1')), @@ -899,11 +900,10 @@ class X509Certificate(_Observable): ('subject_public_key_modulus', StringProperty()), ('subject_public_key_exponent', IntegerProperty()), ('x509_v3_extensions', EmbeddedObjectProperty(type=X509V3ExtenstionsType)), - ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), - ('spec_version', StringProperty(fixed='2.1')), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ('defanged', BooleanProperty(default=lambda: False)), + ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) _id_contributing_properties = ["hashes", "serial_number"] diff --git a/stix2/v21/sdo.py b/stix2/v21/sdo.py index decbf07..a359aa6 100644 --- a/stix2/v21/sdo.py +++ b/stix2/v21/sdo.py @@ -122,9 +122,13 @@ class Grouping(_DomainObject): ('type', TypeProperty(_type, spec_version='2.1')), ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond', precision_constraint='min')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond', precision_constraint='min')), - ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), + ('name', StringProperty()), + ('description', StringProperty()), + ('context', StringProperty(required=True)), + ('object_refs', ListProperty(ReferenceProperty(valid_types=["SCO", "SDO", "SRO"], spec_version='2.1'), required=True)), ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('confidence', IntegerProperty()), @@ -132,10 +136,6 @@ class Grouping(_DomainObject): ('external_references', ListProperty(ExternalReference)), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), - ('name', StringProperty()), - ('description', StringProperty()), - ('context', StringProperty(required=True)), - ('object_refs', ListProperty(ReferenceProperty(valid_types=["SCO", "SDO", "SRO"], spec_version='2.1'), required=True)), ]) @@ -240,13 +240,6 @@ class Infrastructure(_DomainObject): ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond', precision_constraint='min')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond', precision_constraint='min')), - ('revoked', BooleanProperty(default=lambda: False)), - ('labels', ListProperty(StringProperty)), - ('confidence', IntegerProperty()), - ('lang', StringProperty()), - ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), - ('granular_markings', ListProperty(GranularMarking)), ('name', StringProperty(required=True)), ('description', StringProperty()), ('infrastructure_types', ListProperty(StringProperty)), @@ -254,6 +247,13 @@ class Infrastructure(_DomainObject): ('kill_chain_phases', ListProperty(KillChainPhase)), ('first_seen', TimestampProperty()), ('last_seen', TimestampProperty()), + ('revoked', BooleanProperty(default=lambda: False)), + ('labels', ListProperty(StringProperty)), + ('confidence', IntegerProperty()), + ('lang', StringProperty()), + ('external_references', ListProperty(ExternalReference)), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), + ('granular_markings', ListProperty(GranularMarking)), ]) def _check_object_constraints(self): @@ -478,16 +478,9 @@ class MalwareAnalysis(_DomainObject): ('type', TypeProperty(_type, spec_version='2.1')), ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond', precision_constraint='min')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond', precision_constraint='min')), - ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), - ('revoked', BooleanProperty(default=lambda: False)), - ('labels', ListProperty(StringProperty)), - ('confidence', IntegerProperty()), - ('lang', StringProperty()), - ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), - ('granular_markings', ListProperty(GranularMarking)), ('product', StringProperty(required=True)), ('version', StringProperty()), ('host_vm_ref', ReferenceProperty(valid_types='software', spec_version='2.1')), @@ -504,6 +497,13 @@ class MalwareAnalysis(_DomainObject): ('result', StringProperty()), ('analysis_sco_refs', ListProperty(ReferenceProperty(valid_types="SCO", spec_version='2.1'))), ('sample_ref', ReferenceProperty(valid_types="SCO", spec_version="2.1")), + ('revoked', BooleanProperty(default=lambda: False)), + ('labels', ListProperty(StringProperty)), + ('confidence', IntegerProperty()), + ('lang', StringProperty()), + ('external_references', ListProperty(ExternalReference)), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), + ('granular_markings', ListProperty(GranularMarking)), ]) def _check_object_constraints(self): From 1bc11ce442a948289e37597f6212ebbe5b4c40fa Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Mon, 19 Oct 2020 22:08:51 -0400 Subject: [PATCH 02/56] define the STIXExtension Object --- stix2/v21/common.py | 47 ++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 44 insertions(+), 3 deletions(-) diff --git a/stix2/v21/common.py b/stix2/v21/common.py index 7c6b388..eae0865 100644 --- a/stix2/v21/common.py +++ b/stix2/v21/common.py @@ -7,9 +7,10 @@ from ..exceptions import InvalidValueError from ..markings import _MarkingsMixin from ..markings.utils import check_tlp_marking from ..properties import ( - BooleanProperty, DictionaryProperty, HashesProperty, IDProperty, - IntegerProperty, ListProperty, Property, ReferenceProperty, - SelectorProperty, StringProperty, TimestampProperty, TypeProperty, + BooleanProperty, DictionaryProperty, EnumProperty, ExtensionsProperty, + HashesProperty, IDProperty, IntegerProperty, ListProperty, Property, + ReferenceProperty, SelectorProperty, StringProperty, TimestampProperty, + TypeProperty, ) from ..utils import NOW, _get_dict from .base import _STIXBase21 @@ -100,6 +101,46 @@ class LanguageContent(_STIXBase21): ('external_references', ListProperty(ExternalReference)), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), + ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ]) + + +class STIXExtension(_STIXBase21): + """For more detailed information on this object's properties, see + `the STIX 2.1 specification `__. + """ + + _type = 'stix-extension' + _properties = OrderedDict([ + ('type', TypeProperty(_type, spec_version='2.1')), + ('spec_version', StringProperty(fixed='2.1')), + ('id', IDProperty(_type, spec_version='2.1')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1', required=True)), + ('created', TimestampProperty(default=lambda: NOW, precision='millisecond', precision_constraint='min')), + ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond', precision_constraint='min')), + ('name', StringProperty(required=True)), + ('description', StringProperty()), + ('schema', StringProperty(required=True)), + ('version', StringProperty(required=True)), + ( + 'extension_types', ListProperty( + EnumProperty( + allowed=[ + 'new-sdo', + 'new-sco', + 'new-sro', + 'property-extension', + 'toplevel-property-extension', + ], + ), required=True, + ), + ), + ('extension_properties', ListProperty(StringProperty)), + ('revoked', BooleanProperty(default=lambda: False)), + ('labels', ListProperty(StringProperty)), + ('external_references', ListProperty(ExternalReference)), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), + ('granular_markings', ListProperty(GranularMarking)), ]) From b17f502e8fe406ab3c016e2d793b5d5f0a377295 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Mon, 19 Oct 2020 22:18:45 -0400 Subject: [PATCH 03/56] expose the "extensions" property on all object types remove restriction on MarkingDefinition Object --- stix2/v21/__init__.py | 5 +++-- stix2/v21/base.py | 6 +++++- stix2/v21/common.py | 5 +++-- stix2/v21/sdo.py | 26 +++++++++++++++++++++++--- stix2/v21/sro.py | 7 +++++-- 5 files changed, 39 insertions(+), 10 deletions(-) diff --git a/stix2/v21/__init__.py b/stix2/v21/__init__.py index 77a28ee..70f2a79 100644 --- a/stix2/v21/__init__.py +++ b/stix2/v21/__init__.py @@ -21,7 +21,7 @@ from .bundle import Bundle from .common import ( TLP_AMBER, TLP_GREEN, TLP_RED, TLP_WHITE, CustomMarking, ExternalReference, GranularMarking, KillChainPhase, LanguageContent, MarkingDefinition, - StatementMarking, TLPMarking, + StatementMarking, STIXExtension, TLPMarking, ) from .observables import ( URL, AlternateDataStream, ArchiveExt, Artifact, AutonomousSystem, @@ -65,6 +65,7 @@ OBJ_MAP = { 'threat-actor': ThreatActor, 'tool': Tool, 'sighting': Sighting, + 'stix-extension': STIXExtension, 'vulnerability': Vulnerability, } @@ -121,7 +122,7 @@ __all__ = """ TLP_AMBER, TLP_GREEN, TLP_RED, TLP_WHITE, CustomMarking, ExternalReference, GranularMarking, KillChainPhase, LanguageContent, MarkingDefinition, - StatementMarking, TLPMarking, + StatementMarking, STIXExtension, TLPMarking, URL, AlternateDataStream, ArchiveExt, Artifact, AutonomousSystem, CustomExtension, CustomObservable, Directory, DomainName, EmailAddress, diff --git a/stix2/v21/base.py b/stix2/v21/base.py index 8b5a5f1..830a209 100644 --- a/stix2/v21/base.py +++ b/stix2/v21/base.py @@ -6,7 +6,11 @@ from ..base import ( class _STIXBase21(_STIXBase): - pass + + def __init__(self, **kwargs): + if 'extensions' in self._properties: + self._properties['extensions'].allow_custom = kwargs.get('allow_custom', False) + super(_STIXBase21, self).__init__(**kwargs) class _Observable(_Observable, _STIXBase21): diff --git a/stix2/v21/common.py b/stix2/v21/common.py index eae0865..60c4068 100644 --- a/stix2/v21/common.py +++ b/stix2/v21/common.py @@ -197,12 +197,13 @@ class MarkingDefinition(_STIXBase21, _MarkingsMixin): ('id', IDProperty(_type, spec_version='2.1')), ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond', precision_constraint='min')), - ('definition_type', StringProperty(required=True)), + ('definition_type', StringProperty()), ('name', StringProperty()), - ('definition', MarkingProperty(required=True)), + ('definition', MarkingProperty()), ('external_references', ListProperty(ExternalReference)), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), + ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) def __init__(self, **kwargs): diff --git a/stix2/v21/sdo.py b/stix2/v21/sdo.py index a359aa6..d538852 100644 --- a/stix2/v21/sdo.py +++ b/stix2/v21/sdo.py @@ -12,9 +12,10 @@ from ..exceptions import ( InvalidValueError, PropertyPresenceError, STIXDeprecationWarning, ) from ..properties import ( - BooleanProperty, EnumProperty, FloatProperty, IDProperty, IntegerProperty, - ListProperty, ObservableProperty, PatternProperty, ReferenceProperty, - StringProperty, TimestampProperty, TypeProperty, + BooleanProperty, EnumProperty, ExtensionsProperty, FloatProperty, + IDProperty, IntegerProperty, ListProperty, ObservableProperty, + PatternProperty, ReferenceProperty, StringProperty, TimestampProperty, + TypeProperty, ) from ..utils import NOW from .base import _DomainObject @@ -45,6 +46,7 @@ class AttackPattern(_DomainObject): ('external_references', ListProperty(ExternalReference)), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), + ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) @@ -74,6 +76,7 @@ class Campaign(_DomainObject): ('external_references', ListProperty(ExternalReference)), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), + ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) def _check_object_constraints(self): @@ -109,6 +112,7 @@ class CourseOfAction(_DomainObject): ('external_references', ListProperty(ExternalReference)), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), + ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) @@ -136,6 +140,7 @@ class Grouping(_DomainObject): ('external_references', ListProperty(ExternalReference)), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), + ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) @@ -165,6 +170,7 @@ class Identity(_DomainObject): ('external_references', ListProperty(ExternalReference)), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), + ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) @@ -197,6 +203,7 @@ class Indicator(_DomainObject): ('external_references', ListProperty(ExternalReference)), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), + ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) def __init__(self, *args, **kwargs): @@ -254,6 +261,7 @@ class Infrastructure(_DomainObject): ('external_references', ListProperty(ExternalReference)), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), + ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) def _check_object_constraints(self): @@ -296,6 +304,7 @@ class IntrusionSet(_DomainObject): ('external_references', ListProperty(ExternalReference)), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), + ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) def _check_object_constraints(self): @@ -340,6 +349,7 @@ class Location(_DomainObject): ('external_references', ListProperty(ExternalReference)), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), + ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) def _check_object_constraints(self): @@ -449,6 +459,7 @@ class Malware(_DomainObject): ('external_references', ListProperty(ExternalReference)), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), + ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) def _check_object_constraints(self): @@ -504,6 +515,7 @@ class MalwareAnalysis(_DomainObject): ('external_references', ListProperty(ExternalReference)), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), + ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) def _check_object_constraints(self): @@ -536,6 +548,7 @@ class Note(_DomainObject): ('external_references', ListProperty(ExternalReference)), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), + ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) @@ -564,6 +577,7 @@ class ObservedData(_DomainObject): ('external_references', ListProperty(ExternalReference)), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), + ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) def __init__(self, *args, **kwargs): @@ -628,6 +642,7 @@ class Opinion(_DomainObject): ('external_references', ListProperty(ExternalReference)), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), + ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) @@ -656,6 +671,7 @@ class Report(_DomainObject): ('external_references', ListProperty(ExternalReference)), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), + ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) @@ -692,6 +708,7 @@ class ThreatActor(_DomainObject): ('external_references', ListProperty(ExternalReference)), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), + ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) def _check_object_constraints(self): @@ -731,6 +748,7 @@ class Tool(_DomainObject): ('external_references', ListProperty(ExternalReference)), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), + ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) @@ -756,6 +774,7 @@ class Vulnerability(_DomainObject): ('external_references', ListProperty(ExternalReference)), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), + ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) @@ -807,6 +826,7 @@ def CustomObject(type='x-custom-type', properties=None): ('external_references', ListProperty(ExternalReference)), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), + ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=type)), ], sorted([x for x in properties if x[0].startswith('x_')], key=lambda x: x[0]), ])) diff --git a/stix2/v21/sro.py b/stix2/v21/sro.py index d287373..d0e07f8 100644 --- a/stix2/v21/sro.py +++ b/stix2/v21/sro.py @@ -3,8 +3,9 @@ from collections import OrderedDict from ..properties import ( - BooleanProperty, IDProperty, IntegerProperty, ListProperty, - ReferenceProperty, StringProperty, TimestampProperty, TypeProperty, + BooleanProperty, ExtensionsProperty, IDProperty, IntegerProperty, + ListProperty, ReferenceProperty, StringProperty, TimestampProperty, + TypeProperty, ) from ..utils import NOW from .base import _RelationshipObject @@ -39,6 +40,7 @@ class Relationship(_RelationshipObject): ('external_references', ListProperty(ExternalReference)), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), + ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) # Explicitly define the first three kwargs to make readable Relationship declarations. @@ -95,6 +97,7 @@ class Sighting(_RelationshipObject): ('external_references', ListProperty(ExternalReference)), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), + ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) # Explicitly define the first kwargs to make readable Sighting declarations. From 5cb52844dcf3ae0f1f3d8b7ea8b6422572852110 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Thu, 5 Nov 2020 12:56:30 -0500 Subject: [PATCH 04/56] fixing typos on Object types --- stix2/v20/__init__.py | 4 ++-- stix2/v20/observables.py | 4 ++-- stix2/v21/__init__.py | 4 ++-- stix2/v21/observables.py | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/stix2/v20/__init__.py b/stix2/v20/__init__.py index 36d09be..002343b 100644 --- a/stix2/v20/__init__.py +++ b/stix2/v20/__init__.py @@ -32,7 +32,7 @@ from .observables import ( UNIXAccountExt, UserAccount, WindowsPEBinaryExt, WindowsPEOptionalHeaderType, WindowsPESection, WindowsProcessExt, WindowsRegistryKey, WindowsRegistryValueType, WindowsServiceExt, - X509Certificate, X509V3ExtenstionsType, + X509Certificate, X509V3ExtensionsType, ) from .sdo import ( AttackPattern, Campaign, CourseOfAction, CustomObject, Identity, Indicator, @@ -123,7 +123,7 @@ __all__ = """ UNIXAccountExt, UserAccount, WindowsPEBinaryExt, WindowsPEOptionalHeaderType, WindowsPESection, WindowsProcessExt, WindowsRegistryKey, WindowsRegistryValueType, WindowsServiceExt, - X509Certificate, X509V3ExtenstionsType, + X509Certificate, X509V3ExtensionsType, AttackPattern, Campaign, CourseOfAction, CustomObject, Identity, Indicator, IntrusionSet, Malware, ObservedData, Report, ThreatActor, Tool, diff --git a/stix2/v20/observables.py b/stix2/v20/observables.py index 79d4e79..cec979e 100644 --- a/stix2/v20/observables.py +++ b/stix2/v20/observables.py @@ -724,7 +724,7 @@ class WindowsRegistryKey(_Observable): ]) -class X509V3ExtenstionsType(_STIXBase20): +class X509V3ExtensionsType(_STIXBase20): """For more detailed information on this object's properties, see `the STIX 2.0 specification `__. """ # noqa @@ -770,7 +770,7 @@ class X509Certificate(_Observable): ('subject_public_key_algorithm', StringProperty()), ('subject_public_key_modulus', StringProperty()), ('subject_public_key_exponent', IntegerProperty()), - ('x509_v3_extensions', EmbeddedObjectProperty(type=X509V3ExtenstionsType)), + ('x509_v3_extensions', EmbeddedObjectProperty(type=X509V3ExtensionsType)), ('extensions', ExtensionsProperty(spec_version="2.0", enclosing_type=_type)), ]) diff --git a/stix2/v21/__init__.py b/stix2/v21/__init__.py index 70f2a79..856ff13 100644 --- a/stix2/v21/__init__.py +++ b/stix2/v21/__init__.py @@ -32,7 +32,7 @@ from .observables import ( UNIXAccountExt, UserAccount, WindowsPEBinaryExt, WindowsPEOptionalHeaderType, WindowsPESection, WindowsProcessExt, WindowsRegistryKey, WindowsRegistryValueType, WindowsServiceExt, - X509Certificate, X509V3ExtenstionsType, + X509Certificate, X509V3ExtensionsType, ) from .sdo import ( AttackPattern, Campaign, CourseOfAction, CustomObject, Grouping, Identity, @@ -132,7 +132,7 @@ __all__ = """ UNIXAccountExt, UserAccount, WindowsPEBinaryExt, WindowsPEOptionalHeaderType, WindowsPESection, WindowsProcessExt, WindowsRegistryKey, WindowsRegistryValueType, WindowsServiceExt, - X509Certificate, X509V3ExtenstionsType, + X509Certificate, X509V3ExtensionsType, AttackPattern, Campaign, CourseOfAction, CustomObject, Grouping, Identity, Indicator, Infrastructure, IntrusionSet, Location, Malware, diff --git a/stix2/v21/observables.py b/stix2/v21/observables.py index 7919383..11e1d9f 100644 --- a/stix2/v21/observables.py +++ b/stix2/v21/observables.py @@ -851,7 +851,7 @@ class WindowsRegistryKey(_Observable): _id_contributing_properties = ["key", "values"] -class X509V3ExtenstionsType(_STIXBase21): +class X509V3ExtensionsType(_STIXBase21): """For more detailed information on this object's properties, see `the STIX 2.1 specification `__. """ @@ -899,7 +899,7 @@ class X509Certificate(_Observable): ('subject_public_key_algorithm', StringProperty()), ('subject_public_key_modulus', StringProperty()), ('subject_public_key_exponent', IntegerProperty()), - ('x509_v3_extensions', EmbeddedObjectProperty(type=X509V3ExtenstionsType)), + ('x509_v3_extensions', EmbeddedObjectProperty(type=X509V3ExtensionsType)), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ('defanged', BooleanProperty(default=lambda: False)), From 1180da7cc7cbc9c0472b852998bfebd1bd41aadd Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Mon, 9 Nov 2020 20:35:22 -0500 Subject: [PATCH 05/56] update to test cases removing invalid test cases per new changes. updating some to reflect new changes to the Extensions Property, other minor ones --- stix2/test/v20/test_custom.py | 84 ++++++--------------- stix2/test/v20/test_properties.py | 19 +---- stix2/test/v21/test_custom.py | 94 +++++++----------------- stix2/test/v21/test_deterministic_ids.py | 30 ++------ stix2/test/v21/test_malware_analysis.py | 12 +-- stix2/test/v21/test_properties.py | 19 +---- 6 files changed, 64 insertions(+), 194 deletions(-) diff --git a/stix2/test/v20/test_custom.py b/stix2/test/v20/test_custom.py index 70835c1..2fbcf85 100644 --- a/stix2/test/v20/test_custom.py +++ b/stix2/test/v20/test_custom.py @@ -696,7 +696,7 @@ def test_observed_data_with_custom_observable_object(): @stix2.v20.CustomExtension( - stix2.v20.DomainName, 'x-new-ext', [ + 'x-new-ext', [ ('property1', stix2.properties.StringProperty(required=True)), ('property2', stix2.properties.IntegerProperty()), ], @@ -758,8 +758,8 @@ def test_custom_extension_wrong_observable_type(): ) def test_custom_extension_with_list_and_dict_properties_observable_type(data): @stix2.v20.CustomExtension( - stix2.v20.UserAccount, 'some-extension', [ - ('keys', stix2.properties.ListProperty(stix2.properties.DictionaryProperty, required=True)), + 'some-extension', [ + ('keys', stix2.properties.ListProperty(stix2.properties.DictionaryProperty, required=True)), ], ) class SomeCustomExtension: @@ -769,53 +769,11 @@ def test_custom_extension_with_list_and_dict_properties_observable_type(data): assert data == str(example) -def test_custom_extension_invalid_observable(): - # These extensions are being applied to improperly-created Observables. - # The Observable classes should have been created with the CustomObservable decorator. - class Foo(object): - pass - with pytest.raises(ValueError) as excinfo: - @stix2.v20.CustomExtension( - Foo, 'x-new-ext', [ - ('property1', stix2.properties.StringProperty(required=True)), - ], - ) - class FooExtension(): - pass # pragma: no cover - assert str(excinfo.value) == "'observable' must be a valid Observable class!" - - class Bar(stix2.v20.observables._Observable): - pass - with pytest.raises(ValueError) as excinfo: - @stix2.v20.CustomExtension( - Bar, 'x-new-ext', [ - ('property1', stix2.properties.StringProperty(required=True)), - ], - ) - class BarExtension(): - pass - assert "Unknown observable type" in str(excinfo.value) - assert "Custom observables must be created with the @CustomObservable decorator." in str(excinfo.value) - - class Baz(stix2.v20.observables._Observable): - _type = 'Baz' - with pytest.raises(ValueError) as excinfo: - @stix2.v20.CustomExtension( - Baz, 'x-new-ext', [ - ('property1', stix2.properties.StringProperty(required=True)), - ], - ) - class BazExtension(): - pass - assert "Unknown observable type" in str(excinfo.value) - assert "Custom observables must be created with the @CustomObservable decorator." in str(excinfo.value) - - def test_custom_extension_invalid_type_name(): with pytest.raises(ValueError) as excinfo: @stix2.v20.CustomExtension( - stix2.v20.File, 'x', { - 'property1': stix2.properties.StringProperty(required=True), + 'x', { + 'property1': stix2.properties.StringProperty(required=True), }, ) class FooExtension(): @@ -824,8 +782,8 @@ def test_custom_extension_invalid_type_name(): with pytest.raises(ValueError) as excinfo: @stix2.v20.CustomExtension( - stix2.File, 'x_new_ext', { - 'property1': stix2.properties.StringProperty(required=True), + 'x_new_ext', { + 'property1': stix2.properties.StringProperty(required=True), }, ) class BlaExtension(): @@ -835,29 +793,29 @@ def test_custom_extension_invalid_type_name(): def test_custom_extension_no_properties(): with pytest.raises(ValueError): - @stix2.v20.CustomExtension(stix2.v20.DomainName, 'x-new-ext2', None) + @stix2.v20.CustomExtension('x-new-ext2', None) class BarExtension(): pass def test_custom_extension_empty_properties(): with pytest.raises(ValueError): - @stix2.v20.CustomExtension(stix2.v20.DomainName, 'x-new-ext2', []) + @stix2.v20.CustomExtension('x-new-ext2', []) class BarExtension(): pass def test_custom_extension_dict_properties(): with pytest.raises(ValueError): - @stix2.v20.CustomExtension(stix2.v20.DomainName, 'x-new-ext2', {}) + @stix2.v20.CustomExtension('x-new-ext2', {}) class BarExtension(): pass def test_custom_extension_no_init_1(): @stix2.v20.CustomExtension( - stix2.v20.DomainName, 'x-new-extension', [ - ('property1', stix2.properties.StringProperty(required=True)), + 'x-new-extension', [ + ('property1', stix2.properties.StringProperty(required=True)), ], ) class NewExt(): @@ -869,8 +827,8 @@ def test_custom_extension_no_init_1(): def test_custom_extension_no_init_2(): @stix2.v20.CustomExtension( - stix2.v20.DomainName, 'x-new-ext2', [ - ('property1', stix2.properties.StringProperty(required=True)), + 'x-new-ext2', [ + ('property1', stix2.properties.StringProperty(required=True)), ], ) class NewExt2(object): @@ -986,7 +944,7 @@ def test_register_custom_object(): def test_extension_property_location(): assert 'extensions' in stix2.v20.OBJ_MAP_OBSERVABLE['x-new-observable']._properties - assert 'extensions' not in stix2.v20.EXT_MAP['domain-name']['x-new-ext']._properties + assert 'extensions' not in stix2.v20.EXT_MAP['x-new-ext']._properties @pytest.mark.parametrize( @@ -1107,8 +1065,8 @@ def test_register_marking_with_version(): def test_register_observable_extension_with_version(): @stix2.v20.CustomExtension( - stix2.v20.UserAccount, 'some-extension-2', [ - ('keys', stix2.properties.StringProperty(required=True)), + 'some-extension-2', [ + ('keys', stix2.properties.StringProperty(required=True)), ], ) class SomeCustomExtension2: @@ -1117,15 +1075,15 @@ def test_register_observable_extension_with_version(): v = 'v20' example = SomeCustomExtension2(keys='test123') - assert example._type in parsing.STIX2_OBJ_MAPS[v]['observable-extensions']['user-account'] + assert example._type in parsing.STIX2_OBJ_MAPS[v]['extensions'] def test_register_duplicate_observable_extension(): with pytest.raises(DuplicateRegistrationError) as excinfo: @stix2.v20.CustomExtension( - stix2.v20.UserAccount, 'some-extension-2', [ - ('property1', stix2.properties.StringProperty(required=True)), - ('property2', stix2.properties.IntegerProperty()), + 'some-extension-2', [ + ('property1', stix2.properties.StringProperty(required=True)), + ('property2', stix2.properties.IntegerProperty()), ], ) class NewExtension2(): diff --git a/stix2/test/v20/test_properties.py b/stix2/test/v20/test_properties.py index b03879c..87f14c9 100644 --- a/stix2/test/v20/test_properties.py +++ b/stix2/test/v20/test_properties.py @@ -213,7 +213,7 @@ def test_embedded_property(): def test_extension_property_valid(): - ext_prop = ExtensionsProperty(spec_version="2.0", enclosing_type='file') + ext_prop = ExtensionsProperty(spec_version="2.0") assert ext_prop({ 'windows-pebinary-ext': { 'pe_type': 'exe', @@ -222,13 +222,13 @@ def test_extension_property_valid(): def test_extension_property_invalid1(): - ext_prop = ExtensionsProperty(spec_version="2.0", enclosing_type='file') + ext_prop = ExtensionsProperty(spec_version="2.0") with pytest.raises(ValueError): ext_prop.clean(1) def test_extension_property_invalid2(): - ext_prop = ExtensionsProperty(spec_version="2.0", enclosing_type='file') + ext_prop = ExtensionsProperty(spec_version="2.0") with pytest.raises(CustomContentError): ext_prop.clean( { @@ -239,19 +239,6 @@ def test_extension_property_invalid2(): ) -def test_extension_property_invalid_type(): - ext_prop = ExtensionsProperty(spec_version="2.0", enclosing_type='indicator') - with pytest.raises(CustomContentError) as excinfo: - ext_prop.clean( - { - 'windows-pebinary-ext': { - 'pe_type': 'exe', - }, - }, - ) - assert "Can't parse unknown extension" in str(excinfo.value) - - def test_extension_at_least_one_property_constraint(): with pytest.raises(AtLeastOnePropertyError): stix2.v20.TCPExt() diff --git a/stix2/test/v21/test_custom.py b/stix2/test/v21/test_custom.py index ea6d3a8..67990d2 100644 --- a/stix2/test/v21/test_custom.py +++ b/stix2/test/v21/test_custom.py @@ -893,7 +893,7 @@ def test_custom_observable_object_no_id_contrib_props(): @stix2.v21.CustomExtension( - stix2.v21.DomainName, 'x-new-ext', [ + 'x-new-ext', [ ('property1', stix2.properties.StringProperty(required=True)), ('property2', stix2.properties.IntegerProperty()), ], @@ -955,8 +955,8 @@ def test_custom_extension_wrong_observable_type(): ) def test_custom_extension_with_list_and_dict_properties_observable_type(data): @stix2.v21.CustomExtension( - stix2.v21.UserAccount, 'x-some-extension-ext', [ - ('keys', stix2.properties.ListProperty(stix2.properties.DictionaryProperty, required=True)), + 'x-some-extension-ext', [ + ('keys', stix2.properties.ListProperty(stix2.properties.DictionaryProperty, required=True)), ], ) class SomeCustomExtension: @@ -966,53 +966,11 @@ def test_custom_extension_with_list_and_dict_properties_observable_type(data): assert data == str(example) -def test_custom_extension_invalid_observable(): - # These extensions are being applied to improperly-created Observables. - # The Observable classes should have been created with the CustomObservable decorator. - class Foo(object): - pass - with pytest.raises(ValueError) as excinfo: - @stix2.v21.CustomExtension( - Foo, 'x-new-ext', [ - ('property1', stix2.properties.StringProperty(required=True)), - ], - ) - class FooExtension(): - pass # pragma: no cover - assert str(excinfo.value) == "'observable' must be a valid Observable class!" - - class Bar(stix2.v21.observables._Observable): - pass - with pytest.raises(ValueError) as excinfo: - @stix2.v21.CustomExtension( - Bar, 'x-new-ext', [ - ('property1', stix2.properties.StringProperty(required=True)), - ], - ) - class BarExtension(): - pass - assert "Unknown observable type" in str(excinfo.value) - assert "Custom observables must be created with the @CustomObservable decorator." in str(excinfo.value) - - class Baz(stix2.v21.observables._Observable): - _type = 'Baz' - with pytest.raises(ValueError) as excinfo: - @stix2.v21.CustomExtension( - Baz, 'x-new-ext', [ - ('property1', stix2.properties.StringProperty(required=True)), - ], - ) - class BazExtension(): - pass - assert "Unknown observable type" in str(excinfo.value) - assert "Custom observables must be created with the @CustomObservable decorator." in str(excinfo.value) - - def test_custom_extension_invalid_type_name(): with pytest.raises(ValueError) as excinfo: @stix2.v21.CustomExtension( - stix2.v21.File, 'x', { - 'property1': stix2.properties.StringProperty(required=True), + 'x', { + 'property1': stix2.properties.StringProperty(required=True), }, ) class FooExtension(): @@ -1021,8 +979,8 @@ def test_custom_extension_invalid_type_name(): with pytest.raises(ValueError) as excinfo: @stix2.v21.CustomExtension( - stix2.v21.File, 'x_new_ext', { - 'property1': stix2.properties.StringProperty(required=True), + 'x_new_ext', { + 'property1': stix2.properties.StringProperty(required=True), }, ) class BlaExtension(): @@ -1031,8 +989,8 @@ def test_custom_extension_invalid_type_name(): with pytest.raises(ValueError) as excinfo: @stix2.v21.CustomExtension( - stix2.v21.File, '7x-new-ext', { - 'property1': stix2.properties.StringProperty(required=True), + '7x-new-ext', { + 'property1': stix2.properties.StringProperty(required=True), }, ) class Bla2Extension(): @@ -1042,29 +1000,29 @@ def test_custom_extension_invalid_type_name(): def test_custom_extension_no_properties(): with pytest.raises(ValueError): - @stix2.v21.CustomExtension(stix2.v21.DomainName, 'x-new2-ext', None) + @stix2.v21.CustomExtension('x-new2-ext', None) class BarExtension(): pass def test_custom_extension_empty_properties(): with pytest.raises(ValueError): - @stix2.v21.CustomExtension(stix2.v21.DomainName, 'x-new2-ext', []) + @stix2.v21.CustomExtension('x-new2-ext', []) class BarExtension(): pass def test_custom_extension_dict_properties(): with pytest.raises(ValueError): - @stix2.v21.CustomExtension(stix2.v21.DomainName, 'x-new2-ext', {}) + @stix2.v21.CustomExtension('x-new2-ext', {}) class BarExtension(): pass def test_custom_extension_no_init_1(): @stix2.v21.CustomExtension( - stix2.v21.DomainName, 'x-new-extension-ext', [ - ('property1', stix2.properties.StringProperty(required=True)), + 'x-new-extension-ext', [ + ('property1', stix2.properties.StringProperty(required=True)), ], ) class NewExt(): @@ -1076,8 +1034,8 @@ def test_custom_extension_no_init_1(): def test_custom_extension_no_init_2(): @stix2.v21.CustomExtension( - stix2.v21.DomainName, 'x-new2-ext', [ - ('property1', stix2.properties.StringProperty(required=True)), + 'x-new2-ext', [ + ('property1', stix2.properties.StringProperty(required=True)), ], ) class NewExt2(object): @@ -1090,8 +1048,8 @@ def test_custom_extension_no_init_2(): def test_invalid_custom_property_in_extension(): with pytest.raises(ValueError) as excinfo: @stix2.v21.CustomExtension( - stix2.v21.DomainName, 'x-new3-ext', [ - ('6property1', stix2.properties.StringProperty(required=True)), + 'x-new3-ext', [ + ('6property1', stix2.properties.StringProperty(required=True)), ], ) class NewExt(): @@ -1205,7 +1163,7 @@ def test_register_custom_object(): def test_extension_property_location(): assert 'extensions' in stix2.v21.OBJ_MAP_OBSERVABLE['x-new-observable']._properties - assert 'extensions' not in stix2.v21.EXT_MAP['domain-name']['x-new-ext']._properties + assert 'extensions' not in stix2.v21.EXT_MAP['x-new-ext']._properties @pytest.mark.parametrize( @@ -1312,9 +1270,9 @@ def test_register_duplicate_observable(): def test_register_observable_custom_extension(): @stix2.v21.CustomExtension( - stix2.v21.DomainName, 'x-new-2-ext', [ - ('property1', stix2.properties.StringProperty(required=True)), - ('property2', stix2.properties.IntegerProperty()), + 'x-new-2-ext', [ + ('property1', stix2.properties.StringProperty(required=True)), + ('property2', stix2.properties.IntegerProperty()), ], ) class NewExtension2(): @@ -1324,15 +1282,15 @@ def test_register_observable_custom_extension(): v = 'v21' assert 'domain-name' in stix2.parsing.STIX2_OBJ_MAPS[v]['observables'] - assert example._type in stix2.parsing.STIX2_OBJ_MAPS[v]['observable-extensions']['domain-name'] + assert example._type in stix2.parsing.STIX2_OBJ_MAPS[v]['extensions'] def test_register_duplicate_observable_extension(): with pytest.raises(DuplicateRegistrationError) as excinfo: @stix2.v21.CustomExtension( - stix2.v21.DomainName, 'x-new-2-ext', [ - ('property1', stix2.properties.StringProperty(required=True)), - ('property2', stix2.properties.IntegerProperty()), + 'x-new-2-ext', [ + ('property1', stix2.properties.StringProperty(required=True)), + ('property2', stix2.properties.IntegerProperty()), ], ) class NewExtension2(): diff --git a/stix2/test/v21/test_deterministic_ids.py b/stix2/test/v21/test_deterministic_ids.py index 1e6e2d4..29bbdc8 100644 --- a/stix2/test/v21/test_deterministic_ids.py +++ b/stix2/test/v21/test_deterministic_ids.py @@ -48,11 +48,7 @@ def test_no_contrib_props_defined(): _properties = OrderedDict(( ('type', TypeProperty(_type, spec_version='2.1')), ('id', IDProperty(_type, spec_version='2.1')), - ( - 'extensions', ExtensionsProperty( - spec_version='2.1', enclosing_type=_type, - ), - ), + ('extensions', ExtensionsProperty(spec_version='2.1')), )) _id_contributing_properties = [] @@ -69,11 +65,7 @@ def test_json_compatible_prop_values(): _properties = OrderedDict(( ('type', TypeProperty(_type, spec_version='2.1')), ('id', IDProperty(_type, spec_version='2.1')), - ( - 'extensions', ExtensionsProperty( - spec_version='2.1', enclosing_type=_type, - ), - ), + ('extensions', ExtensionsProperty(spec_version='2.1')), ('string', StringProperty()), ('int', IntegerProperty()), ('float', FloatProperty()), @@ -109,11 +101,7 @@ def test_json_incompatible_timestamp_value(): _properties = OrderedDict(( ('type', TypeProperty(_type, spec_version='2.1')), ('id', IDProperty(_type, spec_version='2.1')), - ( - 'extensions', ExtensionsProperty( - spec_version='2.1', enclosing_type=_type, - ), - ), + ('extensions', ExtensionsProperty(spec_version='2.1')), ('timestamp', TimestampProperty()), )) _id_contributing_properties = ['timestamp'] @@ -145,11 +133,7 @@ def test_embedded_object(): _properties = OrderedDict(( ('type', TypeProperty(_type, spec_version='2.1')), ('id', IDProperty(_type, spec_version='2.1')), - ( - 'extensions', ExtensionsProperty( - spec_version='2.1', enclosing_type=_type, - ), - ), + ('extensions', ExtensionsProperty(spec_version='2.1')), ('sub_obj', EmbeddedObjectProperty(type=SubObj)), )) _id_contributing_properties = ['sub_obj'] @@ -176,11 +160,7 @@ def test_empty_hash(): _properties = OrderedDict(( ('type', TypeProperty(_type, spec_version='2.1')), ('id', IDProperty(_type, spec_version='2.1')), - ( - 'extensions', ExtensionsProperty( - spec_version='2.1', enclosing_type=_type, - ), - ), + ('extensions', ExtensionsProperty(spec_version='2.1')), ('hashes', HashesProperty()), )) _id_contributing_properties = ['hashes'] diff --git a/stix2/test/v21/test_malware_analysis.py b/stix2/test/v21/test_malware_analysis.py index 22f4171..bf4bbe6 100644 --- a/stix2/test/v21/test_malware_analysis.py +++ b/stix2/test/v21/test_malware_analysis.py @@ -10,13 +10,9 @@ MALWARE_ANALYSIS_JSON = """{ "type": "malware-analysis", "spec_version": "2.1", "id": "malware-analysis--f8afc020-f92f-4906-a971-88ee5882eb46", + "created_by_ref": "identity--e0353ed3-991e-4f71-a332-114c2f10b84f", "created": "2017-11-28T09:44:58.418Z", "modified": "2017-12-31T21:27:49.754Z", - "created_by_ref": "identity--e0353ed3-991e-4f71-a332-114c2f10b84f", - "labels": [ - "label1", - "label2" - ], "product": "Acme Malware Analyzer", "version": "2.5", "host_vm_ref": "software--1bda7336-fe67-469f-a8ca-ab6268b0449b", @@ -40,7 +36,11 @@ MALWARE_ANALYSIS_JSON = """{ "file--fc27e371-6c88-4c5c-868a-4dda0e60b167", "url--6f7a74cd-8eb2-4b88-a4da-aa878e50ac2e" ], - "sample_ref": "email-addr--499a32d7-74c1-4276-ace9-725ac933e243" + "sample_ref": "email-addr--499a32d7-74c1-4276-ace9-725ac933e243", + "labels": [ + "label1", + "label2" + ] }""" diff --git a/stix2/test/v21/test_properties.py b/stix2/test/v21/test_properties.py index 36ff858..2096ccb 100644 --- a/stix2/test/v21/test_properties.py +++ b/stix2/test/v21/test_properties.py @@ -244,7 +244,7 @@ def test_embedded_property(): def test_extension_property_valid(): - ext_prop = ExtensionsProperty(spec_version='2.1', enclosing_type='file') + ext_prop = ExtensionsProperty(spec_version='2.1') assert ext_prop({ 'windows-pebinary-ext': { 'pe_type': 'exe', @@ -253,13 +253,13 @@ def test_extension_property_valid(): def test_extension_property_invalid1(): - ext_prop = ExtensionsProperty(spec_version='2.1', enclosing_type='file') + ext_prop = ExtensionsProperty(spec_version='2.1') with pytest.raises(ValueError): ext_prop.clean(1) def test_extension_property_invalid2(): - ext_prop = ExtensionsProperty(spec_version='2.1', enclosing_type='file') + ext_prop = ExtensionsProperty(spec_version='2.1') with pytest.raises(CustomContentError): ext_prop.clean( { @@ -270,19 +270,6 @@ def test_extension_property_invalid2(): ) -def test_extension_property_invalid_type(): - ext_prop = ExtensionsProperty(spec_version='2.1', enclosing_type='indicator') - with pytest.raises(CustomContentError) as excinfo: - ext_prop.clean( - { - 'windows-pebinary-ext': { - 'pe_type': 'exe', - }, - }, - ) - assert "Can't parse unknown extension" in str(excinfo.value) - - def test_extension_at_least_one_property_constraint(): with pytest.raises(AtLeastOnePropertyError): stix2.v21.TCPExt() From 4340be13c44c94a9cb2be1eb40b8af22ee0f545b Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Mon, 9 Nov 2020 20:58:34 -0500 Subject: [PATCH 06/56] changes to the ExtensionsProperty --- stix2/v20/bundle.py | 2 +- stix2/v20/observables.py | 52 ++++++++++++++++++++-------------------- stix2/v20/sdo.py | 2 +- stix2/v21/common.py | 4 ++-- stix2/v21/observables.py | 44 +++++++++++++++++----------------- stix2/v21/sdo.py | 40 +++++++++++++++---------------- stix2/v21/sro.py | 4 ++-- 7 files changed, 74 insertions(+), 74 deletions(-) diff --git a/stix2/v20/bundle.py b/stix2/v20/bundle.py index 6a663d6..e05522e 100644 --- a/stix2/v20/bundle.py +++ b/stix2/v20/bundle.py @@ -20,7 +20,7 @@ class Bundle(_STIXBase20): # Not technically correct: STIX 2.0 spec doesn't say spec_version must # have this value, but it's all we support for now. ('spec_version', StringProperty(fixed='2.0')), - ('objects', ListProperty(STIXObjectProperty(spec_version="2.0"))), + ('objects', ListProperty(STIXObjectProperty(spec_version='2.0'))), ]) def __init__(self, *args, **kwargs): diff --git a/stix2/v20/observables.py b/stix2/v20/observables.py index cec979e..8dd7ff8 100644 --- a/stix2/v20/observables.py +++ b/stix2/v20/observables.py @@ -31,7 +31,7 @@ class Artifact(_Observable): ('payload_bin', BinaryProperty()), ('url', StringProperty()), ('hashes', HashesProperty(spec_version='2.0')), - ('extensions', ExtensionsProperty(spec_version="2.0", enclosing_type=_type)), + ('extensions', ExtensionsProperty(spec_version='2.0')), ]) def _check_object_constraints(self): @@ -51,7 +51,7 @@ class AutonomousSystem(_Observable): ('number', IntegerProperty(required=True)), ('name', StringProperty()), ('rir', StringProperty()), - ('extensions', ExtensionsProperty(spec_version="2.0", enclosing_type=_type)), + ('extensions', ExtensionsProperty(spec_version='2.0')), ]) @@ -70,7 +70,7 @@ class Directory(_Observable): ('modified', TimestampProperty()), ('accessed', TimestampProperty()), ('contains_refs', ListProperty(ObjectReferenceProperty(valid_types=['file', 'directory']))), - ('extensions', ExtensionsProperty(spec_version="2.0", enclosing_type=_type)), + ('extensions', ExtensionsProperty(spec_version='2.0')), ]) @@ -84,7 +84,7 @@ class DomainName(_Observable): ('type', TypeProperty(_type, spec_version='2.0')), ('value', StringProperty(required=True)), ('resolves_to_refs', ListProperty(ObjectReferenceProperty(valid_types=['ipv4-addr', 'ipv6-addr', 'domain-name']))), - ('extensions', ExtensionsProperty(spec_version="2.0", enclosing_type=_type)), + ('extensions', ExtensionsProperty(spec_version='2.0')), ]) @@ -99,7 +99,7 @@ class EmailAddress(_Observable): ('value', StringProperty(required=True)), ('display_name', StringProperty()), ('belongs_to_ref', ObjectReferenceProperty(valid_types='user-account')), - ('extensions', ExtensionsProperty(spec_version="2.0", enclosing_type=_type)), + ('extensions', ExtensionsProperty(spec_version='2.0')), ]) @@ -138,11 +138,11 @@ class EmailMessage(_Observable): ('bcc_refs', ListProperty(ObjectReferenceProperty(valid_types='email-addr'))), ('subject', StringProperty()), ('received_lines', ListProperty(StringProperty)), - ('additional_header_fields', DictionaryProperty(spec_version="2.0")), + ('additional_header_fields', DictionaryProperty(spec_version='2.0')), ('body', StringProperty()), ('body_multipart', ListProperty(EmbeddedObjectProperty(type=EmailMIMEComponent))), ('raw_email_ref', ObjectReferenceProperty(valid_types='artifact')), - ('extensions', ExtensionsProperty(spec_version="2.0", enclosing_type=_type)), + ('extensions', ExtensionsProperty(spec_version='2.0')), ]) def _check_object_constraints(self): @@ -199,7 +199,7 @@ class PDFExt(_Extension): _properties = OrderedDict([ ('version', StringProperty()), ('is_optimized', BooleanProperty()), - ('document_info_dict', DictionaryProperty(spec_version="2.0")), + ('document_info_dict', DictionaryProperty(spec_version='2.0')), ('pdfid0', StringProperty()), ('pdfid1', StringProperty()), ]) @@ -216,7 +216,7 @@ class RasterImageExt(_Extension): ('image_width', IntegerProperty()), ('bits_per_pixel', IntegerProperty()), ('image_compression_algorithm', StringProperty()), - ('exif_tags', DictionaryProperty(spec_version="2.0")), + ('exif_tags', DictionaryProperty(spec_version='2.0')), ]) @@ -323,7 +323,7 @@ class File(_Observable): ('decryption_key', StringProperty()), ('contains_refs', ListProperty(ObjectReferenceProperty)), ('content_ref', ObjectReferenceProperty(valid_types='artifact')), - ('extensions', ExtensionsProperty(spec_version="2.0", enclosing_type=_type)), + ('extensions', ExtensionsProperty(spec_version='2.0')), ]) def _check_object_constraints(self): @@ -343,7 +343,7 @@ class IPv4Address(_Observable): ('value', StringProperty(required=True)), ('resolves_to_refs', ListProperty(ObjectReferenceProperty(valid_types='mac-addr'))), ('belongs_to_refs', ListProperty(ObjectReferenceProperty(valid_types='autonomous-system'))), - ('extensions', ExtensionsProperty(spec_version="2.0", enclosing_type=_type)), + ('extensions', ExtensionsProperty(spec_version='2.0')), ]) @@ -358,7 +358,7 @@ class IPv6Address(_Observable): ('value', StringProperty(required=True)), ('resolves_to_refs', ListProperty(ObjectReferenceProperty(valid_types='mac-addr'))), ('belongs_to_refs', ListProperty(ObjectReferenceProperty(valid_types='autonomous-system'))), - ('extensions', ExtensionsProperty(spec_version="2.0", enclosing_type=_type)), + ('extensions', ExtensionsProperty(spec_version='2.0')), ]) @@ -371,7 +371,7 @@ class MACAddress(_Observable): _properties = OrderedDict([ ('type', TypeProperty(_type, spec_version='2.0')), ('value', StringProperty(required=True)), - ('extensions', ExtensionsProperty(spec_version="2.0", enclosing_type=_type)), + ('extensions', ExtensionsProperty(spec_version='2.0')), ]) @@ -384,7 +384,7 @@ class Mutex(_Observable): _properties = OrderedDict([ ('type', TypeProperty(_type, spec_version='2.0')), ('name', StringProperty(required=True)), - ('extensions', ExtensionsProperty(spec_version="2.0", enclosing_type=_type)), + ('extensions', ExtensionsProperty(spec_version='2.0')), ]) @@ -398,7 +398,7 @@ class HTTPRequestExt(_Extension): ('request_method', StringProperty(required=True)), ('request_value', StringProperty(required=True)), ('request_version', StringProperty()), - ('request_header', DictionaryProperty(spec_version="2.0")), + ('request_header', DictionaryProperty(spec_version='2.0')), ('message_body_length', IntegerProperty()), ('message_body_data_ref', ObjectReferenceProperty(valid_types='artifact')), ]) @@ -449,7 +449,7 @@ class SocketExt(_Extension): "PF_NETROM", ]), ), - ('options', DictionaryProperty(spec_version="2.0")), + ('options', DictionaryProperty(spec_version='2.0')), ( 'socket_type', EnumProperty(allowed=[ "SOCK_STREAM", @@ -496,12 +496,12 @@ class NetworkTraffic(_Observable): ('dst_byte_count', IntegerProperty()), ('src_packets', IntegerProperty()), ('dst_packets', IntegerProperty()), - ('ipfix', DictionaryProperty(spec_version="2.0")), + ('ipfix', DictionaryProperty(spec_version='2.0')), ('src_payload_ref', ObjectReferenceProperty(valid_types='artifact')), ('dst_payload_ref', ObjectReferenceProperty(valid_types='artifact')), ('encapsulates_refs', ListProperty(ObjectReferenceProperty(valid_types='network-traffic'))), ('encapsulates_by_ref', ObjectReferenceProperty(valid_types='network-traffic')), - ('extensions', ExtensionsProperty(spec_version="2.0", enclosing_type=_type)), + ('extensions', ExtensionsProperty(spec_version='2.0')), ]) def _check_object_constraints(self): @@ -521,7 +521,7 @@ class WindowsProcessExt(_Extension): ('priority', StringProperty()), ('owner_sid', StringProperty()), ('window_title', StringProperty()), - ('startup_info', DictionaryProperty(spec_version="2.0")), + ('startup_info', DictionaryProperty(spec_version='2.0')), ]) @@ -584,13 +584,13 @@ class Process(_Observable): ('cwd', StringProperty()), ('arguments', ListProperty(StringProperty)), ('command_line', StringProperty()), - ('environment_variables', DictionaryProperty(spec_version="2.0")), + ('environment_variables', DictionaryProperty(spec_version='2.0')), ('opened_connection_refs', ListProperty(ObjectReferenceProperty(valid_types='network-traffic'))), ('creator_user_ref', ObjectReferenceProperty(valid_types='user-account')), ('binary_ref', ObjectReferenceProperty(valid_types='file')), ('parent_ref', ObjectReferenceProperty(valid_types='process')), ('child_refs', ListProperty(ObjectReferenceProperty('process'))), - ('extensions', ExtensionsProperty(spec_version="2.0", enclosing_type=_type)), + ('extensions', ExtensionsProperty(spec_version='2.0')), ]) def _check_object_constraints(self): @@ -621,7 +621,7 @@ class Software(_Observable): ('languages', ListProperty(StringProperty)), ('vendor', StringProperty()), ('version', StringProperty()), - ('extensions', ExtensionsProperty(spec_version="2.0", enclosing_type=_type)), + ('extensions', ExtensionsProperty(spec_version='2.0')), ]) @@ -634,7 +634,7 @@ class URL(_Observable): _properties = OrderedDict([ ('type', TypeProperty(_type, spec_version='2.0')), ('value', StringProperty(required=True)), - ('extensions', ExtensionsProperty(spec_version="2.0", enclosing_type=_type)), + ('extensions', ExtensionsProperty(spec_version='2.0')), ]) @@ -673,7 +673,7 @@ class UserAccount(_Observable): ('password_last_changed', TimestampProperty()), ('account_first_login', TimestampProperty()), ('account_last_login', TimestampProperty()), - ('extensions', ExtensionsProperty(spec_version="2.0", enclosing_type=_type)), + ('extensions', ExtensionsProperty(spec_version='2.0')), ]) @@ -720,7 +720,7 @@ class WindowsRegistryKey(_Observable): ('modified', TimestampProperty()), ('creator_user_ref', ObjectReferenceProperty(valid_types='user-account')), ('number_of_subkeys', IntegerProperty()), - ('extensions', ExtensionsProperty(spec_version="2.0", enclosing_type=_type)), + ('extensions', ExtensionsProperty(spec_version='2.0')), ]) @@ -771,7 +771,7 @@ class X509Certificate(_Observable): ('subject_public_key_modulus', StringProperty()), ('subject_public_key_exponent', IntegerProperty()), ('x509_v3_extensions', EmbeddedObjectProperty(type=X509V3ExtensionsType)), - ('extensions', ExtensionsProperty(spec_version="2.0", enclosing_type=_type)), + ('extensions', ExtensionsProperty(spec_version='2.0')), ]) diff --git a/stix2/v20/sdo.py b/stix2/v20/sdo.py index 8cbd94b..630c125 100644 --- a/stix2/v20/sdo.py +++ b/stix2/v20/sdo.py @@ -211,7 +211,7 @@ class ObservedData(_DomainObject): ('first_observed', TimestampProperty(required=True)), ('last_observed', TimestampProperty(required=True)), ('number_observed', IntegerProperty(min=1, max=999999999, required=True)), - ('objects', ObservableProperty(spec_version="2.0", required=True)), + ('objects', ObservableProperty(spec_version='2.0', required=True)), ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('external_references', ListProperty(ExternalReference)), diff --git a/stix2/v21/common.py b/stix2/v21/common.py index 60c4068..b179103 100644 --- a/stix2/v21/common.py +++ b/stix2/v21/common.py @@ -101,7 +101,7 @@ class LanguageContent(_STIXBase21): ('external_references', ListProperty(ExternalReference)), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), - ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ('extensions', ExtensionsProperty(spec_version='2.1')), ]) @@ -203,7 +203,7 @@ class MarkingDefinition(_STIXBase21, _MarkingsMixin): ('external_references', ListProperty(ExternalReference)), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), - ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ('extensions', ExtensionsProperty(spec_version='2.1')), ]) def __init__(self, **kwargs): diff --git a/stix2/v21/observables.py b/stix2/v21/observables.py index 11e1d9f..9af0dab 100644 --- a/stix2/v21/observables.py +++ b/stix2/v21/observables.py @@ -39,7 +39,7 @@ class Artifact(_Observable): ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ('defanged', BooleanProperty(default=lambda: False)), - ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ('extensions', ExtensionsProperty(spec_version='2.1')), ]) _id_contributing_properties = ["hashes", "payload_bin"] @@ -65,7 +65,7 @@ class AutonomousSystem(_Observable): ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ('defanged', BooleanProperty(default=lambda: False)), - ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ('extensions', ExtensionsProperty(spec_version='2.1')), ]) _id_contributing_properties = ["number"] @@ -90,7 +90,7 @@ class Directory(_Observable): ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ('defanged', BooleanProperty(default=lambda: False)), - ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ('extensions', ExtensionsProperty(spec_version='2.1')), ]) _id_contributing_properties = ["path"] @@ -110,7 +110,7 @@ class DomainName(_Observable): ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ('defanged', BooleanProperty(default=lambda: False)), - ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ('extensions', ExtensionsProperty(spec_version='2.1')), ]) _id_contributing_properties = ["value"] @@ -131,7 +131,7 @@ class EmailAddress(_Observable): ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ('defanged', BooleanProperty(default=lambda: False)), - ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ('extensions', ExtensionsProperty(spec_version='2.1')), ]) _id_contributing_properties = ["value"] @@ -143,7 +143,7 @@ class EmailMIMEComponent(_STIXBase21): _properties = OrderedDict([ ('body', StringProperty()), - ('body_raw_ref', ReferenceProperty(valid_types=['artifact', 'file'], spec_version="2.1")), + ('body_raw_ref', ReferenceProperty(valid_types=['artifact', 'file'], spec_version='2.1')), ('content_type', StringProperty()), ('content_disposition', StringProperty()), ]) @@ -181,7 +181,7 @@ class EmailMessage(_Observable): ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ('defanged', BooleanProperty(default=lambda: False)), - ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ('extensions', ExtensionsProperty(spec_version='2.1')), ]) _id_contributing_properties = ["from_ref", "subject", "body"] @@ -200,7 +200,7 @@ class ArchiveExt(_Extension): _type = 'archive-ext' _properties = OrderedDict([ - ('contains_refs', ListProperty(ReferenceProperty(valid_types=['file', 'directory'], spec_version="2.1"), required=True)), + ('contains_refs', ListProperty(ReferenceProperty(valid_types=['file', 'directory'], spec_version='2.1'), required=True)), ('comment', StringProperty()), ]) @@ -362,7 +362,7 @@ class File(_Observable): ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ('defanged', BooleanProperty(default=lambda: False)), - ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ('extensions', ExtensionsProperty(spec_version='2.1')), ]) _id_contributing_properties = ["hashes", "name", "parent_directory_ref", "extensions"] @@ -387,7 +387,7 @@ class IPv4Address(_Observable): ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ('defanged', BooleanProperty(default=lambda: False)), - ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ('extensions', ExtensionsProperty(spec_version='2.1')), ]) _id_contributing_properties = ["value"] @@ -408,7 +408,7 @@ class IPv6Address(_Observable): ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ('defanged', BooleanProperty(default=lambda: False)), - ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ('extensions', ExtensionsProperty(spec_version='2.1')), ]) _id_contributing_properties = ["value"] @@ -427,7 +427,7 @@ class MACAddress(_Observable): ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ('defanged', BooleanProperty(default=lambda: False)), - ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ('extensions', ExtensionsProperty(spec_version='2.1')), ]) _id_contributing_properties = ["value"] @@ -446,7 +446,7 @@ class Mutex(_Observable): ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ('defanged', BooleanProperty(default=lambda: False)), - ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ('extensions', ExtensionsProperty(spec_version='2.1')), ]) _id_contributing_properties = ["name"] @@ -463,7 +463,7 @@ class HTTPRequestExt(_Extension): ('request_version', StringProperty()), ('request_header', DictionaryProperty(spec_version='2.1')), ('message_body_length', IntegerProperty()), - ('message_body_data_ref', ReferenceProperty(valid_types='artifact', spec_version="2.1")), + ('message_body_data_ref', ReferenceProperty(valid_types='artifact', spec_version='2.1')), ]) @@ -573,7 +573,7 @@ class NetworkTraffic(_Observable): ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ('defanged', BooleanProperty(default=lambda: False)), - ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ('extensions', ExtensionsProperty(spec_version='2.1')), ]) _id_contributing_properties = ["start", "src_ref", "dst_ref", "src_port", "dst_port", "protocols"] @@ -642,7 +642,7 @@ class WindowsServiceExt(_Extension): "SERVICE_SYSTEM_ALERT", ]), ), - ('service_dll_refs', ListProperty(ReferenceProperty(valid_types='file', spec_version="2.1"))), + ('service_dll_refs', ListProperty(ReferenceProperty(valid_types='file', spec_version='2.1'))), ( 'service_type', EnumProperty(allowed=[ "SERVICE_KERNEL_DRIVER", @@ -690,7 +690,7 @@ class Process(_Observable): ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ('defanged', BooleanProperty(default=lambda: False)), - ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ('extensions', ExtensionsProperty(spec_version='2.1')), ]) _id_contributing_properties = [] @@ -728,7 +728,7 @@ class Software(_Observable): ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ('defanged', BooleanProperty(default=lambda: False)), - ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ('extensions', ExtensionsProperty(spec_version='2.1')), ]) _id_contributing_properties = ["name", "cpe", "swid", "vendor", "version"] @@ -747,7 +747,7 @@ class URL(_Observable): ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ('defanged', BooleanProperty(default=lambda: False)), - ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ('extensions', ExtensionsProperty(spec_version='2.1')), ]) _id_contributing_properties = ["value"] @@ -793,7 +793,7 @@ class UserAccount(_Observable): ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ('defanged', BooleanProperty(default=lambda: False)), - ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ('extensions', ExtensionsProperty(spec_version='2.1')), ]) _id_contributing_properties = ["account_type", "user_id", "account_login"] @@ -846,7 +846,7 @@ class WindowsRegistryKey(_Observable): ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ('defanged', BooleanProperty(default=lambda: False)), - ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ('extensions', ExtensionsProperty(spec_version='2.1')), ]) _id_contributing_properties = ["key", "values"] @@ -903,7 +903,7 @@ class X509Certificate(_Observable): ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ('defanged', BooleanProperty(default=lambda: False)), - ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ('extensions', ExtensionsProperty(spec_version='2.1')), ]) _id_contributing_properties = ["hashes", "serial_number"] diff --git a/stix2/v21/sdo.py b/stix2/v21/sdo.py index d538852..ba479b7 100644 --- a/stix2/v21/sdo.py +++ b/stix2/v21/sdo.py @@ -46,7 +46,7 @@ class AttackPattern(_DomainObject): ('external_references', ListProperty(ExternalReference)), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), - ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ('extensions', ExtensionsProperty(spec_version='2.1')), ]) @@ -76,7 +76,7 @@ class Campaign(_DomainObject): ('external_references', ListProperty(ExternalReference)), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), - ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ('extensions', ExtensionsProperty(spec_version='2.1')), ]) def _check_object_constraints(self): @@ -112,7 +112,7 @@ class CourseOfAction(_DomainObject): ('external_references', ListProperty(ExternalReference)), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), - ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ('extensions', ExtensionsProperty(spec_version='2.1')), ]) @@ -140,7 +140,7 @@ class Grouping(_DomainObject): ('external_references', ListProperty(ExternalReference)), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), - ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ('extensions', ExtensionsProperty(spec_version='2.1')), ]) @@ -170,7 +170,7 @@ class Identity(_DomainObject): ('external_references', ListProperty(ExternalReference)), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), - ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ('extensions', ExtensionsProperty(spec_version='2.1')), ]) @@ -203,7 +203,7 @@ class Indicator(_DomainObject): ('external_references', ListProperty(ExternalReference)), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), - ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ('extensions', ExtensionsProperty(spec_version='2.1')), ]) def __init__(self, *args, **kwargs): @@ -261,7 +261,7 @@ class Infrastructure(_DomainObject): ('external_references', ListProperty(ExternalReference)), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), - ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ('extensions', ExtensionsProperty(spec_version='2.1')), ]) def _check_object_constraints(self): @@ -304,7 +304,7 @@ class IntrusionSet(_DomainObject): ('external_references', ListProperty(ExternalReference)), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), - ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ('extensions', ExtensionsProperty(spec_version='2.1')), ]) def _check_object_constraints(self): @@ -349,7 +349,7 @@ class Location(_DomainObject): ('external_references', ListProperty(ExternalReference)), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), - ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ('extensions', ExtensionsProperty(spec_version='2.1')), ]) def _check_object_constraints(self): @@ -459,7 +459,7 @@ class Malware(_DomainObject): ('external_references', ListProperty(ExternalReference)), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), - ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ('extensions', ExtensionsProperty(spec_version='2.1')), ]) def _check_object_constraints(self): @@ -507,7 +507,7 @@ class MalwareAnalysis(_DomainObject): ('result_name', StringProperty()), ('result', StringProperty()), ('analysis_sco_refs', ListProperty(ReferenceProperty(valid_types="SCO", spec_version='2.1'))), - ('sample_ref', ReferenceProperty(valid_types="SCO", spec_version="2.1")), + ('sample_ref', ReferenceProperty(valid_types="SCO", spec_version='2.1')), ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('confidence', IntegerProperty()), @@ -515,7 +515,7 @@ class MalwareAnalysis(_DomainObject): ('external_references', ListProperty(ExternalReference)), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), - ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ('extensions', ExtensionsProperty(spec_version='2.1')), ]) def _check_object_constraints(self): @@ -548,7 +548,7 @@ class Note(_DomainObject): ('external_references', ListProperty(ExternalReference)), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), - ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ('extensions', ExtensionsProperty(spec_version='2.1')), ]) @@ -569,7 +569,7 @@ class ObservedData(_DomainObject): ('last_observed', TimestampProperty(required=True)), ('number_observed', IntegerProperty(min=1, max=999999999, required=True)), ('objects', ObservableProperty(spec_version='2.1')), - ('object_refs', ListProperty(ReferenceProperty(valid_types=["SCO", "SRO"], spec_version="2.1"))), + ('object_refs', ListProperty(ReferenceProperty(valid_types=["SCO", "SRO"], spec_version='2.1'))), ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('confidence', IntegerProperty()), @@ -577,7 +577,7 @@ class ObservedData(_DomainObject): ('external_references', ListProperty(ExternalReference)), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), - ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ('extensions', ExtensionsProperty(spec_version='2.1')), ]) def __init__(self, *args, **kwargs): @@ -642,7 +642,7 @@ class Opinion(_DomainObject): ('external_references', ListProperty(ExternalReference)), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), - ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ('extensions', ExtensionsProperty(spec_version='2.1')), ]) @@ -671,7 +671,7 @@ class Report(_DomainObject): ('external_references', ListProperty(ExternalReference)), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), - ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ('extensions', ExtensionsProperty(spec_version='2.1')), ]) @@ -708,7 +708,7 @@ class ThreatActor(_DomainObject): ('external_references', ListProperty(ExternalReference)), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), - ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ('extensions', ExtensionsProperty(spec_version='2.1')), ]) def _check_object_constraints(self): @@ -748,7 +748,7 @@ class Tool(_DomainObject): ('external_references', ListProperty(ExternalReference)), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), - ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ('extensions', ExtensionsProperty(spec_version='2.1')), ]) @@ -774,7 +774,7 @@ class Vulnerability(_DomainObject): ('external_references', ListProperty(ExternalReference)), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), - ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ('extensions', ExtensionsProperty(spec_version='2.1')), ]) diff --git a/stix2/v21/sro.py b/stix2/v21/sro.py index d0e07f8..a30e319 100644 --- a/stix2/v21/sro.py +++ b/stix2/v21/sro.py @@ -40,7 +40,7 @@ class Relationship(_RelationshipObject): ('external_references', ListProperty(ExternalReference)), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), - ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ('extensions', ExtensionsProperty(spec_version='2.1')), ]) # Explicitly define the first three kwargs to make readable Relationship declarations. @@ -97,7 +97,7 @@ class Sighting(_RelationshipObject): ('external_references', ListProperty(ExternalReference)), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), - ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ('extensions', ExtensionsProperty(spec_version='2.1')), ]) # Explicitly define the first kwargs to make readable Sighting declarations. From 22c435168883d4cd6a2b37beb10a1fe5c05f4768 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Tue, 10 Nov 2020 17:08:51 -0500 Subject: [PATCH 07/56] flatten the extensions map, remove enclosing_type from ExtensionsProperty other minor changes --- stix2/properties.py | 16 +++++++++------- stix2/v20/__init__.py | 32 ++++++++++++-------------------- stix2/v21/__init__.py | 33 ++++++++++++--------------------- stix2/workbench.py | 2 +- 4 files changed, 34 insertions(+), 49 deletions(-) diff --git a/stix2/properties.py b/stix2/properties.py index 1ca2dbe..cea664f 100644 --- a/stix2/properties.py +++ b/stix2/properties.py @@ -24,8 +24,8 @@ try: except ImportError: from collections import Mapping -TYPE_REGEX = re.compile(r'^\-?[a-z0-9]+(-[a-z0-9]+)*\-?$') -TYPE_21_REGEX = re.compile(r'^([a-z][a-z0-9]*)+(-[a-z0-9]+)*\-?$') +TYPE_REGEX = re.compile(r'^-?[a-z0-9]+(-[a-z0-9]+)*-?$') +TYPE_21_REGEX = re.compile(r'^([a-z][a-z0-9]*)+([a-z0-9-]+)*-?$') ERROR_INVALID_ID = ( "not a valid STIX identifier, must match --: {}" ) @@ -638,9 +638,8 @@ class ExtensionsProperty(DictionaryProperty): """Property for representing extensions on Observable objects. """ - def __init__(self, spec_version=stix2.DEFAULT_VERSION, allow_custom=False, enclosing_type=None, required=False): + def __init__(self, spec_version=stix2.DEFAULT_VERSION, allow_custom=False, required=False): self.allow_custom = allow_custom - self.enclosing_type = enclosing_type super(ExtensionsProperty, self).__init__(spec_version=spec_version, required=required) def clean(self, value): @@ -655,10 +654,10 @@ class ExtensionsProperty(DictionaryProperty): v = 'v' + self.spec_version.replace('.', '') - specific_type_map = STIX2_OBJ_MAPS[v]['observable-extensions'].get(self.enclosing_type, {}) + extension_type_map = STIX2_OBJ_MAPS[v].get('extensions', {}) for key, subvalue in dictified.items(): - if key in specific_type_map: - cls = specific_type_map[key] + if key in extension_type_map: + cls = extension_type_map[key] if type(subvalue) is dict: if self.allow_custom: subvalue['allow_custom'] = True @@ -673,6 +672,9 @@ class ExtensionsProperty(DictionaryProperty): else: if self.allow_custom: dictified[key] = subvalue + elif key.startswith('stix-extension--'): + _validate_id(key, '2.1', 'stix-extension') + dictified[key] = subvalue else: raise CustomContentError("Can't parse unknown extension type: {}".format(key)) return dictified diff --git a/stix2/v20/__init__.py b/stix2/v20/__init__.py index 002343b..85609e7 100644 --- a/stix2/v20/__init__.py +++ b/stix2/v20/__init__.py @@ -82,26 +82,18 @@ OBJ_MAP_OBSERVABLE = { } EXT_MAP = { - 'file': { - 'archive-ext': ArchiveExt, - 'ntfs-ext': NTFSExt, - 'pdf-ext': PDFExt, - 'raster-image-ext': RasterImageExt, - 'windows-pebinary-ext': WindowsPEBinaryExt, - }, - 'network-traffic': { - 'http-request-ext': HTTPRequestExt, - 'icmp-ext': ICMPExt, - 'socket-ext': SocketExt, - 'tcp-ext': TCPExt, - }, - 'process': { - 'windows-process-ext': WindowsProcessExt, - 'windows-service-ext': WindowsServiceExt, - }, - 'user-account': { - 'unix-account-ext': UNIXAccountExt, - }, + 'archive-ext': ArchiveExt, + 'ntfs-ext': NTFSExt, + 'pdf-ext': PDFExt, + 'raster-image-ext': RasterImageExt, + 'windows-pebinary-ext': WindowsPEBinaryExt, + 'http-request-ext': HTTPRequestExt, + 'icmp-ext': ICMPExt, + 'socket-ext': SocketExt, + 'tcp-ext': TCPExt, + 'windows-process-ext': WindowsProcessExt, + 'windows-service-ext': WindowsServiceExt, + 'unix-account-ext': UNIXAccountExt, } diff --git a/stix2/v21/__init__.py b/stix2/v21/__init__.py index 856ff13..81ab980 100644 --- a/stix2/v21/__init__.py +++ b/stix2/v21/__init__.py @@ -91,29 +91,20 @@ OBJ_MAP_OBSERVABLE = { } EXT_MAP = { - 'file': { - 'archive-ext': ArchiveExt, - 'ntfs-ext': NTFSExt, - 'pdf-ext': PDFExt, - 'raster-image-ext': RasterImageExt, - 'windows-pebinary-ext': WindowsPEBinaryExt, - }, - 'network-traffic': { - 'http-request-ext': HTTPRequestExt, - 'icmp-ext': ICMPExt, - 'socket-ext': SocketExt, - 'tcp-ext': TCPExt, - }, - 'process': { - 'windows-process-ext': WindowsProcessExt, - 'windows-service-ext': WindowsServiceExt, - }, - 'user-account': { - 'unix-account-ext': UNIXAccountExt, - }, + 'archive-ext': ArchiveExt, + 'ntfs-ext': NTFSExt, + 'pdf-ext': PDFExt, + 'raster-image-ext': RasterImageExt, + 'windows-pebinary-ext': WindowsPEBinaryExt, + 'http-request-ext': HTTPRequestExt, + 'icmp-ext': ICMPExt, + 'socket-ext': SocketExt, + 'tcp-ext': TCPExt, + 'windows-process-ext': WindowsProcessExt, + 'windows-service-ext': WindowsServiceExt, + 'unix-account-ext': UNIXAccountExt, } - # Ensure star-imports from this module get the right symbols. "base" is a # known problem, since there are multiple modules with that name and one can # accidentally overwrite another. diff --git a/stix2/workbench.py b/stix2/workbench.py index 3724bdb..ff5d680 100644 --- a/stix2/workbench.py +++ b/stix2/workbench.py @@ -55,7 +55,7 @@ from . import ( # noqa: F401 WindowsPEBinaryExt, WindowsPEOptionalHeaderType, WindowsPESection, WindowsProcessExt, WindowsRegistryKey, WindowsRegistryValueType, WindowsServiceExt, X509Certificate, - X509V3ExtenstionsType + X509V3ExtensionsType ) from .datastore.filters import FilterSet From e4165f96aabb80bc1139628046fc3365932aab12 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Tue, 10 Nov 2020 18:32:58 -0500 Subject: [PATCH 08/56] major changes to support the extension mechanism moved some version specific logic into their area, changes to decorators to support Object Registration with Extension --- stix2/base.py | 33 +++++++++++--------- stix2/custom.py | 23 +++++++++++--- stix2/parsing.py | 66 ++++++++++++++++------------------------ stix2/v20/base.py | 5 ++- stix2/v20/observables.py | 6 ++-- stix2/v21/base.py | 36 ++++++++++++++++++++-- stix2/v21/observables.py | 19 +++++++++--- stix2/v21/sdo.py | 17 +++++++++-- 8 files changed, 132 insertions(+), 73 deletions(-) diff --git a/stix2/base.py b/stix2/base.py index 21b6011..4a21cf0 100644 --- a/stix2/base.py +++ b/stix2/base.py @@ -127,8 +127,18 @@ class _STIXBase(Mapping): raise ValueError("'custom_properties' must be a dictionary") extra_kwargs = list(set(kwargs) - set(self._properties)) + if extra_kwargs and issubclass(cls, stix2.v21._Extension): + props_to_remove = ['extends_stix_object_definition', 'is_new_object', 'is_extension_so'] + extra_kwargs = [prop for prop in extra_kwargs if prop not in props_to_remove] + if extra_kwargs and not self._allow_custom: - raise ExtraPropertiesError(cls, extra_kwargs) + ext_found = False + for key_id, ext_def in kwargs.get('extensions', {}).items(): + if key_id.startswith('stix-extension--'): + ext_found = True + break + if ext_found is False: + raise ExtraPropertiesError(cls, extra_kwargs) # because allow_custom is true, any extra kwargs are custom if custom_props or extra_kwargs: @@ -155,7 +165,13 @@ class _STIXBase(Mapping): required_properties = set(get_required_properties(self._properties)) missing_kwargs = required_properties - set(setting_kwargs) if missing_kwargs: - raise MissingPropertiesError(cls, missing_kwargs) + new_ext_check = ( + getattr(self, 'extends_stix_object_definition', False) or + getattr(self, 'is_new_object', False) or + getattr(self, 'is_extension_so', False) + ) and issubclass(cls, stix2.v21._Extension) + if new_ext_check is False: + raise MissingPropertiesError(cls, missing_kwargs) for prop_name, prop_metadata in self._properties.items(): self._check_property(prop_name, prop_metadata, setting_kwargs) @@ -274,21 +290,8 @@ class _Observable(_STIXBase): def __init__(self, **kwargs): # the constructor might be called independently of an observed data object self._STIXBase__valid_refs = kwargs.pop('_valid_refs', []) - self._properties['extensions'].allow_custom = kwargs.get('allow_custom', False) super(_Observable, self).__init__(**kwargs) - if 'id' not in kwargs and not isinstance(self, stix2.v20._Observable): - # Specific to 2.1+ observables: generate a deterministic ID - id_ = self._generate_id() - - # Spec says fall back to UUIDv4 if no contributing properties were - # given. That's what already happened (the following is actually - # overwriting the default uuidv4), so nothing to do here. - if id_ is not None: - # Can't assign to self (we're immutable), so slip the ID in - # more sneakily. - self._inner["id"] = id_ - def _check_ref(self, ref, prop, prop_name): """ Only for checking `*_ref` or `*_refs` properties in spec_version 2.0 diff --git a/stix2/custom.py b/stix2/custom.py index 08574ef..0a646fc 100644 --- a/stix2/custom.py +++ b/stix2/custom.py @@ -4,8 +4,8 @@ import six from .base import _cls_init from .parsing import ( - _register_marking, _register_object, _register_observable, - _register_observable_extension, + _get_extension_class, _register_extension, _register_marking, + _register_object, _register_observable, ) @@ -34,6 +34,11 @@ def _custom_object_builder(cls, type, properties, version, base_class): def __init__(self, **kwargs): base_class.__init__(self, **kwargs) _cls_init(cls, self, kwargs) + ext = getattr(self, 'with_extension', None) + if ext and version != '2.0': + if 'extensions' not in self._inner: + self._inner['extensions'] = {} + self._inner['extensions'][ext] = _get_extension_class(ext, version)() _CustomObject.__name__ = cls.__name__ @@ -51,6 +56,11 @@ def _custom_marking_builder(cls, type, properties, version, base_class): def __init__(self, **kwargs): base_class.__init__(self, **kwargs) + ext = getattr(self, 'with_extension', None) + if ext and version != '2.0': + if 'extensions' not in self._inner: + self._inner['extensions'] = {} + self._inner['extensions'][ext] = _get_extension_class(ext, version)() _cls_init(cls, self, kwargs) _CustomMarking.__name__ = cls.__name__ @@ -75,6 +85,11 @@ def _custom_observable_builder(cls, type, properties, version, base_class, id_co def __init__(self, **kwargs): base_class.__init__(self, **kwargs) _cls_init(cls, self, kwargs) + ext = getattr(self, 'with_extension', None) + if ext and version != '2.0': + if 'extensions' not in self._inner: + self._inner['extensions'] = {} + self._inner['extensions'][ext] = _get_extension_class(ext, version)() _CustomObservable.__name__ = cls.__name__ @@ -82,7 +97,7 @@ def _custom_observable_builder(cls, type, properties, version, base_class, id_co return _CustomObservable -def _custom_extension_builder(cls, observable, type, properties, version, base_class): +def _custom_extension_builder(cls, type, properties, version, base_class): prop_dict = _get_properties_dict(properties) class _CustomExtension(cls, base_class): @@ -96,5 +111,5 @@ def _custom_extension_builder(cls, observable, type, properties, version, base_c _CustomExtension.__name__ = cls.__name__ - _register_observable_extension(observable, _CustomExtension, version=version) + _register_extension(_CustomExtension, version=version) return _CustomExtension diff --git a/stix2/parsing.py b/stix2/parsing.py index c0c7bf8..59bfd4a 100644 --- a/stix2/parsing.py +++ b/stix2/parsing.py @@ -7,7 +7,7 @@ import re import stix2 -from .base import _DomainObject, _Observable +from .base import _DomainObject from .exceptions import DuplicateRegistrationError, ParseError from .utils import PREFIX_21_REGEX, _get_dict, get_class_hierarchy_names @@ -90,6 +90,12 @@ def _detect_spec_version(stix_dict): return v +def _get_extension_class(extension_uuid, version): + """Retrieve a registered class Extension""" + v = 'v' + version.replace('.', '') + return STIX2_OBJ_MAPS[v]['extensions'].get(extension_uuid) + + def dict_to_stix2(stix_dict, allow_custom=False, version=None): """convert dictionary to full python-stix2 object @@ -137,6 +143,13 @@ def dict_to_stix2(stix_dict, allow_custom=False, version=None): # flag allows for unknown custom objects too, but will not # be parsed into STIX object, returned as is return stix_dict + for key_id, ext_def in stix_dict.get('extensions', {}).items(): + if key_id.startswith('stix-extension--') and ( + ext_def.get('is_new_object', False) or ext_def.get('is_extension_so', False) + ): + # prevents ParseError for unregistered objects when + # 'is_new_object' or 'is_extension_so' are set to True and allow_custom=False + return stix_dict raise ParseError("Can't parse unknown object type '%s'! For custom types, use the CustomObject decorator." % stix_dict['type']) return obj_class(allow_custom=allow_custom, **stix_dict) @@ -318,27 +331,20 @@ def _register_observable(new_observable, version=stix2.DEFAULT_VERSION): OBJ_MAP_OBSERVABLE[new_observable._type] = new_observable -def _register_observable_extension( - observable, new_extension, version=stix2.DEFAULT_VERSION, +def _register_extension( + new_extension, version=stix2.DEFAULT_VERSION, ): - """Register a custom extension to a STIX Cyber Observable type. + """Register a custom extension to any STIX Object type. Args: - observable: An observable class or instance - new_extension (class): A class to register in the Observables - Extensions map. + new_extension (class): A class to register in the Extensions map. version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). Defaults to the latest supported version. """ - obs_class = observable if isinstance(observable, type) else \ - type(observable) ext_type = new_extension._type properties = new_extension._properties - if not issubclass(obs_class, _Observable): - raise ValueError("'observable' must be a valid Observable class!") - stix2.properties._validate_type(ext_type, version) if not new_extension._properties: @@ -348,42 +354,22 @@ def _register_observable_extension( ) if version == "2.1": - if not ext_type.endswith('-ext'): + if not (ext_type.endswith('-ext') or ext_type.startswith('stix-extension--')): raise ValueError( - "Invalid extension type name '%s': must end with '-ext'." % + "Invalid extension type name '%s': must end with '-ext' or start with 'stix-extension--'." % ext_type, ) - for prop_name, prop_value in properties.items(): + for prop_name in properties.keys(): if not re.match(PREFIX_21_REGEX, prop_name): raise ValueError("Property name '%s' must begin with an alpha character." % prop_name) v = 'v' + version.replace('.', '') + EXT_MAP = STIX2_OBJ_MAPS[v]['extensions'] - try: - observable_type = observable._type - except AttributeError: - raise ValueError( - "Unknown observable type. Custom observables must be " - "created with the @CustomObservable decorator.", - ) - - OBJ_MAP_OBSERVABLE = STIX2_OBJ_MAPS[v]['observables'] - EXT_MAP = STIX2_OBJ_MAPS[v]['observable-extensions'] - - try: - if ext_type in EXT_MAP[observable_type].keys(): - raise DuplicateRegistrationError("Observable Extension", ext_type) - EXT_MAP[observable_type][ext_type] = new_extension - except KeyError: - if observable_type not in OBJ_MAP_OBSERVABLE: - raise ValueError( - "Unknown observable type '%s'. Custom observables " - "must be created with the @CustomObservable decorator." - % observable_type, - ) - else: - EXT_MAP[observable_type] = {ext_type: new_extension} + if ext_type in EXT_MAP: + raise DuplicateRegistrationError("Extension", ext_type) + EXT_MAP[ext_type] = new_extension def _collect_stix2_mappings(): @@ -401,7 +387,7 @@ def _collect_stix2_mappings(): STIX2_OBJ_MAPS[ver] = {} STIX2_OBJ_MAPS[ver]['objects'] = mod.OBJ_MAP STIX2_OBJ_MAPS[ver]['observables'] = mod.OBJ_MAP_OBSERVABLE - STIX2_OBJ_MAPS[ver]['observable-extensions'] = mod.EXT_MAP + STIX2_OBJ_MAPS[ver]['extensions'] = mod.EXT_MAP elif re.match(r'^stix2\.v2[0-9]\.common$', name) and is_pkg is False: mod = importlib.import_module(name, str(top_level_module.__name__)) STIX2_OBJ_MAPS[ver]['markings'] = mod.OBJ_MAP_MARKING diff --git a/stix2/v20/base.py b/stix2/v20/base.py index b5437ca..a0d65c6 100644 --- a/stix2/v20/base.py +++ b/stix2/v20/base.py @@ -10,7 +10,10 @@ class _STIXBase20(_STIXBase): class _Observable(_Observable, _STIXBase20): - pass + + def __init__(self, **kwargs): + self._properties['extensions'].allow_custom = kwargs.get('allow_custom', False) + super(_Observable, self).__init__(**kwargs) class _Extension(_Extension, _STIXBase20): diff --git a/stix2/v20/observables.py b/stix2/v20/observables.py index 8dd7ff8..cf06579 100644 --- a/stix2/v20/observables.py +++ b/stix2/v20/observables.py @@ -793,15 +793,15 @@ def CustomObservable(type='x-custom-observable', properties=None): _properties = list(itertools.chain.from_iterable([ [('type', TypeProperty(type, spec_version='2.0'))], properties, - [('extensions', ExtensionsProperty(spec_version="2.0", enclosing_type=type))], + [('extensions', ExtensionsProperty(spec_version='2.0'))], ])) return _custom_observable_builder(cls, type, _properties, '2.0', _Observable) return wrapper -def CustomExtension(observable=None, type='x-custom-observable-ext', properties=None): +def CustomExtension(type='x-custom-observable-ext', properties=None): """Decorator for custom extensions to STIX Cyber Observables. """ def wrapper(cls): - return _custom_extension_builder(cls, observable, type, properties, '2.0', _Extension) + return _custom_extension_builder(cls, type, properties, '2.0', _Extension) return wrapper diff --git a/stix2/v21/base.py b/stix2/v21/base.py index 830a209..0ba7fe3 100644 --- a/stix2/v21/base.py +++ b/stix2/v21/base.py @@ -14,11 +14,43 @@ class _STIXBase21(_STIXBase): class _Observable(_Observable, _STIXBase21): - pass + + def __init__(self, **kwargs): + super(_Observable, self).__init__(**kwargs) + if 'id' not in kwargs: + # Specific to 2.1+ observables: generate a deterministic ID + id_ = self._generate_id() + + # Spec says fall back to UUIDv4 if no contributing properties were + # given. That's what already happened (the following is actually + # overwriting the default uuidv4), so nothing to do here. + if id_ is not None: + # Can't assign to self (we're immutable), so slip the ID in + # more sneakily. + self._inner["id"] = id_ class _Extension(_Extension, _STIXBase21): - pass + extends_stix_object_definition = False + is_new_object = False + is_extension_so = False + + def __init__(self, **kwargs): + super(_Extension, self).__init__(**kwargs) + if getattr(self, "extends_stix_object_definition", False): + self._inner["extends_stix_object_definition"] = True + elif getattr(self, "is_new_object", False): + self._inner["is_new_object"] = True + elif getattr(self, "is_extension_so", False): + self._inner["is_extension_so"] = True + + def _check_at_least_one_property(self, list_of_properties=None): + new_ext_check = (getattr(self, "extends_stix_object_definition", False) or + getattr(self, "is_new_object", False) or + getattr(self, "is_extension_so", False)) + + if new_ext_check is False: + super(_Extension, self)._check_at_least_one_property(list_of_properties=list_of_properties) class _DomainObject(_DomainObject, _STIXBase21): diff --git a/stix2/v21/observables.py b/stix2/v21/observables.py index 9af0dab..8c148b0 100644 --- a/stix2/v21/observables.py +++ b/stix2/v21/observables.py @@ -920,7 +920,7 @@ class X509Certificate(_Observable): self._check_at_least_one_property(att_list) -def CustomObservable(type='x-custom-observable', properties=None, id_contrib_props=None): +def CustomObservable(type='x-custom-observable', properties=None, id_contrib_props=None, extension_name=None): """Custom STIX Cyber Observable Object type decorator. Example: @@ -939,15 +939,24 @@ def CustomObservable(type='x-custom-observable', properties=None, id_contrib_pro [('type', TypeProperty(type, spec_version='2.1'))], [('id', IDProperty(type, spec_version='2.1'))], properties, - [('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=type))], + [('extensions', ExtensionsProperty(spec_version='2.1'))], ])) + if extension_name: + @CustomExtension(type=extension_name, properties=properties) + class NameExtension: + is_extension_so = True + + extension = extension_name.split('--')[1] + extension = extension.replace('-', '') + NameExtension.__name__ = 'STIXExtension' + extension + cls.with_extension = extension_name return _custom_observable_builder(cls, type, _properties, '2.1', _Observable, id_contrib_props) return wrapper -def CustomExtension(observable=None, type='x-custom-observable-ext', properties=None): - """Decorator for custom extensions to STIX Cyber Observables. +def CustomExtension(type='x-custom-observable-ext', properties=None): + """Custom STIX Object Extension decorator. """ def wrapper(cls): - return _custom_extension_builder(cls, observable, type, properties, '2.1', _Extension) + return _custom_extension_builder(cls, type, properties, '2.1', _Extension) return wrapper diff --git a/stix2/v21/sdo.py b/stix2/v21/sdo.py index ba479b7..f3d6bcd 100644 --- a/stix2/v21/sdo.py +++ b/stix2/v21/sdo.py @@ -7,6 +7,7 @@ import warnings from six.moves.urllib.parse import quote_plus from stix2patterns.validator import run_validator +from . import observables from ..custom import _custom_object_builder from ..exceptions import ( InvalidValueError, PropertyPresenceError, STIXDeprecationWarning, @@ -778,7 +779,7 @@ class Vulnerability(_DomainObject): ]) -def CustomObject(type='x-custom-type', properties=None): +def CustomObject(type='x-custom-type', properties=None, extension_name=None): """Custom STIX Object type decorator. Example: @@ -808,6 +809,7 @@ def CustomObject(type='x-custom-type', properties=None): """ def wrapper(cls): + extension_properties = [x for x in properties if not x[0].startswith('x_')] _properties = list(itertools.chain.from_iterable([ [ ('type', TypeProperty(type, spec_version='2.1')), @@ -817,7 +819,7 @@ def CustomObject(type='x-custom-type', properties=None): ('created', TimestampProperty(default=lambda: NOW, precision='millisecond', precision_constraint='min')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond', precision_constraint='min')), ], - [x for x in properties if not x[0].startswith('x_')], + extension_properties, [ ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), @@ -826,10 +828,19 @@ def CustomObject(type='x-custom-type', properties=None): ('external_references', ListProperty(ExternalReference)), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), - ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=type)), + ('extensions', ExtensionsProperty(spec_version='2.1')), ], sorted([x for x in properties if x[0].startswith('x_')], key=lambda x: x[0]), ])) + if extension_name: + @observables.CustomExtension(type=extension_name, properties=extension_properties) + class NameExtension: + is_new_object = True + + extension = extension_name.split('--')[1] + extension = extension.replace('-', '') + NameExtension.__name__ = 'STIXExtension' + extension + cls.with_extension = extension_name return _custom_object_builder(cls, type, _properties, '2.1', _DomainObject) return wrapper From aa77f2b174ba0c1073bdab05d41cfaa634a0c613 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Tue, 10 Nov 2020 21:09:17 -0500 Subject: [PATCH 09/56] add test cases additional file changes based on test cases --- stix2/custom.py | 2 +- stix2/markings/utils.py | 2 +- stix2/test/v21/test_custom.py | 227 ++++++++++++++++++++++++++++++++++ stix2/v21/common.py | 14 ++- stix2/v21/observables.py | 1 + 5 files changed, 243 insertions(+), 3 deletions(-) diff --git a/stix2/custom.py b/stix2/custom.py index 0a646fc..1ba63b6 100644 --- a/stix2/custom.py +++ b/stix2/custom.py @@ -56,12 +56,12 @@ def _custom_marking_builder(cls, type, properties, version, base_class): def __init__(self, **kwargs): base_class.__init__(self, **kwargs) + _cls_init(cls, self, kwargs) ext = getattr(self, 'with_extension', None) if ext and version != '2.0': if 'extensions' not in self._inner: self._inner['extensions'] = {} self._inner['extensions'][ext] = _get_extension_class(ext, version)() - _cls_init(cls, self, kwargs) _CustomMarking.__name__ = cls.__name__ diff --git a/stix2/markings/utils.py b/stix2/markings/utils.py index 41516cc..5650717 100644 --- a/stix2/markings/utils.py +++ b/stix2/markings/utils.py @@ -259,7 +259,7 @@ def iterpath(obj, path=None): def check_tlp_marking(marking_obj, spec_version): # Specific TLP Marking validation case. - if marking_obj["definition_type"] == "tlp": + if marking_obj.get("definition_type", "") == "tlp": color = marking_obj["definition"]["tlp"] if color == "white": diff --git a/stix2/test/v21/test_custom.py b/stix2/test/v21/test_custom.py index 67990d2..aecbb0f 100644 --- a/stix2/test/v21/test_custom.py +++ b/stix2/test/v21/test_custom.py @@ -1296,3 +1296,230 @@ def test_register_duplicate_observable_extension(): class NewExtension2(): pass assert "cannot be registered again" in str(excinfo.value) + + +def test_unregistered_top_level_extension_passes_with_allow_custom_false(): + indicator = stix2.v21.Indicator( + id='indicator--e97bfccf-8970-4a3c-9cd1-5b5b97ed5d0c', + created='2014-02-20T09:16:08.989000Z', + modified='2014-02-20T09:16:08.989000Z', + name='File hash for Poison Ivy variant', + description='This file hash indicates that a sample of Poison Ivy is present.', + labels=[ + 'malicious-activity', + ], + rank=5, + toxicity=8, + pattern='[file:hashes.\'SHA-256\' = \'ef537f25c895bfa782526529a9b63d97aa631564d5d789c2b765448c8635fb6c\']', + pattern_type='stix', + valid_from='2014-02-20T09:00:00.000000Z', + extensions={ + 'stix-extension--d83fce45-ef58-4c6c-a3f4-1fbc32e98c6e': { + 'extends_stix_object_definition': True, + }, + }, + allow_custom=False, + ) + assert indicator.rank == 5 + assert indicator.toxicity == 8 + assert isinstance(indicator.extensions['stix-extension--d83fce45-ef58-4c6c-a3f4-1fbc32e98c6e'], dict) + + +def test_unregistered_embedded_extension_passes_with_allow_custom_false(): + indicator = stix2.v21.Indicator( + id='indicator--e97bfccf-8970-4a3c-9cd1-5b5b97ed5d0c', + created='2014-02-20T09:16:08.989000Z', + modified='2014-02-20T09:16:08.989000Z', + name='File hash for Poison Ivy variant', + description='This file hash indicates that a sample of Poison Ivy is present.', + labels=[ + 'malicious-activity', + ], + pattern='[file:hashes.\'SHA-256\' = \'ef537f25c895bfa782526529a9b63d97aa631564d5d789c2b765448c8635fb6c\']', + pattern_type='stix', + valid_from='2014-02-20T09:00:00.000000Z', + extensions={ + 'stix-extension--d83fce45-ef58-4c6c-a3f4-1fbc32e98c6e': { + 'rank': 5, + 'toxicity': 8, + }, + }, + allow_custom=False, + ) + assert indicator.extensions['stix-extension--d83fce45-ef58-4c6c-a3f4-1fbc32e98c6e']['rank'] == 5 + assert indicator.extensions['stix-extension--d83fce45-ef58-4c6c-a3f4-1fbc32e98c6e']['toxicity'] == 8 + assert isinstance(indicator.extensions['stix-extension--d83fce45-ef58-4c6c-a3f4-1fbc32e98c6e'], dict) + + +def test_registered_top_level_extension_passes_with_allow_custom_false(): + @stix2.v21.CustomExtension( + 'stix-extension--d83fce45-ef58-4c6c-a3f4-1fbc32e98c6e', [ + ('rank', stix2.properties.IntegerProperty(required=True)), + ('toxicity', stix2.properties.IntegerProperty(required=True)), + ], + ) + class ExtensionFoo1: + extends_stix_object_definition = True + + indicator = stix2.v21.Indicator( + id='indicator--e97bfccf-8970-4a3c-9cd1-5b5b97ed5d0c', + created='2014-02-20T09:16:08.989000Z', + modified='2014-02-20T09:16:08.989000Z', + name='File hash for Poison Ivy variant', + description='This file hash indicates that a sample of Poison Ivy is present.', + labels=[ + 'malicious-activity', + ], + rank=5, + toxicity=8, + pattern='[file:hashes.\'SHA-256\' = \'ef537f25c895bfa782526529a9b63d97aa631564d5d789c2b765448c8635fb6c\']', + pattern_type='stix', + valid_from='2014-02-20T09:00:00.000000Z', + extensions={ + 'stix-extension--d83fce45-ef58-4c6c-a3f4-1fbc32e98c6e': { + 'extends_stix_object_definition': True, + }, + }, + allow_custom=False, + ) + assert indicator.rank == 5 + assert indicator.toxicity == 8 + assert isinstance(indicator.extensions['stix-extension--d83fce45-ef58-4c6c-a3f4-1fbc32e98c6e'], ExtensionFoo1) + + +def test_registered_embedded_extension_passes_with_allow_custom_false(): + @stix2.v21.CustomExtension( + 'stix-extension--d83fce45-ef58-4c6c-a3ff-1fbc32e98c6e', [ + ('rank', stix2.properties.IntegerProperty(required=True)), + ('toxicity', stix2.properties.IntegerProperty(required=True)), + ], + ) + class ExtensionFoo1: + pass + + indicator = stix2.v21.Indicator( + id='indicator--e97bfccf-8970-4a3c-9cd1-5b5b97ed5d0c', + created='2014-02-20T09:16:08.989000Z', + modified='2014-02-20T09:16:08.989000Z', + name='File hash for Poison Ivy variant', + description='This file hash indicates that a sample of Poison Ivy is present.', + labels=[ + 'malicious-activity', + ], + pattern='[file:hashes.\'SHA-256\' = \'ef537f25c895bfa782526529a9b63d97aa631564d5d789c2b765448c8635fb6c\']', + pattern_type='stix', + valid_from='2014-02-20T09:00:00.000000Z', + extensions={ + 'stix-extension--d83fce45-ef58-4c6c-a3ff-1fbc32e98c6e': { + 'rank': 5, + 'toxicity': 8, + }, + }, + allow_custom=False, + ) + assert indicator.extensions['stix-extension--d83fce45-ef58-4c6c-a3ff-1fbc32e98c6e']['rank'] == 5 + assert indicator.extensions['stix-extension--d83fce45-ef58-4c6c-a3ff-1fbc32e98c6e']['toxicity'] == 8 + assert isinstance(indicator.extensions['stix-extension--d83fce45-ef58-4c6c-a3ff-1fbc32e98c6e'], ExtensionFoo1) + + +def test_registered_new_extension_sdo_allow_custom_false(): + @stix2.v21.CustomObject( + 'my-favorite-sdo', [ + ('name', stix2.properties.StringProperty(required=True)), + ('some_property_name1', stix2.properties.StringProperty(required=True)), + ('some_property_name2', stix2.properties.StringProperty()), + ], 'stix-extension--d83fce45-ef58-4c6c-a3f4-1fbc32e9999', + ) + class MyFavSDO: + pass + + my_favorite_sdo = { + 'type': 'my-favorite-sdo', + 'spec_version': '2.1', + 'id': 'my-favorite-sdo--c5ba9dba-5ad9-4bbe-9825-df4cb8675774', + 'created': '2014-02-20T09:16:08.989000Z', + 'modified': '2014-02-20T09:16:08.989000Z', + 'name': 'This is the name of my favorite', + 'some_property_name1': 'value1', + 'some_property_name2': 'value2', + # 'extensions': { + # 'stix-extension--d83fce45-ef58-4c6c-a3f4-1fbc32e9999': STIXExtensiond83fce45ef584c6ca3f41fbc32e98c6e() + # } + } + sdo_object = stix2.parse(my_favorite_sdo) + assert isinstance(sdo_object, MyFavSDO) + assert isinstance( + sdo_object.extensions['stix-extension--d83fce45-ef58-4c6c-a3f4-1fbc32e9999'], + stix2.v21.EXT_MAP['stix-extension--d83fce45-ef58-4c6c-a3f4-1fbc32e9999'], + ) + + sdo_serialized = sdo_object.serialize() + assert '"extensions": {"stix-extension--d83fce45-ef58-4c6c-a3f4-1fbc32e9999": {"is_new_object": true}}' in sdo_serialized + + +def test_registered_new_extension_sco_allow_custom_false(): + @stix2.v21.CustomObservable( + 'my-favorite-sco', [ + ('name', stix2.properties.StringProperty(required=True)), + ('some_network_protocol_field', stix2.properties.StringProperty(required=True)), + ], ['name', 'some_network_protocol_field'], 'stix-extension--a932fcc6-e032-177c-126f-cb970a5a1fff', + ) + class MyFavSCO: + pass + + my_favorite_sco = { + 'type': 'my-favorite-sco', + 'spec_version': '2.1', + 'id': 'my-favorite-sco--f9dbe89c-0030-4a9d-8b78-0dcd0a0de874', + 'name': 'This is the name of my favorite SCO', + 'some_network_protocol_field': 'value', + # 'extensions': { + # 'stix-extension--a932fcc6-e032-177c-126f-cb970a5a1fff': { + # 'is_extension_so': true + # } + # } + } + + sco_object = stix2.parse(my_favorite_sco) + assert isinstance(sco_object, MyFavSCO) + assert isinstance( + sco_object.extensions['stix-extension--a932fcc6-e032-177c-126f-cb970a5a1fff'], + stix2.v21.EXT_MAP['stix-extension--a932fcc6-e032-177c-126f-cb970a5a1fff'], + ) + + sco_serialized = sco_object.serialize() + assert '"extensions": {"stix-extension--a932fcc6-e032-177c-126f-cb970a5a1fff": {"is_extension_so": true}}' in sco_serialized + + +def test_registered_new_extension_marking_allow_custom_false(): + @stix2.v21.CustomMarking( + 'my-favorite-marking', [ + ('name', stix2.properties.StringProperty(required=True)), + ('some_marking_field', stix2.properties.StringProperty(required=True)), + ], 'stix-extension--a932fcc6-e032-176c-126f-cb970a5a1fff', + ) + class MyFavMarking: + pass + + my_favorite_marking = { + 'type': 'marking-definition', + 'spec_version': '2.1', + 'id': 'marking-definition--f9dbe89c-0030-4a9d-8b78-0dcd0a0de874', + 'name': 'This is the name of my favorite Marking', + 'some_marking_field': 'value', + 'extensions': { + 'stix-extension--a932fcc6-e032-176c-126f-cb970a5a1fff': { + 'extends_stix_object_definition': True, + }, + }, + } + + marking_object = stix2.parse(my_favorite_marking) + assert isinstance(marking_object, stix2.v21.MarkingDefinition) + assert isinstance( + marking_object.extensions['stix-extension--a932fcc6-e032-176c-126f-cb970a5a1fff'], + stix2.v21.EXT_MAP['stix-extension--a932fcc6-e032-176c-126f-cb970a5a1fff'], + ) + + marking_serialized = marking_object.serialize() + assert '"extensions": {"stix-extension--a932fcc6-e032-176c-126f-cb970a5a1fff": {"extends_stix_object_definition": true}}' in marking_serialized diff --git a/stix2/v21/common.py b/stix2/v21/common.py index b179103..55bd8e1 100644 --- a/stix2/v21/common.py +++ b/stix2/v21/common.py @@ -235,7 +235,7 @@ OBJ_MAP_MARKING = { } -def CustomMarking(type='x-custom-marking', properties=None): +def CustomMarking(type='x-custom-marking', properties=None, extension_name=None): """Custom STIX Marking decorator. Example: @@ -250,6 +250,18 @@ def CustomMarking(type='x-custom-marking', properties=None): """ def wrapper(cls): + if extension_name: + from . import observables + + @observables.CustomExtension(type=extension_name, properties=properties) + class NameExtension: + # might not be correct + extends_stix_object_definition = True + + extension = extension_name.split('--')[1] + extension = extension.replace('-', '') + NameExtension.__name__ = 'STIXExtension' + extension + cls.with_extension = extension_name return _custom_marking_builder(cls, type, properties, '2.1', _STIXBase21) return wrapper diff --git a/stix2/v21/observables.py b/stix2/v21/observables.py index 8c148b0..7aa5c70 100644 --- a/stix2/v21/observables.py +++ b/stix2/v21/observables.py @@ -938,6 +938,7 @@ def CustomObservable(type='x-custom-observable', properties=None, id_contrib_pro _properties = list(itertools.chain.from_iterable([ [('type', TypeProperty(type, spec_version='2.1'))], [('id', IDProperty(type, spec_version='2.1'))], + [('spec_version', StringProperty(fixed='2.1'))], properties, [('extensions', ExtensionsProperty(spec_version='2.1'))], ])) From 33a472d8f5e5d837152ebfa9a3830e45db034c33 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Tue, 22 Dec 2020 17:46:29 -0500 Subject: [PATCH 10/56] define Incident stub --- stix2/v21/__init__.py | 5 +++-- stix2/v21/sdo.py | 27 +++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/stix2/v21/__init__.py b/stix2/v21/__init__.py index 81ab980..be7e343 100644 --- a/stix2/v21/__init__.py +++ b/stix2/v21/__init__.py @@ -36,7 +36,7 @@ from .observables import ( ) from .sdo import ( AttackPattern, Campaign, CourseOfAction, CustomObject, Grouping, Identity, - Indicator, Infrastructure, IntrusionSet, Location, Malware, + Incident, Indicator, Infrastructure, IntrusionSet, Location, Malware, MalwareAnalysis, Note, ObservedData, Opinion, Report, ThreatActor, Tool, Vulnerability, ) @@ -49,6 +49,7 @@ OBJ_MAP = { 'course-of-action': CourseOfAction, 'grouping': Grouping, 'identity': Identity, + 'incident': Incident, 'indicator': Indicator, 'infrastructure': Infrastructure, 'intrusion-set': IntrusionSet, @@ -126,7 +127,7 @@ __all__ = """ X509Certificate, X509V3ExtensionsType, AttackPattern, Campaign, CourseOfAction, CustomObject, Grouping, Identity, - Indicator, Infrastructure, IntrusionSet, Location, Malware, + Incident, Indicator, Infrastructure, IntrusionSet, Location, Malware, MalwareAnalysis, Note, ObservedData, Opinion, Report, ThreatActor, Tool, Vulnerability, diff --git a/stix2/v21/sdo.py b/stix2/v21/sdo.py index f3d6bcd..8f811b3 100644 --- a/stix2/v21/sdo.py +++ b/stix2/v21/sdo.py @@ -175,6 +175,33 @@ class Identity(_DomainObject): ]) +class Incident(_DomainObject): + """For more detailed information on this object's properties, see + `the STIX 2.1 specification `__. + """ + + _type = 'incident' + _properties = OrderedDict([ + ('type', TypeProperty(_type, spec_version='2.1')), + ('spec_version', StringProperty(fixed='2.1')), + ('id', IDProperty(_type, spec_version='2.1')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), + ('created', TimestampProperty(default=lambda: NOW, precision='millisecond', precision_constraint='min')), + ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond', precision_constraint='min')), + ('name', StringProperty(required=True)), + ('description', StringProperty()), + ('kill_chain_phases', ListProperty(KillChainPhase)), + ('revoked', BooleanProperty(default=lambda: False)), + ('labels', ListProperty(StringProperty)), + ('confidence', IntegerProperty()), + ('lang', StringProperty()), + ('external_references', ListProperty(ExternalReference)), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), + ('granular_markings', ListProperty(GranularMarking)), + ('extensions', ExtensionsProperty(spec_version='2.1')), + ]) + + class Indicator(_DomainObject): """For more detailed information on this object's properties, see `the STIX 2.1 specification `__. From 4127ea9afd91af1ca30599e4c2da8a120749cc7a Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Tue, 22 Dec 2020 21:41:46 -0500 Subject: [PATCH 11/56] modify main codebase based on 7.3 Extension changes --- stix2/base.py | 6 ++---- stix2/parsing.py | 4 +--- stix2/v21/base.py | 18 +++++------------- stix2/v21/common.py | 3 +-- stix2/v21/observables.py | 2 +- stix2/v21/sdo.py | 2 +- 6 files changed, 11 insertions(+), 24 deletions(-) diff --git a/stix2/base.py b/stix2/base.py index 4a21cf0..1ff4608 100644 --- a/stix2/base.py +++ b/stix2/base.py @@ -128,7 +128,7 @@ class _STIXBase(Mapping): extra_kwargs = list(set(kwargs) - set(self._properties)) if extra_kwargs and issubclass(cls, stix2.v21._Extension): - props_to_remove = ['extends_stix_object_definition', 'is_new_object', 'is_extension_so'] + props_to_remove = ['extension_type'] extra_kwargs = [prop for prop in extra_kwargs if prop not in props_to_remove] if extra_kwargs and not self._allow_custom: @@ -166,9 +166,7 @@ class _STIXBase(Mapping): missing_kwargs = required_properties - set(setting_kwargs) if missing_kwargs: new_ext_check = ( - getattr(self, 'extends_stix_object_definition', False) or - getattr(self, 'is_new_object', False) or - getattr(self, 'is_extension_so', False) + bool(getattr(self, "extension_type", None)) ) and issubclass(cls, stix2.v21._Extension) if new_ext_check is False: raise MissingPropertiesError(cls, missing_kwargs) diff --git a/stix2/parsing.py b/stix2/parsing.py index 59bfd4a..ad66665 100644 --- a/stix2/parsing.py +++ b/stix2/parsing.py @@ -144,9 +144,7 @@ def dict_to_stix2(stix_dict, allow_custom=False, version=None): # be parsed into STIX object, returned as is return stix_dict for key_id, ext_def in stix_dict.get('extensions', {}).items(): - if key_id.startswith('stix-extension--') and ( - ext_def.get('is_new_object', False) or ext_def.get('is_extension_so', False) - ): + if key_id.startswith('stix-extension--') and ext_def.get('extension_type', None): # prevents ParseError for unregistered objects when # 'is_new_object' or 'is_extension_so' are set to True and allow_custom=False return stix_dict diff --git a/stix2/v21/base.py b/stix2/v21/base.py index 0ba7fe3..8598a83 100644 --- a/stix2/v21/base.py +++ b/stix2/v21/base.py @@ -31,25 +31,17 @@ class _Observable(_Observable, _STIXBase21): class _Extension(_Extension, _STIXBase21): - extends_stix_object_definition = False - is_new_object = False - is_extension_so = False + extension_type = None def __init__(self, **kwargs): super(_Extension, self).__init__(**kwargs) - if getattr(self, "extends_stix_object_definition", False): - self._inner["extends_stix_object_definition"] = True - elif getattr(self, "is_new_object", False): - self._inner["is_new_object"] = True - elif getattr(self, "is_extension_so", False): - self._inner["is_extension_so"] = True + if getattr(self, "extension_type", None): + self._inner["extension_type"] = self.extension_type def _check_at_least_one_property(self, list_of_properties=None): - new_ext_check = (getattr(self, "extends_stix_object_definition", False) or - getattr(self, "is_new_object", False) or - getattr(self, "is_extension_so", False)) + new_ext_check = getattr(self, "extension_type", None) - if new_ext_check is False: + if new_ext_check is None: super(_Extension, self)._check_at_least_one_property(list_of_properties=list_of_properties) diff --git a/stix2/v21/common.py b/stix2/v21/common.py index 55bd8e1..06d9c43 100644 --- a/stix2/v21/common.py +++ b/stix2/v21/common.py @@ -255,8 +255,7 @@ def CustomMarking(type='x-custom-marking', properties=None, extension_name=None) @observables.CustomExtension(type=extension_name, properties=properties) class NameExtension: - # might not be correct - extends_stix_object_definition = True + extension_type = 'property-extension' extension = extension_name.split('--')[1] extension = extension.replace('-', '') diff --git a/stix2/v21/observables.py b/stix2/v21/observables.py index 7aa5c70..cf23cfd 100644 --- a/stix2/v21/observables.py +++ b/stix2/v21/observables.py @@ -945,7 +945,7 @@ def CustomObservable(type='x-custom-observable', properties=None, id_contrib_pro if extension_name: @CustomExtension(type=extension_name, properties=properties) class NameExtension: - is_extension_so = True + extension_type = 'new-sco' extension = extension_name.split('--')[1] extension = extension.replace('-', '') diff --git a/stix2/v21/sdo.py b/stix2/v21/sdo.py index 8f811b3..af22fc5 100644 --- a/stix2/v21/sdo.py +++ b/stix2/v21/sdo.py @@ -862,7 +862,7 @@ def CustomObject(type='x-custom-type', properties=None, extension_name=None): if extension_name: @observables.CustomExtension(type=extension_name, properties=extension_properties) class NameExtension: - is_new_object = True + extension_type = 'new-sdo' extension = extension_name.split('--')[1] extension = extension.replace('-', '') From ef2ef95c3a79fdc296ba3eacd8da6321e1778c74 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Tue, 22 Dec 2020 21:59:18 -0500 Subject: [PATCH 12/56] `stix-extension` to `extension-definition` --- stix2/base.py | 2 +- stix2/parsing.py | 6 +-- stix2/properties.py | 4 +- stix2/test/v21/test_custom.py | 86 +++++++++++++++++++---------------- stix2/v21/__init__.py | 14 +++--- stix2/v21/common.py | 4 +- 6 files changed, 61 insertions(+), 55 deletions(-) diff --git a/stix2/base.py b/stix2/base.py index 1ff4608..bd59fc5 100644 --- a/stix2/base.py +++ b/stix2/base.py @@ -134,7 +134,7 @@ class _STIXBase(Mapping): if extra_kwargs and not self._allow_custom: ext_found = False for key_id, ext_def in kwargs.get('extensions', {}).items(): - if key_id.startswith('stix-extension--'): + if key_id.startswith('extension-definition--'): ext_found = True break if ext_found is False: diff --git a/stix2/parsing.py b/stix2/parsing.py index ad66665..77a04d9 100644 --- a/stix2/parsing.py +++ b/stix2/parsing.py @@ -144,7 +144,7 @@ def dict_to_stix2(stix_dict, allow_custom=False, version=None): # be parsed into STIX object, returned as is return stix_dict for key_id, ext_def in stix_dict.get('extensions', {}).items(): - if key_id.startswith('stix-extension--') and ext_def.get('extension_type', None): + if key_id.startswith('extension-definition--') and ext_def.get('extension_type', None): # prevents ParseError for unregistered objects when # 'is_new_object' or 'is_extension_so' are set to True and allow_custom=False return stix_dict @@ -352,9 +352,9 @@ def _register_extension( ) if version == "2.1": - if not (ext_type.endswith('-ext') or ext_type.startswith('stix-extension--')): + if not (ext_type.endswith('-ext') or ext_type.startswith('extension-definition--')): raise ValueError( - "Invalid extension type name '%s': must end with '-ext' or start with 'stix-extension--'." % + "Invalid extension type name '%s': must end with '-ext' or start with 'extension-definition--'." % ext_type, ) diff --git a/stix2/properties.py b/stix2/properties.py index cea664f..732a7dd 100644 --- a/stix2/properties.py +++ b/stix2/properties.py @@ -672,8 +672,8 @@ class ExtensionsProperty(DictionaryProperty): else: if self.allow_custom: dictified[key] = subvalue - elif key.startswith('stix-extension--'): - _validate_id(key, '2.1', 'stix-extension') + elif key.startswith('extension-definition--'): + _validate_id(key, '2.1', 'extension-definition') dictified[key] = subvalue else: raise CustomContentError("Can't parse unknown extension type: {}".format(key)) diff --git a/stix2/test/v21/test_custom.py b/stix2/test/v21/test_custom.py index aecbb0f..a2e41b2 100644 --- a/stix2/test/v21/test_custom.py +++ b/stix2/test/v21/test_custom.py @@ -1314,15 +1314,16 @@ def test_unregistered_top_level_extension_passes_with_allow_custom_false(): pattern_type='stix', valid_from='2014-02-20T09:00:00.000000Z', extensions={ - 'stix-extension--d83fce45-ef58-4c6c-a3f4-1fbc32e98c6e': { - 'extends_stix_object_definition': True, + 'extension-definition--d83fce45-ef58-4c6c-a3f4-1fbc32e98c6e': { + 'extension_type': 'toplevel-property-extension', }, }, allow_custom=False, ) assert indicator.rank == 5 assert indicator.toxicity == 8 - assert isinstance(indicator.extensions['stix-extension--d83fce45-ef58-4c6c-a3f4-1fbc32e98c6e'], dict) + assert indicator.extensions['extension-definition--d83fce45-ef58-4c6c-a3f4-1fbc32e98c6e']['extension_type'] == 'toplevel-property-extension' + assert isinstance(indicator.extensions['extension-definition--d83fce45-ef58-4c6c-a3f4-1fbc32e98c6e'], dict) def test_unregistered_embedded_extension_passes_with_allow_custom_false(): @@ -1339,27 +1340,29 @@ def test_unregistered_embedded_extension_passes_with_allow_custom_false(): pattern_type='stix', valid_from='2014-02-20T09:00:00.000000Z', extensions={ - 'stix-extension--d83fce45-ef58-4c6c-a3f4-1fbc32e98c6e': { + 'extension-definition--d83fce45-ef58-4c6c-a3f4-1fbc32e98c6e': { + 'extension_type': 'property-extension', 'rank': 5, 'toxicity': 8, }, }, allow_custom=False, ) - assert indicator.extensions['stix-extension--d83fce45-ef58-4c6c-a3f4-1fbc32e98c6e']['rank'] == 5 - assert indicator.extensions['stix-extension--d83fce45-ef58-4c6c-a3f4-1fbc32e98c6e']['toxicity'] == 8 - assert isinstance(indicator.extensions['stix-extension--d83fce45-ef58-4c6c-a3f4-1fbc32e98c6e'], dict) + assert indicator.extensions['extension-definition--d83fce45-ef58-4c6c-a3f4-1fbc32e98c6e']['rank'] == 5 + assert indicator.extensions['extension-definition--d83fce45-ef58-4c6c-a3f4-1fbc32e98c6e']['toxicity'] == 8 + assert indicator.extensions['extension-definition--d83fce45-ef58-4c6c-a3f4-1fbc32e98c6e']['extension_type'] == 'property-extension' + assert isinstance(indicator.extensions['extension-definition--d83fce45-ef58-4c6c-a3f4-1fbc32e98c6e'], dict) def test_registered_top_level_extension_passes_with_allow_custom_false(): @stix2.v21.CustomExtension( - 'stix-extension--d83fce45-ef58-4c6c-a3f4-1fbc32e98c6e', [ - ('rank', stix2.properties.IntegerProperty(required=True)), - ('toxicity', stix2.properties.IntegerProperty(required=True)), + 'extension-definition--d83fce45-ef58-4c6c-a3f4-1fbc32e98c6e', [ + ('rank', stix2.properties.IntegerProperty(required=True)), + ('toxicity', stix2.properties.IntegerProperty(required=True)), ], ) class ExtensionFoo1: - extends_stix_object_definition = True + extension_type = 'toplevel-property-extension' indicator = stix2.v21.Indicator( id='indicator--e97bfccf-8970-4a3c-9cd1-5b5b97ed5d0c', @@ -1376,22 +1379,23 @@ def test_registered_top_level_extension_passes_with_allow_custom_false(): pattern_type='stix', valid_from='2014-02-20T09:00:00.000000Z', extensions={ - 'stix-extension--d83fce45-ef58-4c6c-a3f4-1fbc32e98c6e': { - 'extends_stix_object_definition': True, + 'extension-definition--d83fce45-ef58-4c6c-a3f4-1fbc32e98c6e': { + 'extension_type': 'toplevel-property-extension', }, }, allow_custom=False, ) assert indicator.rank == 5 assert indicator.toxicity == 8 - assert isinstance(indicator.extensions['stix-extension--d83fce45-ef58-4c6c-a3f4-1fbc32e98c6e'], ExtensionFoo1) + assert indicator.extensions['extension-definition--d83fce45-ef58-4c6c-a3f4-1fbc32e98c6e']['extension_type'] == 'toplevel-property-extension' + assert isinstance(indicator.extensions['extension-definition--d83fce45-ef58-4c6c-a3f4-1fbc32e98c6e'], ExtensionFoo1) def test_registered_embedded_extension_passes_with_allow_custom_false(): @stix2.v21.CustomExtension( - 'stix-extension--d83fce45-ef58-4c6c-a3ff-1fbc32e98c6e', [ - ('rank', stix2.properties.IntegerProperty(required=True)), - ('toxicity', stix2.properties.IntegerProperty(required=True)), + 'extension-definition--d83fce45-ef58-4c6c-a3ff-1fbc32e98c6e', [ + ('rank', stix2.properties.IntegerProperty(required=True)), + ('toxicity', stix2.properties.IntegerProperty(required=True)), ], ) class ExtensionFoo1: @@ -1410,16 +1414,18 @@ def test_registered_embedded_extension_passes_with_allow_custom_false(): pattern_type='stix', valid_from='2014-02-20T09:00:00.000000Z', extensions={ - 'stix-extension--d83fce45-ef58-4c6c-a3ff-1fbc32e98c6e': { + 'extension-definition--d83fce45-ef58-4c6c-a3ff-1fbc32e98c6e': { + 'extension_type': 'property-extension', 'rank': 5, 'toxicity': 8, }, }, allow_custom=False, ) - assert indicator.extensions['stix-extension--d83fce45-ef58-4c6c-a3ff-1fbc32e98c6e']['rank'] == 5 - assert indicator.extensions['stix-extension--d83fce45-ef58-4c6c-a3ff-1fbc32e98c6e']['toxicity'] == 8 - assert isinstance(indicator.extensions['stix-extension--d83fce45-ef58-4c6c-a3ff-1fbc32e98c6e'], ExtensionFoo1) + assert indicator.extensions['extension-definition--d83fce45-ef58-4c6c-a3ff-1fbc32e98c6e']['rank'] == 5 + assert indicator.extensions['extension-definition--d83fce45-ef58-4c6c-a3ff-1fbc32e98c6e']['toxicity'] == 8 + assert indicator.extensions['extension-definition--d83fce45-ef58-4c6c-a3ff-1fbc32e98c6e']['extension_type'] == 'property-extension' + assert isinstance(indicator.extensions['extension-definition--d83fce45-ef58-4c6c-a3ff-1fbc32e98c6e'], ExtensionFoo1) def test_registered_new_extension_sdo_allow_custom_false(): @@ -1428,7 +1434,7 @@ def test_registered_new_extension_sdo_allow_custom_false(): ('name', stix2.properties.StringProperty(required=True)), ('some_property_name1', stix2.properties.StringProperty(required=True)), ('some_property_name2', stix2.properties.StringProperty()), - ], 'stix-extension--d83fce45-ef58-4c6c-a3f4-1fbc32e9999', + ], 'extension-definition--d83fce45-ef58-4c6c-a3f4-1fbc32e9999', ) class MyFavSDO: pass @@ -1443,18 +1449,18 @@ def test_registered_new_extension_sdo_allow_custom_false(): 'some_property_name1': 'value1', 'some_property_name2': 'value2', # 'extensions': { - # 'stix-extension--d83fce45-ef58-4c6c-a3f4-1fbc32e9999': STIXExtensiond83fce45ef584c6ca3f41fbc32e98c6e() + # 'extension-definition--d83fce45-ef58-4c6c-a3f4-1fbc32e9999': STIXExtensiond83fce45ef584c6ca3f41fbc32e98c6e() # } } sdo_object = stix2.parse(my_favorite_sdo) assert isinstance(sdo_object, MyFavSDO) assert isinstance( - sdo_object.extensions['stix-extension--d83fce45-ef58-4c6c-a3f4-1fbc32e9999'], - stix2.v21.EXT_MAP['stix-extension--d83fce45-ef58-4c6c-a3f4-1fbc32e9999'], + sdo_object.extensions['extension-definition--d83fce45-ef58-4c6c-a3f4-1fbc32e9999'], + stix2.v21.EXT_MAP['extension-definition--d83fce45-ef58-4c6c-a3f4-1fbc32e9999'], ) sdo_serialized = sdo_object.serialize() - assert '"extensions": {"stix-extension--d83fce45-ef58-4c6c-a3f4-1fbc32e9999": {"is_new_object": true}}' in sdo_serialized + assert '"extensions": {"extension-definition--d83fce45-ef58-4c6c-a3f4-1fbc32e9999": {"extension_type": "new-sdo"}}' in sdo_serialized def test_registered_new_extension_sco_allow_custom_false(): @@ -1462,7 +1468,7 @@ def test_registered_new_extension_sco_allow_custom_false(): 'my-favorite-sco', [ ('name', stix2.properties.StringProperty(required=True)), ('some_network_protocol_field', stix2.properties.StringProperty(required=True)), - ], ['name', 'some_network_protocol_field'], 'stix-extension--a932fcc6-e032-177c-126f-cb970a5a1fff', + ], ['name', 'some_network_protocol_field'], 'extension-definition--a932fcc6-e032-177c-126f-cb970a5a1fff', ) class MyFavSCO: pass @@ -1474,7 +1480,7 @@ def test_registered_new_extension_sco_allow_custom_false(): 'name': 'This is the name of my favorite SCO', 'some_network_protocol_field': 'value', # 'extensions': { - # 'stix-extension--a932fcc6-e032-177c-126f-cb970a5a1fff': { + # 'extension-definition--a932fcc6-e032-177c-126f-cb970a5a1fff': { # 'is_extension_so': true # } # } @@ -1483,20 +1489,19 @@ def test_registered_new_extension_sco_allow_custom_false(): sco_object = stix2.parse(my_favorite_sco) assert isinstance(sco_object, MyFavSCO) assert isinstance( - sco_object.extensions['stix-extension--a932fcc6-e032-177c-126f-cb970a5a1fff'], - stix2.v21.EXT_MAP['stix-extension--a932fcc6-e032-177c-126f-cb970a5a1fff'], + sco_object.extensions['extension-definition--a932fcc6-e032-177c-126f-cb970a5a1fff'], + stix2.v21.EXT_MAP['extension-definition--a932fcc6-e032-177c-126f-cb970a5a1fff'], ) sco_serialized = sco_object.serialize() - assert '"extensions": {"stix-extension--a932fcc6-e032-177c-126f-cb970a5a1fff": {"is_extension_so": true}}' in sco_serialized + assert '"extensions": {"extension-definition--a932fcc6-e032-177c-126f-cb970a5a1fff": {"extension_type": "new-sco"}}' in sco_serialized def test_registered_new_extension_marking_allow_custom_false(): @stix2.v21.CustomMarking( 'my-favorite-marking', [ - ('name', stix2.properties.StringProperty(required=True)), ('some_marking_field', stix2.properties.StringProperty(required=True)), - ], 'stix-extension--a932fcc6-e032-176c-126f-cb970a5a1fff', + ], 'extension-definition--a932fcc6-e032-176c-126f-cb970a5a1fff', ) class MyFavMarking: pass @@ -1506,10 +1511,10 @@ def test_registered_new_extension_marking_allow_custom_false(): 'spec_version': '2.1', 'id': 'marking-definition--f9dbe89c-0030-4a9d-8b78-0dcd0a0de874', 'name': 'This is the name of my favorite Marking', - 'some_marking_field': 'value', 'extensions': { - 'stix-extension--a932fcc6-e032-176c-126f-cb970a5a1fff': { - 'extends_stix_object_definition': True, + 'extension-definition--a932fcc6-e032-176c-126f-cb970a5a1fff': { + 'extension_type': 'property-extension', + 'some_marking_field': 'value', }, }, } @@ -1517,9 +1522,10 @@ def test_registered_new_extension_marking_allow_custom_false(): marking_object = stix2.parse(my_favorite_marking) assert isinstance(marking_object, stix2.v21.MarkingDefinition) assert isinstance( - marking_object.extensions['stix-extension--a932fcc6-e032-176c-126f-cb970a5a1fff'], - stix2.v21.EXT_MAP['stix-extension--a932fcc6-e032-176c-126f-cb970a5a1fff'], + marking_object.extensions['extension-definition--a932fcc6-e032-176c-126f-cb970a5a1fff'], + stix2.v21.EXT_MAP['extension-definition--a932fcc6-e032-176c-126f-cb970a5a1fff'], ) - marking_serialized = marking_object.serialize() - assert '"extensions": {"stix-extension--a932fcc6-e032-176c-126f-cb970a5a1fff": {"extends_stix_object_definition": true}}' in marking_serialized + marking_serialized = marking_object.serialize(sort_keys=True) + assert '"extensions": {"extension-definition--a932fcc6-e032-176c-126f-cb970a5a1fff": ' \ + '{"extension_type": "property-extension", "some_marking_field": "value"}}' in marking_serialized diff --git a/stix2/v21/__init__.py b/stix2/v21/__init__.py index be7e343..977fdf7 100644 --- a/stix2/v21/__init__.py +++ b/stix2/v21/__init__.py @@ -19,9 +19,9 @@ from .base import ( ) from .bundle import Bundle from .common import ( - TLP_AMBER, TLP_GREEN, TLP_RED, TLP_WHITE, CustomMarking, ExternalReference, - GranularMarking, KillChainPhase, LanguageContent, MarkingDefinition, - StatementMarking, STIXExtension, TLPMarking, + TLP_AMBER, TLP_GREEN, TLP_RED, TLP_WHITE, CustomMarking, + ExtensionDefinition, ExternalReference, GranularMarking, KillChainPhase, + LanguageContent, MarkingDefinition, StatementMarking, TLPMarking, ) from .observables import ( URL, AlternateDataStream, ArchiveExt, Artifact, AutonomousSystem, @@ -66,7 +66,7 @@ OBJ_MAP = { 'threat-actor': ThreatActor, 'tool': Tool, 'sighting': Sighting, - 'stix-extension': STIXExtension, + 'extension-definition': ExtensionDefinition, 'vulnerability': Vulnerability, } @@ -112,9 +112,9 @@ EXT_MAP = { __all__ = """ Bundle, - TLP_AMBER, TLP_GREEN, TLP_RED, TLP_WHITE, CustomMarking, ExternalReference, - GranularMarking, KillChainPhase, LanguageContent, MarkingDefinition, - StatementMarking, STIXExtension, TLPMarking, + TLP_AMBER, TLP_GREEN, TLP_RED, TLP_WHITE, CustomMarking, ExtensionDefinition, + ExternalReference, GranularMarking, KillChainPhase, LanguageContent, + MarkingDefinition, StatementMarking, TLPMarking, URL, AlternateDataStream, ArchiveExt, Artifact, AutonomousSystem, CustomExtension, CustomObservable, Directory, DomainName, EmailAddress, diff --git a/stix2/v21/common.py b/stix2/v21/common.py index 06d9c43..4410975 100644 --- a/stix2/v21/common.py +++ b/stix2/v21/common.py @@ -105,12 +105,12 @@ class LanguageContent(_STIXBase21): ]) -class STIXExtension(_STIXBase21): +class ExtensionDefinition(_STIXBase21): """For more detailed information on this object's properties, see `the STIX 2.1 specification `__. """ - _type = 'stix-extension' + _type = 'extension-definition' _properties = OrderedDict([ ('type', TypeProperty(_type, spec_version='2.1')), ('spec_version', StringProperty(fixed='2.1')), From 6a6f13c37b978b7e73b065f47c34001807387404 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Wed, 23 Dec 2020 16:07:30 -0500 Subject: [PATCH 13/56] rename the Class auto-naming from 'STIXExtension' to 'ExtensionDefinition' --- stix2/test/v21/test_custom.py | 2 +- stix2/v21/common.py | 2 +- stix2/v21/observables.py | 2 +- stix2/v21/sdo.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/stix2/test/v21/test_custom.py b/stix2/test/v21/test_custom.py index a2e41b2..c9c9f5d 100644 --- a/stix2/test/v21/test_custom.py +++ b/stix2/test/v21/test_custom.py @@ -1449,7 +1449,7 @@ def test_registered_new_extension_sdo_allow_custom_false(): 'some_property_name1': 'value1', 'some_property_name2': 'value2', # 'extensions': { - # 'extension-definition--d83fce45-ef58-4c6c-a3f4-1fbc32e9999': STIXExtensiond83fce45ef584c6ca3f41fbc32e98c6e() + # 'extension-definition--d83fce45-ef58-4c6c-a3f4-1fbc32e9999': ExtensionDefinitiond83fce45ef584c6ca3f41fbc32e98c6e() # } } sdo_object = stix2.parse(my_favorite_sdo) diff --git a/stix2/v21/common.py b/stix2/v21/common.py index 4410975..0a6929e 100644 --- a/stix2/v21/common.py +++ b/stix2/v21/common.py @@ -259,7 +259,7 @@ def CustomMarking(type='x-custom-marking', properties=None, extension_name=None) extension = extension_name.split('--')[1] extension = extension.replace('-', '') - NameExtension.__name__ = 'STIXExtension' + extension + NameExtension.__name__ = 'ExtensionDefinition' + extension cls.with_extension = extension_name return _custom_marking_builder(cls, type, properties, '2.1', _STIXBase21) return wrapper diff --git a/stix2/v21/observables.py b/stix2/v21/observables.py index 84ac136..b30455d 100644 --- a/stix2/v21/observables.py +++ b/stix2/v21/observables.py @@ -950,7 +950,7 @@ def CustomObservable(type='x-custom-observable', properties=None, id_contrib_pro extension = extension_name.split('--')[1] extension = extension.replace('-', '') - NameExtension.__name__ = 'STIXExtension' + extension + NameExtension.__name__ = 'ExtensionDefinition' + extension cls.with_extension = extension_name return _custom_observable_builder(cls, type, _properties, '2.1', _Observable, id_contrib_props) return wrapper diff --git a/stix2/v21/sdo.py b/stix2/v21/sdo.py index af22fc5..63e2d13 100644 --- a/stix2/v21/sdo.py +++ b/stix2/v21/sdo.py @@ -866,7 +866,7 @@ def CustomObject(type='x-custom-type', properties=None, extension_name=None): extension = extension_name.split('--')[1] extension = extension.replace('-', '') - NameExtension.__name__ = 'STIXExtension' + extension + NameExtension.__name__ = 'ExtensionDefinition' + extension cls.with_extension = extension_name return _custom_object_builder(cls, type, _properties, '2.1', _DomainObject) From a86b21dbf7a4f7ef47b2900c104ad356532b4737 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Wed, 23 Dec 2020 16:15:31 -0500 Subject: [PATCH 14/56] add checks for MarkingDefinition requirements --- stix2/v21/common.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/stix2/v21/common.py b/stix2/v21/common.py index 0a6929e..308cdf2 100644 --- a/stix2/v21/common.py +++ b/stix2/v21/common.py @@ -3,7 +3,7 @@ from collections import OrderedDict from ..custom import _custom_marking_builder -from ..exceptions import InvalidValueError +from ..exceptions import InvalidValueError, PropertyPresenceError from ..markings import _MarkingsMixin from ..markings.utils import check_tlp_marking from ..properties import ( @@ -222,6 +222,23 @@ class MarkingDefinition(_STIXBase21, _MarkingsMixin): def _check_object_constraints(self): super(MarkingDefinition, self)._check_object_constraints() + + definition = self.get("definition") + definition_type = self.get("definition_type") + extensions = self.get("extensions") + if not extensions and not definition: + raise PropertyPresenceError( + "MarkingDefinition objects must have the property 'definition' " + "if 'extensions' is not present", + MarkingDefinition, + ) + if not extensions and not definition_type: + raise PropertyPresenceError( + "MarkingDefinition objects must have the property 'definition_type' " + "if 'extensions' is not present", + MarkingDefinition, + ) + check_tlp_marking(self, '2.1') def serialize(self, pretty=False, include_optional_defaults=False, **kwargs): From be0ebbad654efede0dcf7374c8da6726dec497a4 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Fri, 15 Jan 2021 13:53:43 -0500 Subject: [PATCH 15/56] add test_incident.py --- stix2/test/v21/test_incident.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 stix2/test/v21/test_incident.py diff --git a/stix2/test/v21/test_incident.py b/stix2/test/v21/test_incident.py new file mode 100644 index 0000000..e69de29 From b696f145605822b4ea284833646ba49e5e135f26 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Fri, 15 Jan 2021 15:13:30 -0500 Subject: [PATCH 16/56] add content for basic incident testing --- stix2/test/v21/constants.py | 1 + stix2/test/v21/test_incident.py | 81 +++++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+) diff --git a/stix2/test/v21/constants.py b/stix2/test/v21/constants.py index 78ee076..8084ea2 100644 --- a/stix2/test/v21/constants.py +++ b/stix2/test/v21/constants.py @@ -9,6 +9,7 @@ CAMPAIGN_ID = "campaign--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f" COURSE_OF_ACTION_ID = "course-of-action--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f" GROUPING_ID = "grouping--753abcde-3141-5926-ace5-0a810b1ff996" IDENTITY_ID = "identity--311b2d2d-f010-4473-83ec-1edf84858f4c" +INCIDENT_ID = "incident--40fc3b35-0dc4-4afd-9927-288d44bfce20" INDICATOR_ID = "indicator--a740531e-63ff-4e49-a9e1-a0a3eed0e3e7" INFRASTRUCTURE_ID = "infrastructure--3000ae1b-784c-f03d-8abc-0a625b2ff018" INTRUSION_SET_ID = "intrusion-set--4e78f46f-a023-4e5f-bc24-71b3ca22ec29" diff --git a/stix2/test/v21/test_incident.py b/stix2/test/v21/test_incident.py index e69de29..7ee07b8 100644 --- a/stix2/test/v21/test_incident.py +++ b/stix2/test/v21/test_incident.py @@ -0,0 +1,81 @@ +import datetime as dt + +import pytest +import pytz + +import stix2 + +from .constants import INCIDENT_ID + +EXPECTED = """{ + "type": "incident", + "spec_version": "2.1", + "id": "incident--40fc3b35-0dc4-4afd-9927-288d44bfce20", + "created": "2015-12-21T19:59:11.000Z", + "modified": "2015-12-21T19:59:11.000Z", + "name": "Breach of Cyber Tech Dynamics", + "description": "Intrusion into enterprise network" +}""" + + +def test_incident_example(): + incident = stix2.v21.Incident( + id=INCIDENT_ID, + created="2015-12-21T19:59:11.000Z", + modified="2015-12-21T19:59:11.000Z", + name="Breach of Cyber Tech Dynamics", + description="Intrusion into enterprise network", + ) + + assert str(incident) == EXPECTED + + +@pytest.mark.parametrize( + "data", [ + EXPECTED, + { + "created": "2015-12-21T19:59:11.000Z", + "id": INCIDENT_ID, + "description": "Intrusion into enterprise network", + "modified": "2015-12-21T19:59:11.000Z", + "name": "Breach of Cyber Tech Dynamics", + "spec_version": "2.1", + "type": "incident", + }, + ], +) +def test_parse_incident(data): + incident = stix2.parse(data, version="2.1") + + assert incident.type == 'incident' + assert incident.spec_version == '2.1' + assert incident.id == INCIDENT_ID + assert incident.created == dt.datetime(2015, 12, 21, 19, 59, 11, tzinfo=pytz.utc) + assert incident.modified == dt.datetime(2015, 12, 21, 19, 59, 11, tzinfo=pytz.utc) + assert incident.name == 'Breach of Cyber Tech Dynamics' + assert incident.description == 'Intrusion into enterprise network' + + +def test_parse_no_type(): + with pytest.raises(stix2.exceptions.ParseError): + stix2.parse( + """ + { + "id": "incident--40fc3b35-0dc4-4afd-9927-288d44bfce20", + "created": "2015-12-21T19:59:11.000Z", + "modified": "2015-12-21T19:59:11.000Z", + "name": "Breach of Cyber Tech Dynamics", + "description": "Intrusion into enterprise network" + }""", version="2.1", + ) + + +def test_incident_with_custom(): + incident = stix2.v21.Incident( + name="Breach of Cyber Tech Dynamics", + description="Intrusion into enterprise network", + custom_properties={'x_foo': 'bar'}, + ) + + assert incident.x_foo == "bar" + assert "x_foo" in incident.object_properties() From 367b485fcd6dd96b1277430c16eafc5cf22ba96e Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Fri, 15 Jan 2021 16:01:36 -0500 Subject: [PATCH 17/56] write some tests to check new MarkingDefinition constraints --- stix2/test/v21/test_marking_definition.py | 28 ++++++++++++++++++++++- stix2/v21/common.py | 13 ++++------- 2 files changed, 31 insertions(+), 10 deletions(-) diff --git a/stix2/test/v21/test_marking_definition.py b/stix2/test/v21/test_marking_definition.py index 232bdf2..7b4fc62 100644 --- a/stix2/test/v21/test_marking_definition.py +++ b/stix2/test/v21/test_marking_definition.py @@ -1,7 +1,7 @@ import pytest -from stix2 import exceptions +from stix2 import exceptions, parse from stix2.v21 import ( TLP_AMBER, TLP_GREEN, TLP_RED, TLP_WHITE, MarkingDefinition, TLPMarking, ) @@ -143,3 +143,29 @@ def test_unknown_tlp_marking(): definition_type='tlp', definition=TLPMarking(tlp='gray'), ) + + +def test_marking_definition_missing_definition(): + my_favorite_marking = { + 'type': 'marking-definition', + 'spec_version': '2.1', + 'id': 'marking-definition--f9dbe89c-0030-4a9d-8b78-0dcd0a0de874', + 'name': 'This is the name of my favorite Marking', + 'definition_type': 'foobar' + } + with pytest.raises(exceptions.PropertyPresenceError): + parse(my_favorite_marking) + + +def test_marking_definition_missing_definition_type(): + my_favorite_marking = { + 'type': 'marking-definition', + 'spec_version': '2.1', + 'id': 'marking-definition--f9dbe89c-0030-4a9d-8b78-0dcd0a0de874', + 'name': 'This is the name of my favorite Marking', + 'definition': { + 'some_type': 'foobar' + } + } + with pytest.raises(exceptions.InvalidValueError): + parse(my_favorite_marking) diff --git a/stix2/v21/common.py b/stix2/v21/common.py index 308cdf2..bf3aa0f 100644 --- a/stix2/v21/common.py +++ b/stix2/v21/common.py @@ -226,16 +226,11 @@ class MarkingDefinition(_STIXBase21, _MarkingsMixin): definition = self.get("definition") definition_type = self.get("definition_type") extensions = self.get("extensions") - if not extensions and not definition: + + if not (definition_type and definition) and not extensions: raise PropertyPresenceError( - "MarkingDefinition objects must have the property 'definition' " - "if 'extensions' is not present", - MarkingDefinition, - ) - if not extensions and not definition_type: - raise PropertyPresenceError( - "MarkingDefinition objects must have the property 'definition_type' " - "if 'extensions' is not present", + "MarkingDefinition objects must have the properties " + "'definition_type' and 'definition' if 'extensions' is not present", MarkingDefinition, ) From a48af17cf93db9e2ce410278b469bedcded84f35 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Fri, 15 Jan 2021 16:05:57 -0500 Subject: [PATCH 18/56] styling fixes --- stix2/test/v21/test_marking_definition.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/stix2/test/v21/test_marking_definition.py b/stix2/test/v21/test_marking_definition.py index 7b4fc62..7fe32a8 100644 --- a/stix2/test/v21/test_marking_definition.py +++ b/stix2/test/v21/test_marking_definition.py @@ -151,7 +151,7 @@ def test_marking_definition_missing_definition(): 'spec_version': '2.1', 'id': 'marking-definition--f9dbe89c-0030-4a9d-8b78-0dcd0a0de874', 'name': 'This is the name of my favorite Marking', - 'definition_type': 'foobar' + 'definition_type': 'foobar', } with pytest.raises(exceptions.PropertyPresenceError): parse(my_favorite_marking) @@ -164,8 +164,8 @@ def test_marking_definition_missing_definition_type(): 'id': 'marking-definition--f9dbe89c-0030-4a9d-8b78-0dcd0a0de874', 'name': 'This is the name of my favorite Marking', 'definition': { - 'some_type': 'foobar' - } + 'some_type': 'foobar', + }, } with pytest.raises(exceptions.InvalidValueError): parse(my_favorite_marking) From a62dfbb41bcb74451a68dcfb65060140790f7ae0 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Tue, 26 Jan 2021 22:17:03 -0500 Subject: [PATCH 19/56] missing common properties on CustomObservable decorator --- stix2/v21/observables.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/stix2/v21/observables.py b/stix2/v21/observables.py index e39c709..a68e3d5 100644 --- a/stix2/v21/observables.py +++ b/stix2/v21/observables.py @@ -952,8 +952,10 @@ def CustomObservable(type='x-custom-observable', properties=None, id_contrib_pro [('type', TypeProperty(type, spec_version='2.1'))], [('spec_version', StringProperty(fixed='2.1'))], [('id', IDProperty(type, spec_version='2.1'))], - [('spec_version', StringProperty(fixed='2.1'))], properties, + [('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1')))], + [('granular_markings', ListProperty(GranularMarking))], + [('defanged', BooleanProperty(default=lambda: False))], [('extensions', ExtensionsProperty(spec_version='2.1'))], ]), ) From 5067a3ff76c3c9d613cb49f908c4d6616b943aca Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Fri, 19 Feb 2021 10:05:56 -0500 Subject: [PATCH 20/56] couple of changes after merging against master --- stix2/custom.py | 4 +-- stix2/parsing.py | 3 -- stix2/registration.py | 70 +++++++++++++++---------------------------- stix2/registry.py | 2 +- stix2/v21/bundle.py | 2 +- 5 files changed, 28 insertions(+), 53 deletions(-) diff --git a/stix2/custom.py b/stix2/custom.py index 5912dbf..0d83742 100644 --- a/stix2/custom.py +++ b/stix2/custom.py @@ -4,8 +4,8 @@ import six from .base import _cls_init from .registration import ( - _register_marking, _register_object, _register_observable, - _register_observable_extension, + _get_extension_class, _register_extension, _register_marking, + _register_object, _register_observable, ) diff --git a/stix2/parsing.py b/stix2/parsing.py index ea6ffd1..001f523 100644 --- a/stix2/parsing.py +++ b/stix2/parsing.py @@ -1,9 +1,6 @@ """STIX2 Core parsing methods.""" import copy -import importlib -import pkgutil -import re from . import registry from .exceptions import ParseError diff --git a/stix2/registration.py b/stix2/registration.py index 4ec019a..28d43ba 100644 --- a/stix2/registration.py +++ b/stix2/registration.py @@ -1,14 +1,13 @@ import re -from . import registry -from .base import _DomainObject, _Observable +from . import registry, version +from .base import _DomainObject from .exceptions import DuplicateRegistrationError from .properties import _validate_type from .utils import PREFIX_21_REGEX, get_class_hierarchy_names -from .version import DEFAULT_VERSION -def _register_object(new_type, version=DEFAULT_VERSION): +def _register_object(new_type, version=version.DEFAULT_VERSION): """Register a custom STIX Object type. Args: @@ -32,7 +31,7 @@ def _register_object(new_type, version=DEFAULT_VERSION): properties = new_type._properties if not version: - version = DEFAULT_VERSION + version = version.DEFAULT_VERSION if version == "2.1": for prop_name, prop in properties.items(): @@ -45,7 +44,7 @@ def _register_object(new_type, version=DEFAULT_VERSION): OBJ_MAP[new_type._type] = new_type -def _register_marking(new_marking, version=DEFAULT_VERSION): +def _register_marking(new_marking, version=version.DEFAULT_VERSION): """Register a custom STIX Marking Definition type. Args: @@ -59,7 +58,7 @@ def _register_marking(new_marking, version=DEFAULT_VERSION): properties = new_marking._properties if not version: - version = DEFAULT_VERSION + version = version.DEFAULT_VERSION _validate_type(mark_type, version) @@ -74,7 +73,7 @@ def _register_marking(new_marking, version=DEFAULT_VERSION): OBJ_MAP_MARKING[mark_type] = new_marking -def _register_observable(new_observable, version=DEFAULT_VERSION): +def _register_observable(new_observable, version=version.DEFAULT_VERSION): """Register a custom STIX Cyber Observable type. Args: @@ -86,7 +85,7 @@ def _register_observable(new_observable, version=DEFAULT_VERSION): properties = new_observable._properties if not version: - version = DEFAULT_VERSION + version = version.DEFAULT_VERSION if version == "2.0": # If using STIX2.0, check properties ending in "_ref/s" are ObjectReferenceProperties @@ -133,27 +132,25 @@ def _register_observable(new_observable, version=DEFAULT_VERSION): OBJ_MAP_OBSERVABLE[new_observable._type] = new_observable -def _register_observable_extension( - observable, new_extension, version=DEFAULT_VERSION, +def _get_extension_class(extension_uuid, version): + """Retrieve a registered class Extension""" + return registry.STIX2_OBJ_MAPS[version]['extensions'].get(extension_uuid) + + +def _register_extension( + new_extension, version=version.DEFAULT_VERSION, ): - """Register a custom extension to a STIX Cyber Observable type. + """Register a custom extension to any STIX Object type. Args: - observable: An observable class or instance - new_extension (class): A class to register in the Observables - Extensions map. + new_extension (class): A class to register in the Extensions map. version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). Defaults to the latest supported version. """ - obs_class = observable if isinstance(observable, type) else \ - type(observable) ext_type = new_extension._type properties = new_extension._properties - if not issubclass(obs_class, _Observable): - raise ValueError("'observable' must be a valid Observable class!") - _validate_type(ext_type, version) if not new_extension._properties: @@ -163,37 +160,18 @@ def _register_observable_extension( ) if version == "2.1": - if not ext_type.endswith('-ext'): + if not (ext_type.endswith('-ext') or ext_type.startswith('extension-definition--')): raise ValueError( - "Invalid extension type name '%s': must end with '-ext'." % + "Invalid extension type name '%s': must end with '-ext' or start with 'extension-definition--'." % ext_type, ) - for prop_name, prop_value in properties.items(): + for prop_name in properties.keys(): if not re.match(PREFIX_21_REGEX, prop_name): raise ValueError("Property name '%s' must begin with an alpha character." % prop_name) - try: - observable_type = observable._type - except AttributeError: - raise ValueError( - "Unknown observable type. Custom observables must be " - "created with the @CustomObservable decorator.", - ) + EXT_MAP = registry.STIX2_OBJ_MAPS[version]['extensions'] - OBJ_MAP_OBSERVABLE = registry.STIX2_OBJ_MAPS[version]['observables'] - EXT_MAP = registry.STIX2_OBJ_MAPS[version]['observable-extensions'] - - try: - if ext_type in EXT_MAP[observable_type].keys(): - raise DuplicateRegistrationError("Observable Extension", ext_type) - EXT_MAP[observable_type][ext_type] = new_extension - except KeyError: - if observable_type not in OBJ_MAP_OBSERVABLE: - raise ValueError( - "Unknown observable type '%s'. Custom observables " - "must be created with the @CustomObservable decorator." - % observable_type, - ) - else: - EXT_MAP[observable_type] = {ext_type: new_extension} + if ext_type in EXT_MAP: + raise DuplicateRegistrationError("Extension", ext_type) + EXT_MAP[ext_type] = new_extension diff --git a/stix2/registry.py b/stix2/registry.py index 3dcc3a5..90e2826 100644 --- a/stix2/registry.py +++ b/stix2/registry.py @@ -37,7 +37,7 @@ def _collect_stix2_mappings(): STIX2_OBJ_MAPS[ver] = {} STIX2_OBJ_MAPS[ver]['objects'] = mod.OBJ_MAP STIX2_OBJ_MAPS[ver]['observables'] = mod.OBJ_MAP_OBSERVABLE - STIX2_OBJ_MAPS[ver]['observable-extensions'] = mod.EXT_MAP + STIX2_OBJ_MAPS[ver]['extensions'] = mod.EXT_MAP elif re.match(r'^stix2\.v2[0-9]\.common$', name) and is_pkg is False: ver = _stix_vid_to_version(stix_vid) mod = importlib.import_module(name, str(top_level_module.__name__)) diff --git a/stix2/v21/bundle.py b/stix2/v21/bundle.py index 5497da5..990dfc1 100644 --- a/stix2/v21/bundle.py +++ b/stix2/v21/bundle.py @@ -40,7 +40,7 @@ class Bundle(_STIXBase21): def get_obj(self, obj_uuid): if "objects" in self._inner: found_objs = [elem for elem in self.objects if elem['id'] == obj_uuid] - if found_objs == []: + if not found_objs: raise KeyError("'%s' does not match the id property of any of the bundle's objects" % obj_uuid) return found_objs else: From 479cff818abae2ede8fae9b170b924e4062d2563 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Fri, 19 Feb 2021 22:15:11 -0500 Subject: [PATCH 21/56] few leftover changes... --- stix2/base.py | 2 +- stix2/patterns.py | 4 ++-- stix2/v20/common.py | 2 +- stix2/v21/common.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/stix2/base.py b/stix2/base.py index 12ef373..375030d 100644 --- a/stix2/base.py +++ b/stix2/base.py @@ -94,7 +94,7 @@ class _STIXBase(Mapping): current_properties = self.properties_populated() list_of_properties_populated = set(list_of_properties).intersection(current_properties) - if list_of_properties and (not list_of_properties_populated or list_of_properties_populated == set(['extensions'])): + if list_of_properties and (not list_of_properties_populated or list_of_properties_populated == {'extensions'}): raise AtLeastOnePropertyError(self.__class__, list_of_properties) def _check_properties_dependency(self, list_of_properties, list_of_dependent_properties): diff --git a/stix2/patterns.py b/stix2/patterns.py index f9f451e..a718bf7 100644 --- a/stix2/patterns.py +++ b/stix2/patterns.py @@ -56,7 +56,7 @@ class TimestampConstant(_Constant): class IntegerConstant(_Constant): - """Pattern interger constant + """Pattern integer constant Args: value (int): integer value @@ -265,7 +265,7 @@ class BasicObjectPathComponent(_ObjectPathComponent): """Basic object path component (for an observation or expression) By "Basic", implies that the object path component is not a - list, object reference or futher referenced property, i.e. terminal + list, object reference or further referenced property, i.e. terminal component Args: diff --git a/stix2/v20/common.py b/stix2/v20/common.py index 6695c9a..d68b4a7 100644 --- a/stix2/v20/common.py +++ b/stix2/v20/common.py @@ -130,7 +130,7 @@ class MarkingDefinition(_STIXBase20, _MarkingsMixin): ]) def __init__(self, **kwargs): - if set(('definition_type', 'definition')).issubset(kwargs.keys()): + if {'definition_type', 'definition'}.issubset(kwargs.keys()): # Create correct marking type object try: marking_type = OBJ_MAP_MARKING[kwargs['definition_type']] diff --git a/stix2/v21/common.py b/stix2/v21/common.py index bf3aa0f..df95911 100644 --- a/stix2/v21/common.py +++ b/stix2/v21/common.py @@ -207,7 +207,7 @@ class MarkingDefinition(_STIXBase21, _MarkingsMixin): ]) def __init__(self, **kwargs): - if set(('definition_type', 'definition')).issubset(kwargs.keys()): + if {'definition_type', 'definition'}.issubset(kwargs.keys()): # Create correct marking type object try: marking_type = OBJ_MAP_MARKING[kwargs['definition_type']] From d5e6226c829db118d7af0f9a6f8933abe2a90df5 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Fri, 19 Feb 2021 23:27:41 -0500 Subject: [PATCH 22/56] update hyperlinks to STIX 2.1 CS 02 --- stix2/v21/bundle.py | 2 +- stix2/v21/common.py | 16 ++++----- stix2/v21/observables.py | 72 ++++++++++++++++++++-------------------- stix2/v21/sdo.py | 38 ++++++++++----------- stix2/v21/sro.py | 4 +-- 5 files changed, 66 insertions(+), 66 deletions(-) diff --git a/stix2/v21/bundle.py b/stix2/v21/bundle.py index 990dfc1..0d073ca 100644 --- a/stix2/v21/bundle.py +++ b/stix2/v21/bundle.py @@ -10,7 +10,7 @@ from .base import _STIXBase21 class Bundle(_STIXBase21): """For more detailed information on this object's properties, see - `the STIX 2.1 specification `__. + `the STIX 2.1 specification `__. """ _type = 'bundle' diff --git a/stix2/v21/common.py b/stix2/v21/common.py index df95911..12e70f2 100644 --- a/stix2/v21/common.py +++ b/stix2/v21/common.py @@ -18,7 +18,7 @@ from .base import _STIXBase21 class ExternalReference(_STIXBase21): """For more detailed information on this object's properties, see - `the STIX 2.1 specification `__. + `the STIX 2.1 specification `__. """ _properties = OrderedDict([ @@ -52,7 +52,7 @@ class ExternalReference(_STIXBase21): class KillChainPhase(_STIXBase21): """For more detailed information on this object's properties, see - `the STIX 2.1 specification `__. + `the STIX 2.1 specification `__. """ _properties = OrderedDict([ @@ -63,7 +63,7 @@ class KillChainPhase(_STIXBase21): class GranularMarking(_STIXBase21): """For more detailed information on this object's properties, see - `the STIX 2.1 specification `__. + `the STIX 2.1 specification `__. """ _properties = OrderedDict([ @@ -79,7 +79,7 @@ class GranularMarking(_STIXBase21): class LanguageContent(_STIXBase21): """For more detailed information on this object's properties, see - `the STIX 2.1 specification `__. + `the STIX 2.1 specification `__. """ _type = 'language-content' @@ -107,7 +107,7 @@ class LanguageContent(_STIXBase21): class ExtensionDefinition(_STIXBase21): """For more detailed information on this object's properties, see - `the STIX 2.1 specification `__. + `the STIX 2.1 specification `__. """ _type = 'extension-definition' @@ -146,7 +146,7 @@ class ExtensionDefinition(_STIXBase21): class TLPMarking(_STIXBase21): """For more detailed information on this object's properties, see - `the STIX 2.1 specification `__. + `the STIX 2.1 specification `__. """ _type = 'tlp' @@ -157,7 +157,7 @@ class TLPMarking(_STIXBase21): class StatementMarking(_STIXBase21): """For more detailed information on this object's properties, see - `the STIX 2.1 specification `__. + `the STIX 2.1 specification `__. """ _type = 'statement' @@ -187,7 +187,7 @@ class MarkingProperty(Property): class MarkingDefinition(_STIXBase21, _MarkingsMixin): """For more detailed information on this object's properties, see - `the STIX 2.1 specification `__. + `the STIX 2.1 specification `__. """ _type = 'marking-definition' diff --git a/stix2/v21/observables.py b/stix2/v21/observables.py index a68e3d5..6fb218c 100644 --- a/stix2/v21/observables.py +++ b/stix2/v21/observables.py @@ -22,7 +22,7 @@ from .common import GranularMarking class Artifact(_Observable): """For more detailed information on this object's properties, see - `the STIX 2.1 specification `__. + `the STIX 2.1 specification `__. """ _type = 'artifact' @@ -51,7 +51,7 @@ class Artifact(_Observable): class AutonomousSystem(_Observable): """For more detailed information on this object's properties, see - `the STIX 2.1 specification `__. + `the STIX 2.1 specification `__. """ _type = 'autonomous-system' @@ -72,7 +72,7 @@ class AutonomousSystem(_Observable): class Directory(_Observable): """For more detailed information on this object's properties, see - `the STIX 2.1 specification `__. + `the STIX 2.1 specification `__. """ _type = 'directory' @@ -97,7 +97,7 @@ class Directory(_Observable): class DomainName(_Observable): """For more detailed information on this object's properties, see - `the STIX 2.1 specification `__. + `the STIX 2.1 specification `__. """ _type = 'domain-name' @@ -117,7 +117,7 @@ class DomainName(_Observable): class EmailAddress(_Observable): """For more detailed information on this object's properties, see - `the STIX 2.1 specification `__. + `the STIX 2.1 specification `__. """ _type = 'email-addr' @@ -138,7 +138,7 @@ class EmailAddress(_Observable): class EmailMIMEComponent(_STIXBase21): """For more detailed information on this object's properties, see - `the STIX 2.1 specification `__. + `the STIX 2.1 specification `__. """ _properties = OrderedDict([ @@ -155,7 +155,7 @@ class EmailMIMEComponent(_STIXBase21): class EmailMessage(_Observable): """For more detailed information on this object's properties, see - `the STIX 2.1 specification `__. + `the STIX 2.1 specification `__. """ _type = 'email-message' @@ -195,7 +195,7 @@ class EmailMessage(_Observable): class ArchiveExt(_Extension): """For more detailed information on this object's properties, see - `the STIX 2.1 specification `__. + `the STIX 2.1 specification `__. """ _type = 'archive-ext' @@ -207,7 +207,7 @@ class ArchiveExt(_Extension): class AlternateDataStream(_STIXBase21): """For more detailed information on this object's properties, see - `the STIX 2.1 specification `__. + `the STIX 2.1 specification `__. """ _properties = OrderedDict([ @@ -219,7 +219,7 @@ class AlternateDataStream(_STIXBase21): class NTFSExt(_Extension): """For more detailed information on this object's properties, see - `the STIX 2.1 specification `__. + `the STIX 2.1 specification `__. """ _type = 'ntfs-ext' @@ -231,7 +231,7 @@ class NTFSExt(_Extension): class PDFExt(_Extension): """For more detailed information on this object's properties, see - `the STIX 2.1 specification `__. + `the STIX 2.1 specification `__. """ _type = 'pdf-ext' @@ -246,7 +246,7 @@ class PDFExt(_Extension): class RasterImageExt(_Extension): """For more detailed information on this object's properties, see - `the STIX 2.1 specification `__. + `the STIX 2.1 specification `__. """ _type = 'raster-image-ext' @@ -260,7 +260,7 @@ class RasterImageExt(_Extension): class WindowsPEOptionalHeaderType(_STIXBase21): """For more detailed information on this object's properties, see - `the STIX 2.1 specification `__. + `the STIX 2.1 specification `__. """ _properties = OrderedDict([ @@ -304,7 +304,7 @@ class WindowsPEOptionalHeaderType(_STIXBase21): class WindowsPESection(_STIXBase21): """For more detailed information on this object's properties, see - `the STIX 2.1 specification `__. + `the STIX 2.1 specification `__. """ _properties = OrderedDict([ @@ -317,7 +317,7 @@ class WindowsPESection(_STIXBase21): class WindowsPEBinaryExt(_Extension): """For more detailed information on this object's properties, see - `the STIX 2.1 specification `__. + `the STIX 2.1 specification `__. """ _type = 'windows-pebinary-ext' @@ -339,7 +339,7 @@ class WindowsPEBinaryExt(_Extension): class File(_Observable): """For more detailed information on this object's properties, see - `the STIX 2.1 specification `__. + `the STIX 2.1 specification `__. """ _type = 'file' @@ -373,7 +373,7 @@ class File(_Observable): class IPv4Address(_Observable): """For more detailed information on this object's properties, see - `the STIX 2.1 specification `__. + `the STIX 2.1 specification `__. """ _type = 'ipv4-addr' @@ -394,7 +394,7 @@ class IPv4Address(_Observable): class IPv6Address(_Observable): """For more detailed information on this object's properties, see - `the STIX 2.1 specification `__. + `the STIX 2.1 specification `__. """ _type = 'ipv6-addr' @@ -415,7 +415,7 @@ class IPv6Address(_Observable): class MACAddress(_Observable): """For more detailed information on this object's properties, see - `the STIX 2.1 specification `__. + `the STIX 2.1 specification `__. """ _type = 'mac-addr' @@ -434,7 +434,7 @@ class MACAddress(_Observable): class Mutex(_Observable): """For more detailed information on this object's properties, see - `the STIX 2.1 specification `__. + `the STIX 2.1 specification `__. """ _type = 'mutex' @@ -453,7 +453,7 @@ class Mutex(_Observable): class HTTPRequestExt(_Extension): """For more detailed information on this object's properties, see - `the STIX 2.1 specification `__. + `the STIX 2.1 specification `__. """ _type = 'http-request-ext' @@ -470,7 +470,7 @@ class HTTPRequestExt(_Extension): class ICMPExt(_Extension): # TODO: Add link """For more detailed information on this object's properties, see - `the STIX 2.1 specification `__. + `the STIX 2.1 specification `__. """ _type = 'icmp-ext' @@ -482,7 +482,7 @@ class ICMPExt(_Extension): class SocketExt(_Extension): """For more detailed information on this object's properties, see - `the STIX 2.1 specification `__. + `the STIX 2.1 specification `__. """ _type = 'socket-ext' @@ -535,7 +535,7 @@ class SocketExt(_Extension): class TCPExt(_Extension): """For more detailed information on this object's properties, see - `the STIX 2.1 specification `__. + `the STIX 2.1 specification `__. """ _type = 'tcp-ext' @@ -547,7 +547,7 @@ class TCPExt(_Extension): class NetworkTraffic(_Observable): """For more detailed information on this object's properties, see - `the STIX 2.1 specification `__. + `the STIX 2.1 specification `__. """ _type = 'network-traffic' @@ -602,7 +602,7 @@ class NetworkTraffic(_Observable): class WindowsProcessExt(_Extension): """For more detailed information on this object's properties, see - `the STIX 2.1 specification `__. + `the STIX 2.1 specification `__. """ _type = 'windows-process-ext' @@ -628,7 +628,7 @@ class WindowsProcessExt(_Extension): class WindowsServiceExt(_Extension): """For more detailed information on this object's properties, see - `the STIX 2.1 specification `__. + `the STIX 2.1 specification `__. """ _type = 'windows-service-ext' @@ -677,7 +677,7 @@ class WindowsServiceExt(_Extension): class Process(_Observable): """For more detailed information on this object's properties, see - `the STIX 2.1 specification `__. + `the STIX 2.1 specification `__. """ _type = 'process' @@ -721,7 +721,7 @@ class Process(_Observable): class Software(_Observable): """For more detailed information on this object's properties, see - `the STIX 2.1 specification `__. + `the STIX 2.1 specification `__. """ _type = 'software' @@ -745,7 +745,7 @@ class Software(_Observable): class URL(_Observable): """For more detailed information on this object's properties, see - `the STIX 2.1 specification `__. + `the STIX 2.1 specification `__. """ _type = 'url' @@ -764,7 +764,7 @@ class URL(_Observable): class UNIXAccountExt(_Extension): """For more detailed information on this object's properties, see - `the STIX 2.1 specification `__. + `the STIX 2.1 specification `__. """ _type = 'unix-account-ext' @@ -778,7 +778,7 @@ class UNIXAccountExt(_Extension): class UserAccount(_Observable): """For more detailed information on this object's properties, see - `the STIX 2.1 specification `__. + `the STIX 2.1 specification `__. """ _type = 'user-account' @@ -810,7 +810,7 @@ class UserAccount(_Observable): class WindowsRegistryValueType(_STIXBase21): """For more detailed information on this object's properties, see - `the STIX 2.1 specification `__. + `the STIX 2.1 specification `__. """ _type = 'windows-registry-value-type' @@ -841,7 +841,7 @@ class WindowsRegistryValueType(_STIXBase21): class WindowsRegistryKey(_Observable): """For more detailed information on this object's properties, see - `the STIX 2.1 specification `__. + `the STIX 2.1 specification `__. """ _type = 'windows-registry-key' @@ -865,7 +865,7 @@ class WindowsRegistryKey(_Observable): class X509V3ExtensionsType(_STIXBase21): """For more detailed information on this object's properties, see - `the STIX 2.1 specification `__. + `the STIX 2.1 specification `__. """ _type = 'x509-v3-extensions-type' @@ -891,7 +891,7 @@ class X509V3ExtensionsType(_STIXBase21): class X509Certificate(_Observable): """For more detailed information on this object's properties, see - `the STIX 2.1 specification `__. + `the STIX 2.1 specification `__. """ _type = 'x509-certificate' diff --git a/stix2/v21/sdo.py b/stix2/v21/sdo.py index 527900b..1ca448a 100644 --- a/stix2/v21/sdo.py +++ b/stix2/v21/sdo.py @@ -25,7 +25,7 @@ from .common import ExternalReference, GranularMarking, KillChainPhase class AttackPattern(_DomainObject): """For more detailed information on this object's properties, see - `the STIX 2.1 specification `__. + `the STIX 2.1 specification `__. """ _type = 'attack-pattern' @@ -53,7 +53,7 @@ class AttackPattern(_DomainObject): class Campaign(_DomainObject): """For more detailed information on this object's properties, see - `the STIX 2.1 specification `__. + `the STIX 2.1 specification `__. """ _type = 'campaign' @@ -93,7 +93,7 @@ class Campaign(_DomainObject): class CourseOfAction(_DomainObject): """For more detailed information on this object's properties, see - `the STIX 2.1 specification `__. + `the STIX 2.1 specification `__. """ _type = 'course-of-action' @@ -119,7 +119,7 @@ class CourseOfAction(_DomainObject): class Grouping(_DomainObject): """For more detailed information on this object's properties, see - `the STIX 2.1 specification `__. + `the STIX 2.1 specification `__. """ _type = 'grouping' @@ -147,7 +147,7 @@ class Grouping(_DomainObject): class Identity(_DomainObject): """For more detailed information on this object's properties, see - `the STIX 2.1 specification `__. + `the STIX 2.1 specification `__. """ _type = 'identity' @@ -177,7 +177,7 @@ class Identity(_DomainObject): class Incident(_DomainObject): """For more detailed information on this object's properties, see - `the STIX 2.1 specification `__. + `the STIX 2.1 specification `__. """ _type = 'incident' @@ -204,7 +204,7 @@ class Incident(_DomainObject): class Indicator(_DomainObject): """For more detailed information on this object's properties, see - `the STIX 2.1 specification `__. + `the STIX 2.1 specification `__. """ _type = 'indicator' @@ -264,7 +264,7 @@ class Indicator(_DomainObject): class Infrastructure(_DomainObject): """For more detailed information on this object's properties, see - `the STIX 2.1 specification `__. + `the STIX 2.1 specification `__. """ _type = 'infrastructure' @@ -305,7 +305,7 @@ class Infrastructure(_DomainObject): class IntrusionSet(_DomainObject): """For more detailed information on this object's properties, see - `the STIX 2.1 specification `__. + `the STIX 2.1 specification `__. """ _type = 'intrusion-set' @@ -348,7 +348,7 @@ class IntrusionSet(_DomainObject): class Location(_DomainObject): """For more detailed information on this object's properties, see - `the STIX 2.1 specification `__. + `the STIX 2.1 specification `__. """ _type = 'location' @@ -456,7 +456,7 @@ class Location(_DomainObject): class Malware(_DomainObject): """For more detailed information on this object's properties, see - `the STIX 2.1 specification `__. + `the STIX 2.1 specification `__. """ _type = 'malware' @@ -509,7 +509,7 @@ class Malware(_DomainObject): class MalwareAnalysis(_DomainObject): """For more detailed information on this object's properties, see - `the STIX 2.1 specification `__. + `the STIX 2.1 specification `__. """ _type = 'malware-analysis' @@ -554,7 +554,7 @@ class MalwareAnalysis(_DomainObject): class Note(_DomainObject): """For more detailed information on this object's properties, see - `the STIX 2.1 specification `__. + `the STIX 2.1 specification `__. """ _type = 'note' @@ -582,7 +582,7 @@ class Note(_DomainObject): class ObservedData(_DomainObject): """For more detailed information on this object's properties, see - `the STIX 2.1 specification `__. + `the STIX 2.1 specification `__. """ _type = 'observed-data' @@ -638,7 +638,7 @@ class ObservedData(_DomainObject): class Opinion(_DomainObject): """For more detailed information on this object's properties, see - `the STIX 2.1 specification `__. + `the STIX 2.1 specification `__. """ _type = 'opinion' @@ -676,7 +676,7 @@ class Opinion(_DomainObject): class Report(_DomainObject): """For more detailed information on this object's properties, see - `the STIX 2.1 specification `__. + `the STIX 2.1 specification `__. """ _type = 'report' @@ -705,7 +705,7 @@ class Report(_DomainObject): class ThreatActor(_DomainObject): """For more detailed information on this object's properties, see - `the STIX 2.1 specification `__. + `the STIX 2.1 specification `__. """ _type = 'threat-actor' @@ -752,7 +752,7 @@ class ThreatActor(_DomainObject): class Tool(_DomainObject): """For more detailed information on this object's properties, see - `the STIX 2.1 specification `__. + `the STIX 2.1 specification `__. """ _type = 'tool' @@ -782,7 +782,7 @@ class Tool(_DomainObject): class Vulnerability(_DomainObject): """For more detailed information on this object's properties, see - `the STIX 2.1 specification `__. + `the STIX 2.1 specification `__. """ _type = 'vulnerability' diff --git a/stix2/v21/sro.py b/stix2/v21/sro.py index 9a0b9c6..bfe3f6d 100644 --- a/stix2/v21/sro.py +++ b/stix2/v21/sro.py @@ -14,7 +14,7 @@ from .common import ExternalReference, GranularMarking class Relationship(_RelationshipObject): """For more detailed information on this object's properties, see - `the STIX 2.1 specification `__. + `the STIX 2.1 specification `__. """ _invalid_source_target_types = ['bundle', 'language-content', 'marking-definition', 'relationship', 'sighting'] @@ -71,7 +71,7 @@ class Relationship(_RelationshipObject): class Sighting(_RelationshipObject): """For more detailed information on this object's properties, see - `the STIX 2.1 specification `__. + `the STIX 2.1 specification `__. """ _type = 'sighting' From e4039231f41fe2453ef6ea9bd251054ee0f6c432 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Sat, 20 Feb 2021 00:10:38 -0500 Subject: [PATCH 23/56] missing link and test case --- README.rst | 4 ++-- stix2/test/v20/test_custom.py | 12 +++++++++++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 662e020..f8f8304 100644 --- a/README.rst +++ b/README.rst @@ -68,8 +68,8 @@ For more in-depth documentation, please see `https://stix2.readthedocs.io/ `__ -published on 20 March 2020 currently at the Committee Specification (CS) level. +This version of cti-python-stix2 brings support to `STIX Version 2.1 `__ +published on 25 January 2021 currently at the Committee Specification (CS) 02 level. The stix2 Python library supports multiple versions of the STIX 2 Technical Specification. The library will be updated to support new Committee diff --git a/stix2/test/v20/test_custom.py b/stix2/test/v20/test_custom.py index 65daa93..e1eb600 100644 --- a/stix2/test/v20/test_custom.py +++ b/stix2/test/v20/test_custom.py @@ -438,8 +438,18 @@ class NewObservable(): def test_custom_observable_object_1(): - no = NewObservable(property1='something') + no = NewObservable( + property1='something', + extensions={ + 'archive-ext': stix2.v20.observables.ArchiveExt( + contains_refs=['file--e277603e-1060-5ad4-9937-c26c97f1ca68'], + version='2.0', + comment='for real', + ) + } + ) assert no.property1 == 'something' + assert no.extensions['archive-ext'].comment == 'for real' def test_custom_observable_object_2(): From b3d0c7b58a23985463172740f1916592e6c7c8d6 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Sat, 20 Feb 2021 00:13:36 -0500 Subject: [PATCH 24/56] styling changes --- stix2/test/v20/test_custom.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stix2/test/v20/test_custom.py b/stix2/test/v20/test_custom.py index e1eb600..f5e7024 100644 --- a/stix2/test/v20/test_custom.py +++ b/stix2/test/v20/test_custom.py @@ -445,8 +445,8 @@ def test_custom_observable_object_1(): contains_refs=['file--e277603e-1060-5ad4-9937-c26c97f1ca68'], version='2.0', comment='for real', - ) - } + ), + }, ) assert no.property1 == 'something' assert no.extensions['archive-ext'].comment == 'for real' From bde9aaa13e50868a71b766944e4963c203afbbe3 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Tue, 30 Mar 2021 22:40:41 -0400 Subject: [PATCH 25/56] update some TODOs and clear out some backwards kept imports (breaking) --- stix2/base.py | 8 +------- stix2/v20/common.py | 1 - stix2/v21/common.py | 4 ++-- stix2/v21/observables.py | 1 - 4 files changed, 3 insertions(+), 11 deletions(-) diff --git a/stix2/base.py b/stix2/base.py index 375030d..d905fb0 100644 --- a/stix2/base.py +++ b/stix2/base.py @@ -16,9 +16,7 @@ from .exceptions import ( ) from .markings import _MarkingsMixin from .markings.utils import validate -from .serialization import ( - STIXJSONEncoder, STIXJSONIncludeOptionalDefaultsEncoder, serialize, -) +from .serialization import STIXJSONEncoder, serialize from .utils import NOW, PREFIX_21_REGEX, get_timestamp from .versioning import new_version as _new_version from .versioning import revoke as _revoke @@ -28,10 +26,6 @@ try: except ImportError: from collections import Mapping -# TODO: Remove STIXJSONEncoder, STIXJSONIncludeOptionalDefaultsEncoder, serialize from __all__ on next major release. -# Kept for backwards compatibility. -__all__ = ['STIXJSONEncoder', 'STIXJSONIncludeOptionalDefaultsEncoder', '_STIXBase', 'serialize'] - DEFAULT_ERROR = "{type} must have {property}='{expected}'." SCO_DET_ID_NAMESPACE = uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7") diff --git a/stix2/v20/common.py b/stix2/v20/common.py index d68b4a7..4b7c299 100644 --- a/stix2/v20/common.py +++ b/stix2/v20/common.py @@ -74,7 +74,6 @@ class TLPMarking(_STIXBase20): `the STIX 2.0 specification `__. """ - # TODO: don't allow the creation of any other TLPMarkings than the ones below _type = 'tlp' _properties = OrderedDict([ ('tlp', StringProperty(required=True)), diff --git a/stix2/v21/common.py b/stix2/v21/common.py index 12e70f2..83d6fa4 100644 --- a/stix2/v21/common.py +++ b/stix2/v21/common.py @@ -91,9 +91,9 @@ class LanguageContent(_STIXBase21): ('created', TimestampProperty(default=lambda: NOW, precision='millisecond', precision_constraint='min')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond', precision_constraint='min')), ('object_ref', ReferenceProperty(valid_types=["SCO", "SDO", "SRO"], spec_version='2.1', required=True)), - # TODO: 'object_modified' it MUST be an exact match for the modified time of the STIX Object (SRO or SDO) being referenced. + # TODO: 'object_modified' MUST be an exact match for the modified time of the STIX Object being referenced ('object_modified', TimestampProperty(precision='millisecond')), - # TODO: 'contents' https://docs.google.com/document/d/1ShNq4c3e1CkfANmD9O--mdZ5H0O_GLnjN28a_yrEaco/edit#heading=h.cfz5hcantmvx + # TODO: Implement 'contents' property requirements as defined in STIX 2.1 CS02 ('contents', DictionaryProperty(spec_version='2.1', required=True)), ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), diff --git a/stix2/v21/observables.py b/stix2/v21/observables.py index 6fb218c..a9cf2ae 100644 --- a/stix2/v21/observables.py +++ b/stix2/v21/observables.py @@ -468,7 +468,6 @@ class HTTPRequestExt(_Extension): class ICMPExt(_Extension): - # TODO: Add link """For more detailed information on this object's properties, see `the STIX 2.1 specification `__. """ From 9cc2e5bd3a82b1c71bb65762007117550fb936da Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Wed, 31 Mar 2021 11:02:05 -0400 Subject: [PATCH 26/56] remove backwards compatible imports, remove pretty=True from _STIXBase.__str()__, and simplify _STIXBase.__repr()__ (breaking) --- stix2/base.py | 25 ++++++++++--------------- stix2/properties.py | 7 ++----- 2 files changed, 12 insertions(+), 20 deletions(-) diff --git a/stix2/base.py b/stix2/base.py index d905fb0..8a5a517 100644 --- a/stix2/base.py +++ b/stix2/base.py @@ -1,5 +1,6 @@ """Base classes for type definitions in the STIX2 library.""" +import collections.abc import copy import re import uuid @@ -21,10 +22,6 @@ from .utils import NOW, PREFIX_21_REGEX, get_timestamp from .versioning import new_version as _new_version from .versioning import revoke as _revoke -try: - from collections.abc import Mapping -except ImportError: - from collections import Mapping DEFAULT_ERROR = "{type} must have {property}='{expected}'." SCO_DET_ID_NAMESPACE = uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7") @@ -34,7 +31,7 @@ def get_required_properties(properties): return (k for k, v in properties.items() if v.required) -class _STIXBase(Mapping): +class _STIXBase(collections.abc.Mapping): """Base class for STIX object types""" def object_properties(self): @@ -67,7 +64,7 @@ class _STIXBase(Mapping): self.__class__, prop_name, reason=str(exc), ) from exc - # interproperty constraint methods + # inter-property constraint methods def _check_mutually_exclusive_properties(self, list_of_properties, at_least_one=True): current_properties = self.properties_populated() @@ -80,9 +77,9 @@ class _STIXBase(Mapping): if not list_of_properties: list_of_properties = sorted(list(self.__class__._properties.keys())) if isinstance(self, _Observable): - props_to_remove = ["type", "id", "defanged", "spec_version"] + props_to_remove = {"type", "id", "defanged", "spec_version"} else: - props_to_remove = ["type"] + props_to_remove = {"type"} list_of_properties = [prop for prop in list_of_properties if prop not in props_to_remove] current_properties = self.properties_populated() @@ -211,14 +208,12 @@ class _STIXBase(Mapping): super(_STIXBase, self).__setattr__(name, value) def __str__(self): - return self.serialize(pretty=True) + # Note: use .serialize() or fp_serialize() directly if specific formatting options are needed. + return self.serialize() def __repr__(self): - props = [(k, self[k]) for k in self.object_properties() if self.get(k)] - return '{0}({1})'.format( - self.__class__.__name__, - ', '.join(['{0!s}={1!r}'.format(k, v) for k, v in props]), - ) + props = ', '.join([f"{k}={self[k]!r}" for k in self.object_properties() if self.get(k)]) + return f'{self.__class__.__name__}({props})' def __deepcopy__(self, memo): # Assume: we can ignore the memo argument, because no object will ever contain the same sub-object multiple times. @@ -415,7 +410,7 @@ def _make_json_serializable(value): json_value = value # default assumption - if isinstance(value, Mapping): + if isinstance(value, collections.abc.Mapping): json_value = { k: _make_json_serializable(v) for k, v in value.items() diff --git a/stix2/properties.py b/stix2/properties.py index 9dc1e6c..8fb8a75 100644 --- a/stix2/properties.py +++ b/stix2/properties.py @@ -2,6 +2,7 @@ import base64 import binascii +import collections.abc import copy import inspect import re @@ -16,10 +17,6 @@ from .exceptions import ( from .parsing import parse, parse_observable from .utils import _get_dict, get_class_hierarchy_names, parse_into_datetime -try: - from collections.abc import Mapping -except ImportError: - from collections import Mapping TYPE_REGEX = re.compile(r'^-?[a-z0-9]+(-[a-z0-9]+)*-?$') TYPE_21_REGEX = re.compile(r'^([a-z][a-z0-9]*)+([a-z0-9-]+)*-?$') @@ -239,7 +236,7 @@ class ListProperty(Property): if isinstance(item, self.contained): valid = item - elif isinstance(item, Mapping): + elif isinstance(item, collections.abc.Mapping): # attempt a mapping-like usage... valid = self.contained(**item) From 6d285c47aee59680db7e0dab4cb578736d949280 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Wed, 31 Mar 2021 12:39:14 -0400 Subject: [PATCH 27/56] update test suite for use of .serialize(pretty=True) --- stix2/test/v20/test_attack_pattern.py | 4 ++-- stix2/test/v20/test_bundle.py | 12 ++++++------ stix2/test/v20/test_campaign.py | 2 +- stix2/test/v20/test_course_of_action.py | 2 +- stix2/test/v20/test_custom.py | 4 ++-- stix2/test/v20/test_external_reference.py | 12 ++++++------ stix2/test/v20/test_identity.py | 2 +- stix2/test/v20/test_indicator.py | 2 +- stix2/test/v20/test_intrusion_set.py | 2 +- stix2/test/v20/test_kill_chain_phases.py | 4 ++-- stix2/test/v20/test_malware.py | 4 ++-- stix2/test/v20/test_markings.py | 12 ++++++------ stix2/test/v20/test_observed_data.py | 4 ++-- stix2/test/v20/test_relationship.py | 2 +- stix2/test/v20/test_report.py | 4 ++-- stix2/test/v20/test_sighting.py | 4 ++-- stix2/test/v20/test_threat_actor.py | 2 +- stix2/test/v20/test_tool.py | 2 +- stix2/test/v20/test_vulnerability.py | 2 +- stix2/test/v21/test_attack_pattern.py | 4 ++-- stix2/test/v21/test_bundle.py | 12 ++++++------ stix2/test/v21/test_campaign.py | 2 +- stix2/test/v21/test_course_of_action.py | 2 +- stix2/test/v21/test_custom.py | 4 ++-- stix2/test/v21/test_external_reference.py | 12 ++++++------ stix2/test/v21/test_grouping.py | 2 +- stix2/test/v21/test_identity.py | 2 +- stix2/test/v21/test_incident.py | 2 +- stix2/test/v21/test_indicator.py | 2 +- stix2/test/v21/test_infrastructure.py | 2 +- stix2/test/v21/test_intrusion_set.py | 2 +- stix2/test/v21/test_kill_chain_phases.py | 4 ++-- stix2/test/v21/test_location.py | 6 +++--- stix2/test/v21/test_malware.py | 4 ++-- stix2/test/v21/test_malware_analysis.py | 4 ++-- stix2/test/v21/test_markings.py | 12 ++++++------ stix2/test/v21/test_note.py | 2 +- stix2/test/v21/test_observed_data.py | 2 +- stix2/test/v21/test_opinion.py | 2 +- stix2/test/v21/test_relationship.py | 2 +- stix2/test/v21/test_report.py | 4 ++-- stix2/test/v21/test_sighting.py | 4 ++-- stix2/test/v21/test_threat_actor.py | 2 +- stix2/test/v21/test_tool.py | 2 +- stix2/test/v21/test_vulnerability.py | 2 +- 45 files changed, 91 insertions(+), 91 deletions(-) diff --git a/stix2/test/v20/test_attack_pattern.py b/stix2/test/v20/test_attack_pattern.py index cc24fa9..1f53450 100644 --- a/stix2/test/v20/test_attack_pattern.py +++ b/stix2/test/v20/test_attack_pattern.py @@ -37,7 +37,7 @@ def test_attack_pattern_example(): description="...", ) - assert str(ap) == EXPECTED + assert ap.serialize(pretty=True) == EXPECTED @pytest.mark.parametrize( @@ -111,6 +111,6 @@ def test_less_precise_timestamps(): description="...", ) - assert str(ap) == EXPECTED + assert ap.serialize(pretty=True) == EXPECTED # TODO: Add other examples diff --git a/stix2/test/v20/test_bundle.py b/stix2/test/v20/test_bundle.py index f53d0cb..3e09192 100644 --- a/stix2/test/v20/test_bundle.py +++ b/stix2/test/v20/test_bundle.py @@ -116,7 +116,7 @@ def test_bundle_id_must_start_with_bundle(): def test_create_bundle1(indicator, malware, relationship): bundle = stix2.v20.Bundle(objects=[indicator, malware, relationship]) - assert str(bundle) == EXPECTED_BUNDLE + assert bundle.serialize(pretty=True) == EXPECTED_BUNDLE assert bundle.serialize(pretty=True) == EXPECTED_BUNDLE @@ -129,31 +129,31 @@ def test_create_bundle2(indicator, malware, relationship): def test_create_bundle_with_positional_args(indicator, malware, relationship): bundle = stix2.v20.Bundle(indicator, malware, relationship) - assert str(bundle) == EXPECTED_BUNDLE + assert bundle.serialize(pretty=True) == EXPECTED_BUNDLE def test_create_bundle_with_positional_listarg(indicator, malware, relationship): bundle = stix2.v20.Bundle([indicator, malware, relationship]) - assert str(bundle) == EXPECTED_BUNDLE + assert bundle.serialize(pretty=True) == EXPECTED_BUNDLE def test_create_bundle_with_listarg_and_positional_arg(indicator, malware, relationship): bundle = stix2.v20.Bundle([indicator, malware], relationship) - assert str(bundle) == EXPECTED_BUNDLE + assert bundle.serialize(pretty=True) == EXPECTED_BUNDLE def test_create_bundle_with_listarg_and_kwarg(indicator, malware, relationship): bundle = stix2.v20.Bundle([indicator, malware], objects=[relationship]) - assert str(bundle) == EXPECTED_BUNDLE + assert bundle.serialize(pretty=True) == EXPECTED_BUNDLE def test_create_bundle_with_arg_listarg_and_kwarg(indicator, malware, relationship): bundle = stix2.v20.Bundle([indicator], malware, objects=[relationship]) - assert str(bundle) == EXPECTED_BUNDLE + assert bundle.serialize(pretty=True) == EXPECTED_BUNDLE def test_create_bundle_invalid(indicator, malware, relationship): diff --git a/stix2/test/v20/test_campaign.py b/stix2/test/v20/test_campaign.py index 0d4a202..13b47b0 100644 --- a/stix2/test/v20/test_campaign.py +++ b/stix2/test/v20/test_campaign.py @@ -21,7 +21,7 @@ EXPECTED = """{ def test_campaign_example(): campaign = stix2.v20.Campaign(**CAMPAIGN_MORE_KWARGS) - assert str(campaign) == EXPECTED + assert campaign.serialize(pretty=True) == EXPECTED @pytest.mark.parametrize( diff --git a/stix2/test/v20/test_course_of_action.py b/stix2/test/v20/test_course_of_action.py index f648907..c778316 100644 --- a/stix2/test/v20/test_course_of_action.py +++ b/stix2/test/v20/test_course_of_action.py @@ -28,7 +28,7 @@ def test_course_of_action_example(): description="This is how to add a filter rule to block inbound access to TCP port 80 to the existing UDP 1434 filter ...", ) - assert str(coa) == EXPECTED + assert coa.serialize(pretty=True) == EXPECTED @pytest.mark.parametrize( diff --git a/stix2/test/v20/test_custom.py b/stix2/test/v20/test_custom.py index f5e7024..b5d86b5 100644 --- a/stix2/test/v20/test_custom.py +++ b/stix2/test/v20/test_custom.py @@ -778,7 +778,7 @@ def test_custom_extension_with_list_and_dict_properties_observable_type(data): pass example = SomeCustomExtension(keys=[{'test123': 123, 'test345': 'aaaa'}]) - assert data == str(example) + assert data == example.serialize(pretty=True) def test_custom_extension_invalid_type_name(): @@ -992,7 +992,7 @@ def test_custom_object_nested_dictionary(data): dictionary={'key': {'key_b': 'value', 'key_a': 'value'}}, ) - assert data == str(example) + assert data == example.serialize(pretty=True) @stix2.v20.CustomObject( diff --git a/stix2/test/v20/test_external_reference.py b/stix2/test/v20/test_external_reference.py index 07cf42d..04821cd 100644 --- a/stix2/test/v20/test_external_reference.py +++ b/stix2/test/v20/test_external_reference.py @@ -26,7 +26,7 @@ def test_external_reference_veris(): url="https://github.com/vz-risk/VCDB/blob/master/data/json/0001AA7F-C601-424A-B2B8-BE6C9F5164E7.json", ) - assert str(ref) == VERIS + assert ref.serialize(pretty=True) == VERIS CAPEC = """{ @@ -41,7 +41,7 @@ def test_external_reference_capec(): external_id="CAPEC-550", ) - assert str(ref) == CAPEC + assert ref.serialize(pretty=True) == CAPEC assert re.match("ExternalReference\\(source_name=u?'capec', external_id=u?'CAPEC-550'\\)", repr(ref)) @@ -59,7 +59,7 @@ def test_external_reference_capec_url(): url="http://capec.mitre.org/data/definitions/550.html", ) - assert str(ref) == CAPEC_URL + assert ref.serialize(pretty=True) == CAPEC_URL THREAT_REPORT = """{ @@ -76,7 +76,7 @@ def test_external_reference_threat_report(): url="http://www.example.com/threat-report.pdf", ) - assert str(ref) == THREAT_REPORT + assert ref.serialize(pretty=True) == THREAT_REPORT BUGZILLA = """{ @@ -93,7 +93,7 @@ def test_external_reference_bugzilla(): url="https://www.example.com/bugs/1370", ) - assert str(ref) == BUGZILLA + assert ref.serialize(pretty=True) == BUGZILLA OFFLINE = """{ @@ -108,7 +108,7 @@ def test_external_reference_offline(): description="Threat report", ) - assert str(ref) == OFFLINE + assert ref.serialize(pretty=True) == OFFLINE assert re.match("ExternalReference\\(source_name=u?'ACME Threat Intel', description=u?'Threat report'\\)", repr(ref)) # Yikes! This works assert eval("stix2." + repr(ref)) == ref diff --git a/stix2/test/v20/test_identity.py b/stix2/test/v20/test_identity.py index 750c6f2..c62da46 100644 --- a/stix2/test/v20/test_identity.py +++ b/stix2/test/v20/test_identity.py @@ -26,7 +26,7 @@ def test_identity_example(): identity_class="individual", ) - assert str(identity) == EXPECTED + assert identity.serialize(pretty=True) == EXPECTED @pytest.mark.parametrize( diff --git a/stix2/test/v20/test_indicator.py b/stix2/test/v20/test_indicator.py index 6f3bfea..47f4812 100644 --- a/stix2/test/v20/test_indicator.py +++ b/stix2/test/v20/test_indicator.py @@ -48,7 +48,7 @@ def test_indicator_with_all_required_properties(): ) assert ind.revoked is False - assert str(ind) == EXPECTED_INDICATOR + assert ind.serialize(pretty=True) == EXPECTED_INDICATOR rep = re.sub(r"(\[|=| )u('|\"|\\\'|\\\")", r"\g<1>\g<2>", repr(ind)) assert rep == EXPECTED_INDICATOR_REPR diff --git a/stix2/test/v20/test_intrusion_set.py b/stix2/test/v20/test_intrusion_set.py index 69b336e..59a9905 100644 --- a/stix2/test/v20/test_intrusion_set.py +++ b/stix2/test/v20/test_intrusion_set.py @@ -38,7 +38,7 @@ def test_intrusion_set_example(): goals=["acquisition-theft", "harassment", "damage"], ) - assert str(intrusion_set) == EXPECTED + assert intrusion_set.serialize(pretty=True) == EXPECTED @pytest.mark.parametrize( diff --git a/stix2/test/v20/test_kill_chain_phases.py b/stix2/test/v20/test_kill_chain_phases.py index d150757..7f65f00 100644 --- a/stix2/test/v20/test_kill_chain_phases.py +++ b/stix2/test/v20/test_kill_chain_phases.py @@ -16,7 +16,7 @@ def test_lockheed_martin_cyber_kill_chain(): phase_name="reconnaissance", ) - assert str(recon) == LMCO_RECON + assert recon.serialize(pretty=True) == LMCO_RECON FOO_PRE_ATTACK = """{ @@ -31,7 +31,7 @@ def test_kill_chain_example(): phase_name="pre-attack", ) - assert str(preattack) == FOO_PRE_ATTACK + assert preattack.serialize(pretty=True) == FOO_PRE_ATTACK def test_kill_chain_required_properties(): diff --git a/stix2/test/v20/test_malware.py b/stix2/test/v20/test_malware.py index bd49007..974caf0 100644 --- a/stix2/test/v20/test_malware.py +++ b/stix2/test/v20/test_malware.py @@ -33,7 +33,7 @@ def test_malware_with_all_required_properties(): name="Cryptolocker", ) - assert str(mal) == EXPECTED_MALWARE + assert mal.serialize(pretty=True) == EXPECTED_MALWARE def test_malware_with_empty_optional_field(): @@ -49,7 +49,7 @@ def test_malware_with_empty_optional_field(): external_references=[], ) - assert str(mal) == EXPECTED_MALWARE + assert mal.serialize(pretty=True) == EXPECTED_MALWARE def test_malware_autogenerated_properties(malware): diff --git a/stix2/test/v20/test_markings.py b/stix2/test/v20/test_markings.py index 0753e86..f7d15fa 100644 --- a/stix2/test/v20/test_markings.py +++ b/stix2/test/v20/test_markings.py @@ -71,7 +71,7 @@ EXPECTED_CAMPAIGN_WITH_GRANULAR_MARKINGS = """{ def test_marking_def_example_with_tlp(): - assert str(TLP_WHITE) == EXPECTED_TLP_MARKING_DEFINITION + assert TLP_WHITE.serialize(pretty=True) == EXPECTED_TLP_MARKING_DEFINITION def test_marking_def_example_with_statement_positional_argument(): @@ -82,7 +82,7 @@ def test_marking_def_example_with_statement_positional_argument(): definition=stix2.v20.StatementMarking(statement="Copyright 2016, Example Corp"), ) - assert str(marking_definition) == EXPECTED_STATEMENT_MARKING_DEFINITION + assert marking_definition.serialize(pretty=True) == EXPECTED_STATEMENT_MARKING_DEFINITION def test_marking_def_example_with_kwargs_statement(): @@ -94,7 +94,7 @@ def test_marking_def_example_with_kwargs_statement(): definition=stix2.v20.StatementMarking(**kwargs), ) - assert str(marking_definition) == EXPECTED_STATEMENT_MARKING_DEFINITION + assert marking_definition.serialize(pretty=True) == EXPECTED_STATEMENT_MARKING_DEFINITION def test_marking_def_invalid_type(): @@ -118,7 +118,7 @@ def test_campaign_with_markings_example(): description="Campaign by Green Group against a series of targets in the financial services sector.", object_marking_refs=TLP_WHITE, ) - assert str(campaign) == EXPECTED_CAMPAIGN_WITH_OBJECT_MARKING + assert campaign.serialize(pretty=True) == EXPECTED_CAMPAIGN_WITH_OBJECT_MARKING def test_granular_example(): @@ -127,7 +127,7 @@ def test_granular_example(): selectors=["abc", "abc.[23]", "abc.def", "abc.[2].efg"], ) - assert str(granular_marking) == EXPECTED_GRANULAR_MARKING + assert granular_marking.serialize(pretty=True) == EXPECTED_GRANULAR_MARKING def test_granular_example_with_bad_selector(): @@ -159,7 +159,7 @@ def test_campaign_with_granular_markings_example(): ), ], ) - assert str(campaign) == EXPECTED_CAMPAIGN_WITH_GRANULAR_MARKINGS + assert campaign.serialize(pretty=True) == EXPECTED_CAMPAIGN_WITH_GRANULAR_MARKINGS @pytest.mark.parametrize( diff --git a/stix2/test/v20/test_observed_data.py b/stix2/test/v20/test_observed_data.py index bd60383..79ed6c4 100644 --- a/stix2/test/v20/test_observed_data.py +++ b/stix2/test/v20/test_observed_data.py @@ -47,7 +47,7 @@ def test_observed_data_example(): }, ) - assert str(observed_data) == EXPECTED + assert observed_data.serialize(pretty=True) == EXPECTED EXPECTED_WITH_REF = """{ @@ -97,7 +97,7 @@ def test_observed_data_example_with_refs(): }, ) - assert str(observed_data) == EXPECTED_WITH_REF + assert observed_data.serialize(pretty=True) == EXPECTED_WITH_REF def test_observed_data_example_with_bad_refs(): diff --git a/stix2/test/v20/test_relationship.py b/stix2/test/v20/test_relationship.py index a0fccf4..f8ad67e 100644 --- a/stix2/test/v20/test_relationship.py +++ b/stix2/test/v20/test_relationship.py @@ -32,7 +32,7 @@ def test_relationship_all_required_properties(): source_ref=INDICATOR_ID, target_ref=MALWARE_ID, ) - assert str(rel) == EXPECTED_RELATIONSHIP + assert rel.serialize(pretty=True) == EXPECTED_RELATIONSHIP def test_relationship_autogenerated_properties(relationship): diff --git a/stix2/test/v20/test_report.py b/stix2/test/v20/test_report.py index 53707ce..557a374 100644 --- a/stix2/test/v20/test_report.py +++ b/stix2/test/v20/test_report.py @@ -47,7 +47,7 @@ def test_report_example(): ], ) - assert str(report) == EXPECTED + assert report.serialize(pretty=True) == EXPECTED def test_report_example_objects_in_object_refs(): @@ -67,7 +67,7 @@ def test_report_example_objects_in_object_refs(): ], ) - assert str(report) == EXPECTED + assert report.serialize(pretty=True) == EXPECTED def test_report_example_objects_in_object_refs_with_bad_id(): diff --git a/stix2/test/v20/test_sighting.py b/stix2/test/v20/test_sighting.py index 6bad63c..35cd1f1 100644 --- a/stix2/test/v20/test_sighting.py +++ b/stix2/test/v20/test_sighting.py @@ -33,7 +33,7 @@ BAD_SIGHTING = """{ def test_sighting_all_required_properties(): now = dt.datetime(2016, 4, 6, 20, 6, 37, tzinfo=pytz.utc) - s = stix2.v20.Sighting( + sighting = stix2.v20.Sighting( type='sighting', id=SIGHTING_ID, created=now, @@ -41,7 +41,7 @@ def test_sighting_all_required_properties(): sighting_of_ref=INDICATOR_ID, where_sighted_refs=[IDENTITY_ID], ) - assert str(s) == EXPECTED_SIGHTING + assert sighting.serialize(pretty=True) == EXPECTED_SIGHTING def test_sighting_bad_where_sighted_refs(): diff --git a/stix2/test/v20/test_threat_actor.py b/stix2/test/v20/test_threat_actor.py index 854e77a..0214f9b 100644 --- a/stix2/test/v20/test_threat_actor.py +++ b/stix2/test/v20/test_threat_actor.py @@ -32,7 +32,7 @@ def test_threat_actor_example(): name="Evil Org", ) - assert str(threat_actor) == EXPECTED + assert threat_actor.serialize(pretty=True) == EXPECTED @pytest.mark.parametrize( diff --git a/stix2/test/v20/test_tool.py b/stix2/test/v20/test_tool.py index 435b85e..dcb94be 100644 --- a/stix2/test/v20/test_tool.py +++ b/stix2/test/v20/test_tool.py @@ -43,7 +43,7 @@ def test_tool_example(): name="VNC", ) - assert str(tool) == EXPECTED + assert tool.serialize(pretty=True) == EXPECTED @pytest.mark.parametrize( diff --git a/stix2/test/v20/test_vulnerability.py b/stix2/test/v20/test_vulnerability.py index 5a69d82..b87152f 100644 --- a/stix2/test/v20/test_vulnerability.py +++ b/stix2/test/v20/test_vulnerability.py @@ -36,7 +36,7 @@ def test_vulnerability_example(): ], ) - assert str(vulnerability) == EXPECTED + assert vulnerability.serialize(pretty=True) == EXPECTED @pytest.mark.parametrize( diff --git a/stix2/test/v21/test_attack_pattern.py b/stix2/test/v21/test_attack_pattern.py index 0beda64..96a2869 100644 --- a/stix2/test/v21/test_attack_pattern.py +++ b/stix2/test/v21/test_attack_pattern.py @@ -38,7 +38,7 @@ def test_attack_pattern_example(): description="...", ) - assert str(ap) == EXPECTED + assert ap.serialize(pretty=True) == EXPECTED @pytest.mark.parametrize( @@ -114,7 +114,7 @@ def test_less_precise_timestamps(): description="...", ) - assert str(ap) == EXPECTED + assert ap.serialize(pretty=True) == EXPECTED # TODO: Add other examples diff --git a/stix2/test/v21/test_bundle.py b/stix2/test/v21/test_bundle.py index 4e30c84..2eeaff4 100644 --- a/stix2/test/v21/test_bundle.py +++ b/stix2/test/v21/test_bundle.py @@ -126,7 +126,7 @@ def test_bundle_id_must_start_with_bundle(): def test_create_bundle1(indicator, malware, relationship): bundle = stix2.v21.Bundle(objects=[indicator, malware, relationship]) - assert str(bundle) == EXPECTED_BUNDLE + assert bundle.serialize(pretty=True) == EXPECTED_BUNDLE assert bundle.serialize(pretty=True) == EXPECTED_BUNDLE @@ -139,31 +139,31 @@ def test_create_bundle2(indicator, malware, relationship): def test_create_bundle_with_positional_args(indicator, malware, relationship): bundle = stix2.v21.Bundle(indicator, malware, relationship) - assert str(bundle) == EXPECTED_BUNDLE + assert bundle.serialize(pretty=True) == EXPECTED_BUNDLE def test_create_bundle_with_positional_listarg(indicator, malware, relationship): bundle = stix2.v21.Bundle([indicator, malware, relationship]) - assert str(bundle) == EXPECTED_BUNDLE + assert bundle.serialize(pretty=True) == EXPECTED_BUNDLE def test_create_bundle_with_listarg_and_positional_arg(indicator, malware, relationship): bundle = stix2.v21.Bundle([indicator, malware], relationship) - assert str(bundle) == EXPECTED_BUNDLE + assert bundle.serialize(pretty=True) == EXPECTED_BUNDLE def test_create_bundle_with_listarg_and_kwarg(indicator, malware, relationship): bundle = stix2.v21.Bundle([indicator, malware], objects=[relationship]) - assert str(bundle) == EXPECTED_BUNDLE + assert bundle.serialize(pretty=True) == EXPECTED_BUNDLE def test_create_bundle_with_arg_listarg_and_kwarg(indicator, malware, relationship): bundle = stix2.v21.Bundle([indicator], malware, objects=[relationship]) - assert str(bundle) == EXPECTED_BUNDLE + assert bundle.serialize(pretty=True) == EXPECTED_BUNDLE def test_create_bundle_invalid(indicator, malware, relationship): diff --git a/stix2/test/v21/test_campaign.py b/stix2/test/v21/test_campaign.py index 10f5d7b..edc7d77 100644 --- a/stix2/test/v21/test_campaign.py +++ b/stix2/test/v21/test_campaign.py @@ -24,7 +24,7 @@ def test_campaign_example(): **CAMPAIGN_MORE_KWARGS ) - assert str(campaign) == EXPECTED + assert campaign.serialize(pretty=True) == EXPECTED @pytest.mark.parametrize( diff --git a/stix2/test/v21/test_course_of_action.py b/stix2/test/v21/test_course_of_action.py index 5eea908..964a556 100644 --- a/stix2/test/v21/test_course_of_action.py +++ b/stix2/test/v21/test_course_of_action.py @@ -42,7 +42,7 @@ COA_WITH_REF_DICT = json.loads(COA_WITH_REF_JSON) ) def test_course_of_action_example(sdo_json, sdo_dict): coa = stix2.v21.CourseOfAction(**sdo_dict) - assert str(coa) == sdo_json + assert coa.serialize(pretty=True) == sdo_json @pytest.mark.parametrize( diff --git a/stix2/test/v21/test_custom.py b/stix2/test/v21/test_custom.py index 79a4d03..ac46d84 100644 --- a/stix2/test/v21/test_custom.py +++ b/stix2/test/v21/test_custom.py @@ -965,7 +965,7 @@ def test_custom_extension_with_list_and_dict_properties_observable_type(data): pass example = SomeCustomExtension(keys=[{'test123': 123, 'test345': 'aaaa'}]) - assert data == str(example) + assert data == example.serialize(pretty=True) def test_custom_extension_invalid_type_name(): @@ -1202,7 +1202,7 @@ def test_custom_object_nested_dictionary(data): dictionary={'key': {'key_b': 'value', 'key_a': 'value'}}, ) - assert data == str(example) + assert data == example.serialize(pretty=True) @stix2.v21.CustomObject( diff --git a/stix2/test/v21/test_external_reference.py b/stix2/test/v21/test_external_reference.py index f347191..9316857 100644 --- a/stix2/test/v21/test_external_reference.py +++ b/stix2/test/v21/test_external_reference.py @@ -26,7 +26,7 @@ def test_external_reference_veris(): url="https://github.com/vz-risk/VCDB/blob/master/data/json/0001AA7F-C601-424A-B2B8-BE6C9F5164E7.json", ) - assert str(ref) == VERIS + assert ref.serialize(pretty=True) == VERIS CAPEC = """{ @@ -41,7 +41,7 @@ def test_external_reference_capec(): external_id="CAPEC-550", ) - assert str(ref) == CAPEC + assert ref.serialize(pretty=True) == CAPEC assert re.match("ExternalReference\\(source_name=u?'capec', external_id=u?'CAPEC-550'\\)", repr(ref)) @@ -59,7 +59,7 @@ def test_external_reference_capec_url(): url="http://capec.mitre.org/data/definitions/550.html", ) - assert str(ref) == CAPEC_URL + assert ref.serialize(pretty=True) == CAPEC_URL THREAT_REPORT = """{ @@ -76,7 +76,7 @@ def test_external_reference_threat_report(): url="http://www.example.com/threat-report.pdf", ) - assert str(ref) == THREAT_REPORT + assert ref.serialize(pretty=True) == THREAT_REPORT BUGZILLA = """{ @@ -93,7 +93,7 @@ def test_external_reference_bugzilla(): url="https://www.example.com/bugs/1370", ) - assert str(ref) == BUGZILLA + assert ref.serialize(pretty=True) == BUGZILLA OFFLINE = """{ @@ -108,7 +108,7 @@ def test_external_reference_offline(): description="Threat report", ) - assert str(ref) == OFFLINE + assert ref.serialize(pretty=True) == OFFLINE assert re.match("ExternalReference\\(source_name=u?'ACME Threat Intel', description=u?'Threat report'\\)", repr(ref)) # Yikes! This works assert eval("stix2." + repr(ref)) == ref diff --git a/stix2/test/v21/test_grouping.py b/stix2/test/v21/test_grouping.py index a92a180..e495888 100644 --- a/stix2/test/v21/test_grouping.py +++ b/stix2/test/v21/test_grouping.py @@ -38,7 +38,7 @@ def test_grouping_with_all_required_properties(): ], ) - assert str(grp) == EXPECTED_GROUPING + assert grp.serialize(pretty=True) == EXPECTED_GROUPING def test_grouping_autogenerated_properties(grouping): diff --git a/stix2/test/v21/test_identity.py b/stix2/test/v21/test_identity.py index 9d17723..c235b4d 100644 --- a/stix2/test/v21/test_identity.py +++ b/stix2/test/v21/test_identity.py @@ -27,7 +27,7 @@ def test_identity_example(): identity_class="individual", ) - assert str(identity) == EXPECTED + assert identity.serialize(pretty=True) == EXPECTED @pytest.mark.parametrize( diff --git a/stix2/test/v21/test_incident.py b/stix2/test/v21/test_incident.py index 7ee07b8..27bc254 100644 --- a/stix2/test/v21/test_incident.py +++ b/stix2/test/v21/test_incident.py @@ -27,7 +27,7 @@ def test_incident_example(): description="Intrusion into enterprise network", ) - assert str(incident) == EXPECTED + assert incident.serialize(pretty=True) == EXPECTED @pytest.mark.parametrize( diff --git a/stix2/test/v21/test_indicator.py b/stix2/test/v21/test_indicator.py index 2b22418..c6220d7 100644 --- a/stix2/test/v21/test_indicator.py +++ b/stix2/test/v21/test_indicator.py @@ -50,7 +50,7 @@ def test_indicator_with_all_required_properties(): ) assert ind.revoked is False - assert str(ind) == EXPECTED_INDICATOR + assert ind.serialize(pretty=True) == EXPECTED_INDICATOR rep = re.sub(r"(\[|=| )u('|\"|\\\'|\\\")", r"\g<1>\g<2>", repr(ind)) assert rep == EXPECTED_INDICATOR_REPR diff --git a/stix2/test/v21/test_infrastructure.py b/stix2/test/v21/test_infrastructure.py index 3e9feb7..2beb6aa 100644 --- a/stix2/test/v21/test_infrastructure.py +++ b/stix2/test/v21/test_infrastructure.py @@ -28,7 +28,7 @@ def test_infrastructure_with_all_required_properties(): name="Poison Ivy C2", ) - assert str(infra) == EXPECTED_INFRASTRUCTURE + assert infra.serialize(pretty=True) == EXPECTED_INFRASTRUCTURE def test_infrastructure_autogenerated_properties(infrastructure): diff --git a/stix2/test/v21/test_intrusion_set.py b/stix2/test/v21/test_intrusion_set.py index 778eda7..2b2dc0c 100644 --- a/stix2/test/v21/test_intrusion_set.py +++ b/stix2/test/v21/test_intrusion_set.py @@ -39,7 +39,7 @@ def test_intrusion_set_example(): goals=["acquisition-theft", "harassment", "damage"], ) - assert str(intrusion_set) == EXPECTED + assert intrusion_set.serialize(pretty=True) == EXPECTED @pytest.mark.parametrize( diff --git a/stix2/test/v21/test_kill_chain_phases.py b/stix2/test/v21/test_kill_chain_phases.py index 0acc538..9e29e88 100644 --- a/stix2/test/v21/test_kill_chain_phases.py +++ b/stix2/test/v21/test_kill_chain_phases.py @@ -16,7 +16,7 @@ def test_lockheed_martin_cyber_kill_chain(): phase_name="reconnaissance", ) - assert str(recon) == LMCO_RECON + assert recon.serialize(pretty=True) == LMCO_RECON FOO_PRE_ATTACK = """{ @@ -31,7 +31,7 @@ def test_kill_chain_example(): phase_name="pre-attack", ) - assert str(preattack) == FOO_PRE_ATTACK + assert preattack.serialize(pretty=True) == FOO_PRE_ATTACK def test_kill_chain_required_properties(): diff --git a/stix2/test/v21/test_location.py b/stix2/test/v21/test_location.py index cf9c101..0fc5bcb 100644 --- a/stix2/test/v21/test_location.py +++ b/stix2/test/v21/test_location.py @@ -54,7 +54,7 @@ EXPECTED_LOCATION_2_REPR = "Location(" + " ".join( def test_location_with_some_required_properties(): now = dt.datetime(2016, 4, 6, 20, 3, 0, tzinfo=pytz.utc) - loc = stix2.v21.Location( + location = stix2.v21.Location( id=LOCATION_ID, created=now, modified=now, @@ -62,8 +62,8 @@ def test_location_with_some_required_properties(): longitude=2.3522, ) - assert str(loc) == EXPECTED_LOCATION_1 - rep = re.sub(r"(\[|=| )u('|\"|\\\'|\\\")", r"\g<1>\g<2>", repr(loc)) + assert location.serialize(pretty=True) == EXPECTED_LOCATION_1 + rep = re.sub(r"(\[|=| )u('|\"|\\\'|\\\")", r"\g<1>\g<2>", repr(location)) assert rep == EXPECTED_LOCATION_1_REPR diff --git a/stix2/test/v21/test_malware.py b/stix2/test/v21/test_malware.py index f111826..d71b273 100644 --- a/stix2/test/v21/test_malware.py +++ b/stix2/test/v21/test_malware.py @@ -23,7 +23,7 @@ EXPECTED_MALWARE = """{ def test_malware_with_all_required_properties(): now = dt.datetime(2016, 5, 12, 8, 17, 27, tzinfo=pytz.utc) - mal = stix2.v21.Malware( + malware = stix2.v21.Malware( type="malware", id=MALWARE_ID, created=now, @@ -32,7 +32,7 @@ def test_malware_with_all_required_properties(): is_family=False, ) - assert str(mal) == EXPECTED_MALWARE + assert malware.serialize(pretty=True) == EXPECTED_MALWARE def test_malware_autogenerated_properties(malware): diff --git a/stix2/test/v21/test_malware_analysis.py b/stix2/test/v21/test_malware_analysis.py index bf4bbe6..1390203 100644 --- a/stix2/test/v21/test_malware_analysis.py +++ b/stix2/test/v21/test_malware_analysis.py @@ -48,9 +48,9 @@ MALWARE_ANALYSIS_DICT = json.loads(MALWARE_ANALYSIS_JSON) def test_malware_analysis_example(): - ma = stix2.v21.MalwareAnalysis(**MALWARE_ANALYSIS_DICT) + malware_analysis = stix2.v21.MalwareAnalysis(**MALWARE_ANALYSIS_DICT) - assert str(ma) == MALWARE_ANALYSIS_JSON + assert malware_analysis.serialize(pretty=True) == MALWARE_ANALYSIS_JSON @pytest.mark.parametrize( diff --git a/stix2/test/v21/test_markings.py b/stix2/test/v21/test_markings.py index b0a0d09..d3fd11b 100644 --- a/stix2/test/v21/test_markings.py +++ b/stix2/test/v21/test_markings.py @@ -99,7 +99,7 @@ EXPECTED_CAMPAIGN_WITH_GRANULAR_LANG_MARKINGS = u"""{ def test_marking_def_example_with_tlp(): - assert str(TLP_WHITE) == EXPECTED_TLP_MARKING_DEFINITION + assert TLP_WHITE.serialize(pretty=True) == EXPECTED_TLP_MARKING_DEFINITION def test_marking_def_example_with_statement_positional_argument(): @@ -110,7 +110,7 @@ def test_marking_def_example_with_statement_positional_argument(): definition=stix2.StatementMarking(statement="Copyright 2016, Example Corp"), ) - assert str(marking_definition) == EXPECTED_STATEMENT_MARKING_DEFINITION + assert marking_definition.serialize(pretty=True) == EXPECTED_STATEMENT_MARKING_DEFINITION def test_marking_def_example_with_kwargs_statement(): @@ -122,7 +122,7 @@ def test_marking_def_example_with_kwargs_statement(): definition=stix2.StatementMarking(**kwargs), ) - assert str(marking_definition) == EXPECTED_STATEMENT_MARKING_DEFINITION + assert marking_definition.serialize(pretty=True) == EXPECTED_STATEMENT_MARKING_DEFINITION def test_marking_def_invalid_type(): @@ -145,7 +145,7 @@ def test_campaign_with_markings_example(): description="Campaign by Green Group against a series of targets in the financial services sector.", object_marking_refs=TLP_WHITE, ) - assert str(campaign) == EXPECTED_CAMPAIGN_WITH_OBJECT_MARKING + assert campaign.serialize(pretty=True) == EXPECTED_CAMPAIGN_WITH_OBJECT_MARKING def test_granular_example(): @@ -154,7 +154,7 @@ def test_granular_example(): selectors=["abc", "abc.[23]", "abc.def", "abc.[2].efg"], ) - assert str(granular_marking) == EXPECTED_GRANULAR_MARKING + assert granular_marking.serialize(pretty=True) == EXPECTED_GRANULAR_MARKING def test_granular_example_with_bad_selector(): @@ -185,7 +185,7 @@ def test_campaign_with_granular_markings_example(): ), ], ) - assert str(campaign) == EXPECTED_CAMPAIGN_WITH_GRANULAR_REF_MARKINGS + assert campaign.serialize(pretty=True) == EXPECTED_CAMPAIGN_WITH_GRANULAR_REF_MARKINGS @pytest.mark.parametrize( diff --git a/stix2/test/v21/test_note.py b/stix2/test/v21/test_note.py index 47a191e..2f20c8d 100644 --- a/stix2/test/v21/test_note.py +++ b/stix2/test/v21/test_note.py @@ -73,7 +73,7 @@ def test_note_with_required_properties(): ], ) - assert str(note) == EXPECTED_NOTE + assert note.serialize(pretty=True) == EXPECTED_NOTE rep = re.sub(r"(\[|=| )u('|\"|\\\'|\\\")", r"\g<1>\g<2>", repr(note)) assert rep == EXPECTED_OPINION_REPR diff --git a/stix2/test/v21/test_observed_data.py b/stix2/test/v21/test_observed_data.py index c1cb38e..ccf9a70 100644 --- a/stix2/test/v21/test_observed_data.py +++ b/stix2/test/v21/test_observed_data.py @@ -155,7 +155,7 @@ def test_observed_data_example_with_object_refs(): ], ) - assert str(observed_data) == EXPECTED_OBJECT_REFS + assert observed_data.serialize(pretty=True) == EXPECTED_OBJECT_REFS def test_observed_data_object_constraint(): diff --git a/stix2/test/v21/test_opinion.py b/stix2/test/v21/test_opinion.py index 9634d6e..21c96d7 100644 --- a/stix2/test/v21/test_opinion.py +++ b/stix2/test/v21/test_opinion.py @@ -56,7 +56,7 @@ def test_opinion_with_required_properties(): explanation=EXPLANATION, ) - assert str(opi) == EXPECTED_OPINION + assert opi.serialize(pretty=True) == EXPECTED_OPINION rep = re.sub(r"(\[|=| )u('|\"|\\\'|\\\")", r"\g<1>\g<2>", repr(opi)) assert rep == EXPECTED_OPINION_REPR diff --git a/stix2/test/v21/test_relationship.py b/stix2/test/v21/test_relationship.py index 386e24b..9f76d92 100644 --- a/stix2/test/v21/test_relationship.py +++ b/stix2/test/v21/test_relationship.py @@ -33,7 +33,7 @@ def test_relationship_all_required_properties(): source_ref=INDICATOR_ID, target_ref=MALWARE_ID, ) - assert str(rel) == EXPECTED_RELATIONSHIP + assert rel.serialize(pretty=True) == EXPECTED_RELATIONSHIP def test_relationship_autogenerated_properties(relationship): diff --git a/stix2/test/v21/test_report.py b/stix2/test/v21/test_report.py index e54e11d..87ac1c1 100644 --- a/stix2/test/v21/test_report.py +++ b/stix2/test/v21/test_report.py @@ -48,7 +48,7 @@ def test_report_example(): ], ) - assert str(report) == EXPECTED + assert report.serialize(pretty=True) == EXPECTED def test_report_example_objects_in_object_refs(): @@ -68,7 +68,7 @@ def test_report_example_objects_in_object_refs(): ], ) - assert str(report) == EXPECTED + assert report.serialize(pretty=True) == EXPECTED def test_report_example_objects_in_object_refs_with_bad_id(): diff --git a/stix2/test/v21/test_sighting.py b/stix2/test/v21/test_sighting.py index 0ef5faa..e997d62 100644 --- a/stix2/test/v21/test_sighting.py +++ b/stix2/test/v21/test_sighting.py @@ -38,7 +38,7 @@ BAD_SIGHTING = """{ def test_sighting_all_required_properties(): now = dt.datetime(2016, 4, 6, 20, 6, 37, tzinfo=pytz.utc) - s = stix2.v21.Sighting( + sighting = stix2.v21.Sighting( type='sighting', id=SIGHTING_ID, created=now, @@ -46,7 +46,7 @@ def test_sighting_all_required_properties(): sighting_of_ref=INDICATOR_ID, where_sighted_refs=[IDENTITY_ID, LOCATION_ID], ) - assert str(s) == EXPECTED_SIGHTING + assert sighting.serialize(pretty=True) == EXPECTED_SIGHTING def test_sighting_bad_where_sighted_refs(): diff --git a/stix2/test/v21/test_threat_actor.py b/stix2/test/v21/test_threat_actor.py index 6a782ef..94dea8e 100644 --- a/stix2/test/v21/test_threat_actor.py +++ b/stix2/test/v21/test_threat_actor.py @@ -34,7 +34,7 @@ def test_threat_actor_example(): threat_actor_types=["crime-syndicate"], ) - assert str(threat_actor) == EXPECTED + assert threat_actor.serialize(pretty=True) == EXPECTED @pytest.mark.parametrize( diff --git a/stix2/test/v21/test_tool.py b/stix2/test/v21/test_tool.py index 6a7d89c..427f8de 100644 --- a/stix2/test/v21/test_tool.py +++ b/stix2/test/v21/test_tool.py @@ -45,7 +45,7 @@ def test_tool_example(): tool_types=["remote-access"], ) - assert str(tool) == EXPECTED + assert tool.serialize(pretty=True) == EXPECTED @pytest.mark.parametrize( diff --git a/stix2/test/v21/test_vulnerability.py b/stix2/test/v21/test_vulnerability.py index ee63a0e..bcea913 100644 --- a/stix2/test/v21/test_vulnerability.py +++ b/stix2/test/v21/test_vulnerability.py @@ -37,7 +37,7 @@ def test_vulnerability_example(): ], ) - assert str(vulnerability) == EXPECTED + assert vulnerability.serialize(pretty=True) == EXPECTED @pytest.mark.parametrize( From dd6b847b10eed8604bed7c9d690ef47c86ff7098 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Wed, 31 Mar 2021 12:44:15 -0400 Subject: [PATCH 28/56] small tweaks and pre-commit changes --- stix2/base.py | 3 +-- stix2/properties.py | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/stix2/base.py b/stix2/base.py index 8a5a517..6f29882 100644 --- a/stix2/base.py +++ b/stix2/base.py @@ -22,7 +22,6 @@ from .utils import NOW, PREFIX_21_REGEX, get_timestamp from .versioning import new_version as _new_version from .versioning import revoke as _revoke - DEFAULT_ERROR = "{type} must have {property}='{expected}'." SCO_DET_ID_NAMESPACE = uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7") @@ -75,7 +74,7 @@ class _STIXBase(collections.abc.Mapping): def _check_at_least_one_property(self, list_of_properties=None): if not list_of_properties: - list_of_properties = sorted(list(self.__class__._properties.keys())) + list_of_properties = sorted(self.__class__._properties.keys()) if isinstance(self, _Observable): props_to_remove = {"type", "id", "defanged", "spec_version"} else: diff --git a/stix2/properties.py b/stix2/properties.py index 8fb8a75..1ec1311 100644 --- a/stix2/properties.py +++ b/stix2/properties.py @@ -17,7 +17,6 @@ from .exceptions import ( from .parsing import parse, parse_observable from .utils import _get_dict, get_class_hierarchy_names, parse_into_datetime - TYPE_REGEX = re.compile(r'^-?[a-z0-9]+(-[a-z0-9]+)*-?$') TYPE_21_REGEX = re.compile(r'^([a-z][a-z0-9]*)+([a-z0-9-]+)*-?$') ERROR_INVALID_ID = ( From 294bd7309e3b338d5c69f4adfc5597bbcb56c9bc Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Wed, 31 Mar 2021 13:56:02 -0400 Subject: [PATCH 29/56] add .serialize(pretty=True) around docs/guide/ to accommodate this change --- docs/contributing.rst | 2 +- docs/guide/creating.ipynb | 14 +++++++------- docs/guide/custom.ipynb | 18 +++++++++--------- docs/guide/datastore.ipynb | 6 +++--- docs/guide/environment.ipynb | 10 +++++----- docs/guide/filesystem.ipynb | 2 +- docs/guide/markings.ipynb | 12 ++++++------ docs/guide/memory.ipynb | 8 ++++---- docs/guide/parsing.ipynb | 6 +++--- docs/guide/serializing.ipynb | 18 +++++++++--------- docs/guide/taxii.ipynb | 8 ++++---- docs/guide/ts_support.ipynb | 2 +- docs/guide/versioning.ipynb | 6 +++--- docs/guide/workbench.ipynb | 4 ++-- 14 files changed, 58 insertions(+), 58 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 20cab3a..a904c31 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -27,7 +27,7 @@ the repository on GitHub and clone your fork instead of the main repo: git clone https://github.com/yourusername/cti-python-stix2.git -2. Install develoment-related dependencies: +2. Install development-related dependencies: .. prompt:: bash diff --git a/docs/guide/creating.ipynb b/docs/guide/creating.ipynb index 4919452..d390875 100644 --- a/docs/guide/creating.ipynb +++ b/docs/guide/creating.ipynb @@ -173,7 +173,7 @@ "indicator = Indicator(name=\"File hash for malware variant\",\n", " pattern=\"[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']\",\n", " pattern_type=\"stix\")\n", - "print(indicator)" + "print(indicator.serialize(pretty=True))" ] }, { @@ -503,7 +503,7 @@ "\n", "malware = Malware(name=\"Poison Ivy\",\n", " is_family=False)\n", - "print(malware)" + "print(malware.serialize(pretty=True))" ] }, { @@ -627,7 +627,7 @@ "relationship = Relationship(relationship_type='indicates',\n", " source_ref=indicator.id,\n", " target_ref=malware.id)\n", - "print(relationship)" + "print(relationship.serialize(pretty=True))" ] }, { @@ -736,7 +736,7 @@ ], "source": [ "relationship2 = Relationship(indicator, 'indicates', malware)\n", - "print(relationship2)" + "print(relationship2.serialize(pretty=True))" ] }, { @@ -876,7 +876,7 @@ "from stix2 import Bundle\n", "\n", "bundle = Bundle(indicator, malware, relationship)\n", - "print(bundle)" + "print(bundle.serialize(pretty=True))" ] }, { @@ -993,7 +993,7 @@ " resolves_to_refs=[\"mac-addr--43f380fd-37c6-476d-8643-60849bf9240e\"]\n", ")\n", "\n", - "print(ip4)" + "print(ip4.serialize(pretty=True))" ] }, { @@ -1111,7 +1111,7 @@ " resolves_to_refs=[mac_addr_a.id, mac_addr_b.id]\n", ")\n", "\n", - "print(ip4_valid_refs)" + "print(ip4_valid_refs.serialize(pretty=True))" ] } ], diff --git a/docs/guide/custom.ipynb b/docs/guide/custom.ipynb index 8a4ab10..793702f 100644 --- a/docs/guide/custom.ipynb +++ b/docs/guide/custom.ipynb @@ -201,7 +201,7 @@ " custom_properties={\n", " \"x_foo\": \"bar\"\n", " })\n", - "print(identity)" + "print(identity.serialize(pretty=True))" ] }, { @@ -313,7 +313,7 @@ " identity_class=\"individual\",\n", " x_foo=\"bar\",\n", " allow_custom=True)\n", - "print(identity2)" + "print(identity2.serialize(pretty=True))" ] }, { @@ -533,7 +533,7 @@ ], "source": [ "identity4 = identity3.new_version(x_foo=None)\n", - "print(identity4)" + "print(identity4.serialize(pretty=True))" ] }, { @@ -671,7 +671,7 @@ "source": [ "animal = Animal(species=\"lion\",\n", " animal_class=\"mammal\")\n", - "print(animal)" + "print(animal.serialize(pretty=True))" ] }, { @@ -956,7 +956,7 @@ "\n", "new_observable = NewObservable(a_property=\"something\",\n", " property_2=10)\n", - "print(new_observable)" + "print(new_observable.serialize(pretty=True))" ] }, { @@ -1458,13 +1458,13 @@ " pass\n", "\n", "new_observable_a = NewObservable2(a_property=\"A property\", property_2=2000)\n", - "print(new_observable_a)\n", + "print(new_observable_a.serialize(pretty=True))\n", "\n", "new_observable_b = NewObservable2(a_property=\"A property\", property_2=3000)\n", - "print(new_observable_b)\n", + "print(new_observable_b.serialize(pretty=True))\n", "\n", "new_observable_c = NewObservable2(a_property=\"A different property\", property_2=3000)\n", - "print(new_observable_c)" + "print(new_observable_c.serialize(pretty=True))" ] }, { @@ -1588,7 +1588,7 @@ "\n", "new_ext = NewExtension(property1=\"something\",\n", " property2=10)\n", - "print(new_ext)" + "print(new_ext.serialize(pretty=True))" ] }, { diff --git a/docs/guide/datastore.ipynb b/docs/guide/datastore.ipynb index 9f8e310..6cad628 100644 --- a/docs/guide/datastore.ipynb +++ b/docs/guide/datastore.ipynb @@ -332,11 +332,11 @@ "\n", "# get an object that is only in the filesystem\n", "intrusion_set = cs.get('intrusion-set--f3bdec95-3d62-42d9-a840-29630f6cdc1a')\n", - "print(intrusion_set)\n", + "print(intrusion_set.serialize(pretty=True))\n", "\n", "# get an object that is only in the TAXII collection\n", "ind = cs.get('indicator--a740531e-63ff-4e49-a9e1-a0a3eed0e3e7')\n", - "print(ind)" + "print(ind.serialize(pretty=True))" ] }, { @@ -593,7 +593,7 @@ } ], "source": [ - "print(mem.creator_of(mal))" + "print(mem.creator_of(mal).serialize(pretty=True))" ] }, { diff --git a/docs/guide/environment.ipynb b/docs/guide/environment.ipynb index a3417b5..deb8e71 100644 --- a/docs/guide/environment.ipynb +++ b/docs/guide/environment.ipynb @@ -225,7 +225,7 @@ } ], "source": [ - "print(env.get(\"indicator--a740531e-63ff-4e49-a9e1-a0a3eed0e3e7\"))" + "print(env.get(\"indicator--a740531e-63ff-4e49-a9e1-a0a3eed0e3e7\").serialize(pretty=True))" ] }, { @@ -360,7 +360,7 @@ "ind = factory.create(Indicator,\n", " pattern_type=\"stix\",\n", " pattern=\"[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']\")\n", - "print(ind)" + "print(ind.serialize(pretty=True))" ] }, { @@ -486,7 +486,7 @@ " created_by_ref=None,\n", " pattern_type=\"stix\",\n", " pattern=\"[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']\")\n", - "print(ind2)" + "print(ind2.serialize(pretty=True))" ] }, { @@ -593,7 +593,7 @@ " created_by_ref=\"identity--962cabe5-f7f3-438a-9169-585a8c971d12\",\n", " pattern_type=\"stix\",\n", " pattern=\"[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']\")\n", - "print(ind3)" + "print(ind3.serialize(pretty=True))" ] }, { @@ -712,7 +712,7 @@ " pattern_type=\"stix\",\n", " pattern=\"[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']\")\n", "environ.add(i)\n", - "print(environ.get(i.id))" + "print(environ.get(i.id).serialize(pretty=True))" ] } ], diff --git a/docs/guide/filesystem.ipynb b/docs/guide/filesystem.ipynb index ec09422..a4aa4d0 100644 --- a/docs/guide/filesystem.ipynb +++ b/docs/guide/filesystem.ipynb @@ -262,7 +262,7 @@ "mal = fs.get(\"malware--92ec0cbd-2c30-44a2-b270-73f4ec949841\")\n", "\n", "# for visual purposes\n", - "print(mal)" + "print(mal.serialize(pretty=True))" ] }, { diff --git a/docs/guide/markings.ipynb b/docs/guide/markings.ipynb index 1e1609c..942f629 100644 --- a/docs/guide/markings.ipynb +++ b/docs/guide/markings.ipynb @@ -176,7 +176,7 @@ "indicator = Indicator(pattern_type=\"stix\",\n", " pattern=\"[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']\",\n", " object_marking_refs=TLP_AMBER)\n", - "print(indicator)" + "print(indicator.serialize(pretty=True))" ] }, { @@ -290,7 +290,7 @@ " definition_type=\"statement\", \n", " definition=StatementMarking(statement=\"Copyright 2017, Example Corp\")\n", ")\n", - "print(marking_definition)" + "print(marking_definition.serialize(pretty=True))" ] }, { @@ -405,7 +405,7 @@ "indicator2 = Indicator(pattern_type=\"stix\",\n", " pattern=\"[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']\",\n", " object_marking_refs=marking_definition)\n", - "print(indicator2)" + "print(indicator2.serialize(pretty=True))" ] }, { @@ -513,7 +513,7 @@ "indicator3 = Indicator(pattern_type=\"stix\",\n", " pattern=\"[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']\",\n", " object_marking_refs=\"marking-definition--f88d31f6-486f-44da-b317-01333bde0b82\")\n", - "print(indicator3)" + "print(indicator3.serialize(pretty=True))" ] }, { @@ -650,7 +650,7 @@ " \"marking_ref\": TLP_WHITE\n", " }\n", " ])\n", - "print(malware)" + "print(malware.serialize(pretty=True))" ] }, { @@ -808,7 +808,7 @@ ], "source": [ "indicator4 = indicator.add_markings(marking_definition)\n", - "print(indicator4)" + "print(indicator4.serialize(pretty=True))" ] }, { diff --git a/docs/guide/memory.ipynb b/docs/guide/memory.ipynb index d8a7000..cf9f035 100644 --- a/docs/guide/memory.ipynb +++ b/docs/guide/memory.ipynb @@ -187,7 +187,7 @@ "mem.add(ind)\n", "\n", "# for visual purposes\n", - "print(mem.get(ind.id))\n" + "print(mem.get(ind.id).serialize(pretty=True))\n" ] }, { @@ -304,7 +304,7 @@ "mem.add([ind2,ind3, mal])\n", "\n", "# for visual purposes\n", - "print(mem.get(ind3.id))" + "print(mem.get(ind3.id).serialize(pretty=True))" ] }, { @@ -412,7 +412,7 @@ "from stix2 import Filter\n", "\n", "mal = mem.query([Filter(\"malware_types\",\"=\", \"rootkit\")])[0]\n", - "print(mal)" + "print(mal.serialize(pretty=True))" ] }, { @@ -533,7 +533,7 @@ "report = mem_2.get(\"malware--6cee28b8-4d42-4e72-bd77-ea47897672c0\")\n", "\n", "# for visual purposes\n", - "print(report)" + "print(report.serialize(pretty=True))" ] } ], diff --git a/docs/guide/parsing.ipynb b/docs/guide/parsing.ipynb index 5fd9499..0e89126 100644 --- a/docs/guide/parsing.ipynb +++ b/docs/guide/parsing.ipynb @@ -282,7 +282,7 @@ "\n", "obj = parse(input_string)\n", "print(type(obj))\n", - "print(obj)" + "print(obj.serialize(pretty=True))" ] }, { @@ -483,7 +483,7 @@ "\n", "obj = parse(input_dict)\n", "print(type(obj))\n", - "print(obj)" + "print(obj.serialize(pretty=True))" ] }, { @@ -677,7 +677,7 @@ "\n", "obj = parse(file_handle)\n", "print(type(obj))\n", - "print(obj)" + "print(obj.serialize(pretty=True))" ] }, { diff --git a/docs/guide/serializing.ipynb b/docs/guide/serializing.ipynb index 9e7e268..fd91c60 100644 --- a/docs/guide/serializing.ipynb +++ b/docs/guide/serializing.ipynb @@ -173,14 +173,21 @@ " pattern_type=\"stix\",\n", " pattern=\"[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']\")\n", "\n", - "print(str(indicator))" + "print(indicator.serialize(pretty=True))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "However, the string representation can be slow, as it sorts properties to be in a more readable order. If you need performance and don't care about the human-readability of the output, use the object's `serialize()` function:" + "---\n", + "**New in 3.0.0:** \n", + "\n", + "Calling `str()` on a STIX object will call `serialize()` without any formatting options. The change was made to address the performance penalty induced by unknowingly calling with the pretty formatted option. As shown above, to get effect `str()` would in the past versions of the library use the method directly and pass the pretty argument directly `serialize(pretty=True)`.\n", + "\n", + "---\n", + "\n", + "However, the pretty formatted string representation can be slow, as it sorts properties to be in a more readable order. If you need performance and don't care about the human-readability of the output, use the object's `serialize()` function to pass in any arguments `json.dump()` would understand:" ] }, { @@ -384,13 +391,6 @@ "source": [ "print(indicator.serialize(indent=4))" ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The only difference between this and the string representation from using `str()` is that this will not sort the keys. This works because the keyword arguments are passed to `json.dumps()` internally." - ] } ], "metadata": { diff --git a/docs/guide/taxii.ipynb b/docs/guide/taxii.ipynb index 2ec5c24..c36962e 100644 --- a/docs/guide/taxii.ipynb +++ b/docs/guide/taxii.ipynb @@ -557,10 +557,10 @@ "stix_obj_versions = tc_source.all_versions(\"indicator--6770298f-0fd8-471a-ab8c-1c658a46574e\")\n", "\n", "#for visual purposes\n", - "print(stix_obj)\n", + "print(stix_obj.serialize(pretty=True))\n", "print(\"-------\")\n", "for so in stix_obj_versions:\n", - " print(so)\n" + " print(so.serialize(pretty=True))\n" ] }, { @@ -959,7 +959,7 @@ "\n", "#for visual purposes\n", "for indicator in indicators:\n", - " print(indicator)" + " print(indicator.serialize(pretty=True))" ] }, { @@ -1113,7 +1113,7 @@ "# TAXIICollectionStore\n", "stix_obj2 = tc_source.get(\"malware--c0931cc6-c75e-47e5-9036-78fabc95d4ec\")\n", "\n", - "print(stix_obj2)" + "print(stix_obj2.serialize(pretty=True))" ] }, { diff --git a/docs/guide/ts_support.ipynb b/docs/guide/ts_support.ipynb index 99ef893..8c07134 100644 --- a/docs/guide/ts_support.ipynb +++ b/docs/guide/ts_support.ipynb @@ -356,7 +356,7 @@ " \"pattern\": \"[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']\",\n", " \"valid_from\": \"2017-09-26T23:33:39.829952Z\"\n", "}\"\"\", version=\"2.0\")\n", - "print(indicator)" + "print(indicator.serialize(pretty=True))" ] }, { diff --git a/docs/guide/versioning.ipynb b/docs/guide/versioning.ipynb index 45686bf..62d743f 100644 --- a/docs/guide/versioning.ipynb +++ b/docs/guide/versioning.ipynb @@ -185,7 +185,7 @@ "\n", "indicator2 = indicator.new_version(name=\"File hash for Foobar malware\",\n", " labels=[\"malicious-activity\"])\n", - "print(indicator2)" + "print(indicator2.serialize(pretty=True))" ] }, { @@ -326,7 +326,7 @@ ], "source": [ "indicator3 = indicator.new_version(description=None)\n", - "print(indicator3)" + "print(indicator3.serialize(pretty=True))" ] }, { @@ -443,7 +443,7 @@ ], "source": [ "indicator4 = indicator3.revoke()\n", - "print(indicator4)" + "print(indicator4.serialize(pretty=True))" ] } ], diff --git a/docs/guide/workbench.ipynb b/docs/guide/workbench.ipynb index c0c4ee0..def48b8 100644 --- a/docs/guide/workbench.ipynb +++ b/docs/guide/workbench.ipynb @@ -506,7 +506,7 @@ "source": [ "for i in indicators():\n", " for obj in i.related():\n", - " print(obj)" + " print(obj.serialize(pretty=True))" ] }, { @@ -621,7 +621,7 @@ "source": [ "malware = get('malware--c0931cc6-c75e-47e5-9036-78fabc95d4ec')\n", "indicator = malware.related(filters=Filter('type', '=', 'indicator'))\n", - "print(indicator[0])" + "print(indicator[0].serialize(pretty=True))" ] }, { From 35ef718b70e62ff1cb769c23013a8c728d14b000 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Wed, 31 Mar 2021 14:58:28 -0400 Subject: [PATCH 30/56] add extension-definition ids (fail to validate) but keeping in Git history --- stix2/test/v21/constants.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/stix2/test/v21/constants.py b/stix2/test/v21/constants.py index 8084ea2..b7d0bda 100644 --- a/stix2/test/v21/constants.py +++ b/stix2/test/v21/constants.py @@ -27,6 +27,12 @@ TOOL_ID = "tool--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f" SIGHTING_ID = "sighting--bfbc19db-ec35-4e45-beed-f8bde2a772fb" VULNERABILITY_ID = "vulnerability--0c7b5b88-8ff7-4a4d-aa9d-feb398cd0061" +EXTENSION_DEFINITION_IDS = [ + "extension-definition--a932fcc6-e032-176c-126f-cb970a5a1fff", # new-sdo + "extension-definition--d83fce45-ef58-4c6c-a3f4-1fbc32e98c6e", # property-extension + "extension-definition--71736db5-10db-43d3-b0e3-65cf81601fe1", # top-level-property-extension + "extension-definition--a932fcc6-e032-176c-126f-cb970a5a1fff", # new-sdo, new-sco, property-extension +] MARKING_IDS = [ "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9", "marking-definition--443eb5c3-a76c-4a0a-8caa-e93998e7bc09", From 203a2746e0d03f72dd04029063a519416acc1591 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Wed, 31 Mar 2021 15:00:54 -0400 Subject: [PATCH 31/56] add new file test_extension_definition.py and assigned new UUIDs to the constants --- stix2/test/v21/constants.py | 8 +- stix2/test/v21/test_extension_definition.py | 107 ++++++++++++++++++++ 2 files changed, 111 insertions(+), 4 deletions(-) create mode 100644 stix2/test/v21/test_extension_definition.py diff --git a/stix2/test/v21/constants.py b/stix2/test/v21/constants.py index b7d0bda..528c560 100644 --- a/stix2/test/v21/constants.py +++ b/stix2/test/v21/constants.py @@ -28,10 +28,10 @@ SIGHTING_ID = "sighting--bfbc19db-ec35-4e45-beed-f8bde2a772fb" VULNERABILITY_ID = "vulnerability--0c7b5b88-8ff7-4a4d-aa9d-feb398cd0061" EXTENSION_DEFINITION_IDS = [ - "extension-definition--a932fcc6-e032-176c-126f-cb970a5a1fff", # new-sdo - "extension-definition--d83fce45-ef58-4c6c-a3f4-1fbc32e98c6e", # property-extension - "extension-definition--71736db5-10db-43d3-b0e3-65cf81601fe1", # top-level-property-extension - "extension-definition--a932fcc6-e032-176c-126f-cb970a5a1fff", # new-sdo, new-sco, property-extension + "extension-definition--1f611280-fbe1-48e8-92ab-ff47ce02d5b7", # new-sdo + "extension-definition--368f4787-5b43-467c-9693-0c9de4289c4b", # property-extension + "extension-definition--dd73de4f-a7f3-49ea-8ec1-8e884196b7a8", # top-level-property-extension + "extension-definition--150c1738-28c9-44d0-802d-70523218240b", # new-sdo, new-sco, property-extension ] MARKING_IDS = [ "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9", diff --git a/stix2/test/v21/test_extension_definition.py b/stix2/test/v21/test_extension_definition.py new file mode 100644 index 0000000..7aac866 --- /dev/null +++ b/stix2/test/v21/test_extension_definition.py @@ -0,0 +1,107 @@ +import datetime as dt + +import pytest +import pytz + +import stix2 + +from .constants import EXTENSION_DEFINITION_IDS + +EXPECTED = f"""{{ + "type": "extension-definition", + "spec_version": "2.1", + "id": "{EXTENSION_DEFINITION_IDS[0]}", + "created_by_ref": "identity--11b76a96-5d2b-45e0-8a5a-f6994f370731", + "created": "2014-02-20T09:16:08.000Z", + "modified": "2014-02-20T09:16:08.000Z", + "name": "New SDO 1", + "description": "This schema creates a new object type called my-favorite-sdo-1", + "schema": "https://www.example.com/schema-my-favorite-sdo-1/v1/", + "version": "1.2.1", + "extension_types": [ + "new-sdo" + ] +}}""" + + +def test_extension_definition_example(): + extension_definition = stix2.v21.ExtensionDefinition( + id=EXTENSION_DEFINITION_IDS[0], + created_by_ref="identity--11b76a96-5d2b-45e0-8a5a-f6994f370731", + created="2014-02-20T09:16:08.000Z", + modified="2014-02-20T09:16:08.000Z", + name="New SDO 1", + description="This schema creates a new object type called my-favorite-sdo-1", + schema="https://www.example.com/schema-my-favorite-sdo-1/v1/", + version="1.2.1", + extension_types=["new-sdo"], + ) + + assert extension_definition.serialize(pretty=True) == EXPECTED + + +@pytest.mark.parametrize( + "data", [ + EXPECTED, + { + "id": f"{EXTENSION_DEFINITION_IDS[0]}", + "type": "extension-definition", + "spec_version": "2.1", + "created_by_ref": "identity--11b76a96-5d2b-45e0-8a5a-f6994f370731", + "created": "2014-02-20T09:16:08.000Z", + "modified": "2014-02-20T09:16:08.000Z", + "name": "New SDO 1", + "description": "This schema creates a new object type called my-favorite-sdo-1", + "schema": "https://www.example.com/schema-my-favorite-sdo-1/v1/", + "version": "1.2.1", + "extension_types": ["new-sdo"] + }, + ], +) +def test_parse_extension_definition(data): + extension_definition = stix2.parse(data, version="2.1") + + assert extension_definition.type == 'extension-definition' + assert extension_definition.spec_version == '2.1' + assert extension_definition.id == EXTENSION_DEFINITION_IDS[0] + assert extension_definition.created == dt.datetime(2014, 2, 20, 9, 16, 8, tzinfo=pytz.utc) + assert extension_definition.modified == dt.datetime(2014, 2, 20, 9, 16, 8, tzinfo=pytz.utc) + assert extension_definition.name == 'New SDO 1' + assert extension_definition.description == 'This schema creates a new object type called my-favorite-sdo-1' + assert extension_definition.schema == 'https://www.example.com/schema-my-favorite-sdo-1/v1/' + assert extension_definition.version == '1.2.1' + assert extension_definition.extension_types == ['new-sdo'] + + +def test_parse_no_type(): + with pytest.raises(stix2.exceptions.ParseError): + stix2.parse("""{ + "id": "{EXTENSION_DEFINITION_IDS[0]}", + "spec_version": "2.1", + "name": "New SDO 1", + "description": "This schema creates a new object type called my-favorite-sdo-1", + "created": "2014-02-20T09:16:08.989000Z", + "modified": "2014-02-20T09:16:08.989000Z", + "created_by_ref": "identity--11b76a96-5d2b-45e0-8a5a-f6994f370731", + "schema": "https://www.example.com/schema-my-favorite-sdo-1/v1/", + "version": "1.2.1", + "extension_types": [ "new-sdo" ] + }""", version="2.1", + ) + + +def test_extension_definition_with_custom(): + extension_definition = stix2.v21.ExtensionDefinition( + created_by_ref="identity--11b76a96-5d2b-45e0-8a5a-f6994f370731", + created="2014-02-20T09:16:08.000Z", + modified="2014-02-20T09:16:08.000Z", + name="New SDO 1", + description="This schema creates a new object type called my-favorite-sdo-1", + schema="https://www.example.com/schema-my-favorite-sdo-1/v1/", + version="1.2.1", + extension_types=["new-sdo"], + custom_properties={'x_foo': 'bar'}, + ) + + assert extension_definition.x_foo == "bar" + assert "x_foo" in extension_definition.object_properties() From 08d5781f549f476e0fe3e574759d583144410a10 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Wed, 31 Mar 2021 18:11:38 -0400 Subject: [PATCH 32/56] add entry on extensions.ipynb, typos, and a couple of fixes for registry module and Marking decorator --- docs/guide/extensions.ipynb | 784 ++++++++++++++++++++ docs/overview.rst | 2 +- stix2/registry.py | 9 +- stix2/test/v21/test_extension_definition.py | 5 +- stix2/v21/common.py | 1 + 5 files changed, 795 insertions(+), 6 deletions(-) create mode 100644 docs/guide/extensions.ipynb diff --git a/docs/guide/extensions.ipynb b/docs/guide/extensions.ipynb new file mode 100644 index 0000000..0283981 --- /dev/null +++ b/docs/guide/extensions.ipynb @@ -0,0 +1,784 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "nbsphinx": "hidden" + }, + "outputs": [], + "source": [ + "# Delete this cell to re-enable tracebacks\n", + "import sys\n", + "ipython = get_ipython()\n", + "\n", + "def hide_traceback(exc_tuple=None, filename=None, tb_offset=None,\n", + " exception_only=False, running_compiled_code=False):\n", + " etype, value, tb = sys.exc_info()\n", + " value.__cause__ = None # suppress chained exceptions\n", + " return ipython._showtraceback(etype, value, ipython.InteractiveTB.get_exception_only(etype, value))\n", + "\n", + "ipython.showtraceback = hide_traceback" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "nbsphinx": "hidden" + }, + "outputs": [], + "source": [ + "# JSON output syntax highlighting\n", + "from __future__ import print_function\n", + "from pygments import highlight\n", + "from pygments.lexers import JsonLexer, TextLexer\n", + "from pygments.formatters import HtmlFormatter\n", + "from IPython.display import display, HTML\n", + "from IPython.core.interactiveshell import InteractiveShell\n", + "\n", + "InteractiveShell.ast_node_interactivity = \"all\"\n", + "\n", + "def json_print(inpt):\n", + " string = str(inpt)\n", + " formatter = HtmlFormatter()\n", + " if string[0] == '{':\n", + " lexer = JsonLexer()\n", + " else:\n", + " lexer = TextLexer()\n", + " return HTML('{}'.format(\n", + " formatter.get_style_defs('.highlight'),\n", + " highlight(string, lexer, formatter)))\n", + "\n", + "globals()['print'] = json_print" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "STIX Extensions\n", + "\n", + "This page is specific for the STIX Extensions mechanism defined in STIX 2.1 CS 02, for the deprecated STIX Customization mechanisms see the [Custom](custom.ipynb) section.\n", + "\n", + "The example below shows how to create an `indicator` object with a `top-level-property-extension`. " + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
{\n",
+       "    "type": "extension-definition",\n",
+       "    "spec_version": "2.1",\n",
+       "    "id": "extension-definition--dd73de4f-a7f3-49ea-8ec1-8e884196b7a8",\n",
+       "    "created_by_ref": "identity--11b76a96-5d2b-45e0-8a5a-f6994f370731",\n",
+       "    "created": "2014-02-20T09:16:08.000Z",\n",
+       "    "modified": "2014-02-20T09:16:08.000Z",\n",
+       "    "name": "New SDO 1",\n",
+       "    "description": "This schema adds two properties to a STIX object at the toplevel",\n",
+       "    "schema": "https://www.example.com/schema-foo-1a/v1/",\n",
+       "    "version": "1.2.1",\n",
+       "    "extension_types": [\n",
+       "        "toplevel-property-extension"\n",
+       "    ],\n",
+       "    "extension_properties": [\n",
+       "        "toxicity",\n",
+       "        "rank"\n",
+       "    ]\n",
+       "}\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "text/html": [ + "
{\n",
+       "    "type": "indicator",\n",
+       "    "spec_version": "2.1",\n",
+       "    "id": "indicator--e97bfccf-8970-4a3c-9cd1-5b5b97ed5d0c",\n",
+       "    "created": "2014-02-20T09:16:08.989Z",\n",
+       "    "modified": "2014-02-20T09:16:08.989Z",\n",
+       "    "name": "File hash for Poison Ivy variant",\n",
+       "    "description": "This file hash indicates that a sample of Poison Ivy is present.",\n",
+       "    "pattern": "[file:hashes.'SHA-256' = 'ef537f25c895bfa782526529a9b63d97aa631564d5d789c2b765448c8635fb6c']",\n",
+       "    "pattern_type": "stix",\n",
+       "    "pattern_version": "2.1",\n",
+       "    "valid_from": "2014-02-20T09:00:00Z",\n",
+       "    "labels": [\n",
+       "        "malicious-activity"\n",
+       "    ],\n",
+       "    "extensions": {\n",
+       "        "extension-definition--dd73de4f-a7f3-49ea-8ec1-8e884196b7a8": {\n",
+       "            "extension_type": "toplevel-property-extension"\n",
+       "        }\n",
+       "    },\n",
+       "    "rank": 5,\n",
+       "    "toxicity": 8\n",
+       "}\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import stix2\n", + "\n", + "extension_definition1 = stix2.v21.ExtensionDefinition(\n", + " id=\"extension-definition--dd73de4f-a7f3-49ea-8ec1-8e884196b7a8\",\n", + " created_by_ref=\"identity--11b76a96-5d2b-45e0-8a5a-f6994f370731\",\n", + " created=\"2014-02-20T09:16:08.000Z\",\n", + " modified=\"2014-02-20T09:16:08.000Z\",\n", + " name=\"New SDO 1\",\n", + " description=\"This schema adds two properties to a STIX object at the toplevel\",\n", + " schema=\"https://www.example.com/schema-foo-1a/v1/\",\n", + " version=\"1.2.1\",\n", + " extension_types=[\"toplevel-property-extension\"],\n", + " extension_properties=[\n", + " \"toxicity\",\n", + " \"rank\",\n", + " ],\n", + ")\n", + "\n", + "indicator = stix2.v21.Indicator(\n", + " id='indicator--e97bfccf-8970-4a3c-9cd1-5b5b97ed5d0c',\n", + " created='2014-02-20T09:16:08.989000Z',\n", + " modified='2014-02-20T09:16:08.989000Z',\n", + " name='File hash for Poison Ivy variant',\n", + " description='This file hash indicates that a sample of Poison Ivy is present.',\n", + " labels=[\n", + " 'malicious-activity',\n", + " ],\n", + " rank=5,\n", + " toxicity=8,\n", + " pattern='[file:hashes.\\'SHA-256\\' = \\'ef537f25c895bfa782526529a9b63d97aa631564d5d789c2b765448c8635fb6c\\']',\n", + " pattern_type='stix',\n", + " valid_from='2014-02-20T09:00:00.000000Z',\n", + " extensions={\n", + " extension_definition1.id : {\n", + " 'extension_type': 'toplevel-property-extension',\n", + " },\n", + " }\n", + ")\n", + "\n", + "print(extension_definition1.serialize(pretty=True))\n", + "print(indicator.serialize(pretty=True))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "However, in order to prevent repetitive instantiation of the same extension, the `@CustomExtension` when used in a class for registering the `extension-definition` with stix2, it allows passing the id. Use the `extension_type` class variable to define what kind of extension it is." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
{\n",
+       "    "type": "indicator",\n",
+       "    "spec_version": "2.1",\n",
+       "    "id": "indicator--e97bfccf-8970-4a3c-9cd1-5b5b97ed5d0c",\n",
+       "    "created": "2014-02-20T09:16:08.989Z",\n",
+       "    "modified": "2014-02-20T09:16:08.989Z",\n",
+       "    "name": "File hash for Poison Ivy variant",\n",
+       "    "description": "This file hash indicates that a sample of Poison Ivy is present.",\n",
+       "    "pattern": "[file:hashes.'SHA-256' = 'ef537f25c895bfa782526529a9b63d97aa631564d5d789c2b765448c8635fb6c']",\n",
+       "    "pattern_type": "stix",\n",
+       "    "pattern_version": "2.1",\n",
+       "    "valid_from": "2014-02-20T09:00:00Z",\n",
+       "    "labels": [\n",
+       "        "malicious-activity"\n",
+       "    ],\n",
+       "    "extensions": {\n",
+       "        "extension-definition--dd73de4f-a7f3-49ea-8ec1-8e884196b7a8": {\n",
+       "            "extension_type": "toplevel-property-extension"\n",
+       "        }\n",
+       "    },\n",
+       "    "rank": 5,\n",
+       "    "toxicity": 8\n",
+       "}\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "TOPLEVEL_EXTENSION_DEFINITION_ID = 'extension-definition--dd73de4f-a7f3-49ea-8ec1-8e884196b7a8'\n", + "\n", + "@stix2.v21.CustomExtension(\n", + " TOPLEVEL_EXTENSION_DEFINITION_ID, [\n", + " ('rank', stix2.properties.IntegerProperty(required=True)),\n", + " ('toxicity', stix2.properties.IntegerProperty(required=True)),\n", + " ],\n", + ")\n", + "class ExtensionTopLevel:\n", + " extension_type = 'toplevel-property-extension'\n", + "\n", + "indicator = stix2.v21.Indicator(\n", + " id='indicator--e97bfccf-8970-4a3c-9cd1-5b5b97ed5d0c',\n", + " created='2014-02-20T09:16:08.989000Z',\n", + " modified='2014-02-20T09:16:08.989000Z',\n", + " name='File hash for Poison Ivy variant',\n", + " description='This file hash indicates that a sample of Poison Ivy is present.',\n", + " labels=[\n", + " 'malicious-activity',\n", + " ],\n", + " rank=5,\n", + " toxicity=8,\n", + " pattern='[file:hashes.\\'SHA-256\\' = \\'ef537f25c895bfa782526529a9b63d97aa631564d5d789c2b765448c8635fb6c\\']',\n", + " pattern_type='stix',\n", + " valid_from='2014-02-20T09:00:00.000000Z',\n", + " extensions={\n", + " TOPLEVEL_EXTENSION_DEFINITION_ID : {\n", + " 'extension_type': 'toplevel-property-extension',\n", + " },\n", + " }\n", + ")\n", + "\n", + "print(indicator.serialize(pretty=True))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Similarly, now when registering new objects `@CustomObject` now supports passing an `extension-definition` id. \n", + "\n", + "---\n", + "**Note:**\n", + "Creating an instance of an extension-definition object **does not** mean it is registered in the library. Please use the appropriate decorator for this step: `@CustomExtension`, `@CustomObject`, `@CustomObservable`, `@CustomMarking`\n", + "\n", + "---" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
{\n",
+       "    "type": "my-favorite-sco",\n",
+       "    "spec_version": "2.1",\n",
+       "    "id": "my-favorite-sco--f9dbe89c-0030-4a9d-8b78-0dcd0a0de874",\n",
+       "    "name": "This is the name of my favorite SCO",\n",
+       "    "some_network_protocol_field": "value",\n",
+       "    "extensions": {\n",
+       "        "extension-definition--150c1738-28c9-44d0-802d-70523218240b": {\n",
+       "            "extension_type": "new-sco"\n",
+       "        }\n",
+       "    }\n",
+       "}\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "@stix2.v21.CustomObservable(\n", + " 'my-favorite-sco', [\n", + " ('name', stix2.properties.StringProperty(required=True)),\n", + " ('some_network_protocol_field', stix2.properties.StringProperty(required=True)),\n", + " ], ['name', 'some_network_protocol_field'], 'extension-definition--150c1738-28c9-44d0-802d-70523218240b',\n", + ")\n", + "class MyFavSCO:\n", + " pass\n", + "\n", + "my_favorite_sco = MyFavSCO(\n", + " id='my-favorite-sco--f9dbe89c-0030-4a9d-8b78-0dcd0a0de874',\n", + " name='This is the name of my favorite SCO',\n", + " some_network_protocol_field='value',\n", + ")\n", + "\n", + "print(my_favorite_sco.serialize(pretty=True))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The example below shows the use for MarkingDefinition extensions. Currently this is only supported as a `property-extension`. Now, as another option to building the `extensions` as a dictionary, it can also be built with objects as shown below by extracting the registered class." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
{\n",
+       "    "type": "marking-definition",\n",
+       "    "spec_version": "2.1",\n",
+       "    "id": "marking-definition--28417f9f-1963-4e7f-914d-233f8fd4829f",\n",
+       "    "created": "2021-03-31T21:54:46.652069Z",\n",
+       "    "name": "This is the name of my favorite Marking",\n",
+       "    "extensions": {\n",
+       "        "extension-definition--a932fcc6-e032-176c-126f-cb970a5a1fff": {\n",
+       "            "extension_type": "property-extension"\n",
+       "        }\n",
+       "    }\n",
+       "}\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from stix2 import registry\n", + "\n", + "MARKING_EXTENSION_ID = 'extension-definition--a932fcc6-e032-176c-126f-cb970a5a1fff'\n", + "\n", + "@stix2.v21.CustomMarking(\n", + " 'my-favorite-marking', [\n", + " ('some_marking_field', stix2.properties.StringProperty(required=True)),\n", + " ], MARKING_EXTENSION_ID,\n", + ")\n", + "class MyFavMarking:\n", + " pass\n", + "\n", + "ext_class = registry.class_for_type(MARKING_EXTENSION_ID, '2.1')\n", + "\n", + "my_favorite_marking = MyFavMarking(\n", + " name='This is the name of my favorite Marking',\n", + " extensions={\n", + " MARKING_EXTENSION_ID: ext_class(some_marking_field='value')\n", + " }\n", + ")\n", + "\n", + "print(my_favorite_marking.serialize(pretty=True))" + ] + } + ], + "metadata": { + "celltoolbar": "Edit Metadata", + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.7" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/overview.rst b/docs/overview.rst index 0396ee8..f342c48 100644 --- a/docs/overview.rst +++ b/docs/overview.rst @@ -24,7 +24,7 @@ To accomplish these goals, and to incorporate lessons learned while developing where users would create an object and then assign attributes to it, in ``stix2`` all properties must be provided when creating the object. 2. Where necessary, library objects should act like ``dict``'s. When treated as - a ``str``, the JSON reprentation of the object should be used. + a ``str``, the JSON representation of the object should be used. 3. Core Python data types (including numeric types, ``datetime``) should be used when appropriate, and serialized to the correct format in JSON as specified in the STIX 2 spec. diff --git a/stix2/registry.py b/stix2/registry.py index 90e2826..7cfc9ef 100644 --- a/stix2/registry.py +++ b/stix2/registry.py @@ -69,9 +69,12 @@ def class_for_type(stix_type, stix_version, category=None): if class_map: cls = class_map.get(stix_type) else: - cls = cat_map["objects"].get(stix_type) \ - or cat_map["observables"].get(stix_type) \ - or cat_map["markings"].get(stix_type) + cls = ( + cat_map["objects"].get(stix_type) or + cat_map["observables"].get(stix_type) or + cat_map["markings"].get(stix_type) or + cat_map["extensions"].get(stix_type) + ) # Left "observable-extensions" out; it has a different # substructure. A version->category->type lookup would result diff --git a/stix2/test/v21/test_extension_definition.py b/stix2/test/v21/test_extension_definition.py index 7aac866..97513c4 100644 --- a/stix2/test/v21/test_extension_definition.py +++ b/stix2/test/v21/test_extension_definition.py @@ -54,7 +54,7 @@ def test_extension_definition_example(): "description": "This schema creates a new object type called my-favorite-sdo-1", "schema": "https://www.example.com/schema-my-favorite-sdo-1/v1/", "version": "1.2.1", - "extension_types": ["new-sdo"] + "extension_types": ["new-sdo"], }, ], ) @@ -75,7 +75,8 @@ def test_parse_extension_definition(data): def test_parse_no_type(): with pytest.raises(stix2.exceptions.ParseError): - stix2.parse("""{ + stix2.parse( + """{ "id": "{EXTENSION_DEFINITION_IDS[0]}", "spec_version": "2.1", "name": "New SDO 1", diff --git a/stix2/v21/common.py b/stix2/v21/common.py index 83d6fa4..a18774d 100644 --- a/stix2/v21/common.py +++ b/stix2/v21/common.py @@ -273,6 +273,7 @@ def CustomMarking(type='x-custom-marking', properties=None, extension_name=None) extension = extension.replace('-', '') NameExtension.__name__ = 'ExtensionDefinition' + extension cls.with_extension = extension_name + return _custom_marking_builder(cls, type, MarkingDefinition._properties, '2.1', _STIXBase21) return _custom_marking_builder(cls, type, properties, '2.1', _STIXBase21) return wrapper From af046beeb5a031f9fff603057f2a4ce1d3c04a7d Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Wed, 31 Mar 2021 18:46:37 -0400 Subject: [PATCH 33/56] add new-sdo or new-sro option in CustomObject decorator, add a test, and example in guide --- docs/guide/extensions.ipynb | 156 ++++++++++++++++++++++++++++++++-- stix2/test/v21/test_custom.py | 35 ++++++++ stix2/v21/sdo.py | 7 +- 3 files changed, 191 insertions(+), 7 deletions(-) diff --git a/docs/guide/extensions.ipynb b/docs/guide/extensions.ipynb index 0283981..817b08a 100644 --- a/docs/guide/extensions.ipynb +++ b/docs/guide/extensions.ipynb @@ -57,10 +57,12 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "STIX Extensions\n", + "## STIX Extensions\n", "\n", "This page is specific for the STIX Extensions mechanism defined in STIX 2.1 CS 02, for the deprecated STIX Customization mechanisms see the [Custom](custom.ipynb) section.\n", "\n", + "### Top Level Property Extensions\n", + "\n", "The example below shows how to create an `indicator` object with a `top-level-property-extension`. " ] }, @@ -332,6 +334,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "### Using CustomExtension decorator\n", + "\n", "However, in order to prevent repetitive instantiation of the same extension, the `@CustomExtension` when used in a class for registering the `extension-definition` with stix2, it allows passing the id. Use the `extension_type` class variable to define what kind of extension it is." ] }, @@ -490,7 +494,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Similarly, now when registering new objects `@CustomObject` now supports passing an `extension-definition` id. \n", + "### Using CustomObservable for Extension Definition\n", + "\n", + "Similarly, when registering new objects via `@CustomObservable` you can pass the `extension-definition` id that defines this new SCO. \n", "\n", "---\n", "**Note:**\n", @@ -501,7 +507,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 3, "metadata": {}, "outputs": [ { @@ -598,7 +604,7 @@ "" ] }, - "execution_count": 15, + "execution_count": 3, "metadata": {}, "output_type": "execute_result" } @@ -626,6 +632,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "### Using CustomMarking for Extension Definition\n", + "\n", "The example below shows the use for MarkingDefinition extensions. Currently this is only supported as a `property-extension`. Now, as another option to building the `extensions` as a dictionary, it can also be built with objects as shown below by extracting the registered class." ] }, @@ -757,10 +765,148 @@ "\n", "print(my_favorite_marking.serialize(pretty=True))" ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Using CustomObject for Extension Definition\n", + "\n", + "Similar to the examples above, the same can be done for SDOs and SROs." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
{\n",
+       "    "type": "my-favorite-sro",\n",
+       "    "spec_version": "2.1",\n",
+       "    "id": "my-favorite-sro--d6306d62-c08d-4d78-baf7-11e7a4c9bc36",\n",
+       "    "created": "2021-03-31T22:43:42.807698Z",\n",
+       "    "modified": "2021-03-31T22:43:42.807698Z",\n",
+       "    "name": "My First SRO",\n",
+       "    "some_source_ref": "identity--b1da8c3e-34d8-470f-9d2b-392e275f1f7d",\n",
+       "    "some_target_ref": "identity--1ddfed54-e8cd-49c9-9c7d-8d1b03c94685",\n",
+       "    "extensions": {\n",
+       "        "extension-definition--e96690a5-dc13-4f27-99dd-0f2188ad74ce": {\n",
+       "            "extension_type": "new-sro"\n",
+       "        }\n",
+       "    }\n",
+       "}\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "invalid_refs = ['bundle', 'language-content', 'marking-definition', 'relationship', 'sighting', 'foobar']\n", + "\n", + "@stix2.v21.CustomObject(\n", + " 'my-favorite-sro', [\n", + " ('name', stix2.properties.StringProperty(required=False)),\n", + " ('some_source_ref', stix2.properties.ReferenceProperty(invalid_types=invalid_refs, spec_version='2.1', required=True)),\n", + " ('some_target_ref', stix2.properties.ReferenceProperty(invalid_types=invalid_refs, spec_version='2.1', required=True)),\n", + " ], extension_name='extension-definition--e96690a5-dc13-4f27-99dd-0f2188ad74ce', is_sdo=False,\n", + ")\n", + "class MyFavSRO:\n", + " pass\n", + "\n", + "\n", + "my_favorite_sro = MyFavSRO(\n", + " name=\"My First SRO\",\n", + " some_source_ref=\"identity--b1da8c3e-34d8-470f-9d2b-392e275f1f7d\",\n", + " some_target_ref=\"identity--1ddfed54-e8cd-49c9-9c7d-8d1b03c94685\",\n", + ")\n", + "\n", + "print(my_favorite_sro.serialize(pretty=True))" + ] } ], "metadata": { - "celltoolbar": "Edit Metadata", "kernelspec": { "display_name": "Python 3", "language": "python", diff --git a/stix2/test/v21/test_custom.py b/stix2/test/v21/test_custom.py index ac46d84..2fc8db0 100644 --- a/stix2/test/v21/test_custom.py +++ b/stix2/test/v21/test_custom.py @@ -1462,6 +1462,41 @@ def test_registered_new_extension_sdo_allow_custom_false(): assert '"extensions": {"extension-definition--d83fce45-ef58-4c6c-a3f4-1fbc32e9999": {"extension_type": "new-sdo"}}' in sdo_serialized +def test_registered_new_extension_sro_allow_custom_false(): + @stix2.v21.CustomObject( + 'my-favorite-sro', [ + ('name', stix2.properties.StringProperty(required=True)), + ('some_property_name1', stix2.properties.StringProperty(required=True)), + ('some_property_name2', stix2.properties.StringProperty()), + ], 'extension-definition--e96690a5-dc13-4f27-99dd-0f2188ad74ce', False, + ) + class MyFavSRO: + pass + + my_favorite_sro = { + 'type': 'my-favorite-sro', + 'spec_version': '2.1', + 'id': 'my-favorite-sro--c5ba9dba-5ad9-4bbe-9825-df4cb8675774', + 'created': '2014-02-20T09:16:08.989000Z', + 'modified': '2014-02-20T09:16:08.989000Z', + 'name': 'This is the name of my favorite', + 'some_property_name1': 'value1', + 'some_property_name2': 'value2', + # 'extensions': { + # 'extension-definition--e96690a5-dc13-4f27-99dd-0f2188ad74ce': ExtensionDefinitiond83fce45ef584c6ca3f41fbc32e98c6e() + # } + } + sro_object = stix2.parse(my_favorite_sro) + assert isinstance(sro_object, MyFavSRO) + assert isinstance( + sro_object.extensions['extension-definition--e96690a5-dc13-4f27-99dd-0f2188ad74ce'], + stix2.v21.EXT_MAP['extension-definition--e96690a5-dc13-4f27-99dd-0f2188ad74ce'], + ) + + sdo_serialized = sro_object.serialize() + assert '"extensions": {"extension-definition--e96690a5-dc13-4f27-99dd-0f2188ad74ce": {"extension_type": "new-sro"}}' in sdo_serialized + + def test_registered_new_extension_sco_allow_custom_false(): @stix2.v21.CustomObservable( 'my-favorite-sco', [ diff --git a/stix2/v21/sdo.py b/stix2/v21/sdo.py index 1ca448a..a834ce5 100644 --- a/stix2/v21/sdo.py +++ b/stix2/v21/sdo.py @@ -806,7 +806,7 @@ class Vulnerability(_DomainObject): ]) -def CustomObject(type='x-custom-type', properties=None, extension_name=None): +def CustomObject(type='x-custom-type', properties=None, extension_name=None, is_sdo=True): """Custom STIX Object type decorator. Example: @@ -864,7 +864,10 @@ def CustomObject(type='x-custom-type', properties=None, extension_name=None): if extension_name: @observables.CustomExtension(type=extension_name, properties=extension_properties) class NameExtension: - extension_type = 'new-sdo' + if is_sdo: + extension_type = 'new-sdo' + else: + extension_type = 'new-sro' extension = extension_name.split('--')[1] extension = extension.replace('-', '') From 40023ec98744e42f4dbb4aa3908bb80008bd07f2 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Wed, 31 Mar 2021 18:55:55 -0400 Subject: [PATCH 34/56] forgot to add this in the rebase... --- stix2/test/v20/test_bundle.py | 1 - stix2/test/v21/test_bundle.py | 1 - 2 files changed, 2 deletions(-) diff --git a/stix2/test/v20/test_bundle.py b/stix2/test/v20/test_bundle.py index 35588a5..3843375 100644 --- a/stix2/test/v20/test_bundle.py +++ b/stix2/test/v20/test_bundle.py @@ -120,7 +120,6 @@ def test_create_bundle_fp_serialize_pretty(indicator, malware, relationship): bundle.fp_serialize(buffer, pretty=True) - assert str(bundle) == EXPECTED_BUNDLE assert bundle.serialize(pretty=True) == EXPECTED_BUNDLE assert buffer.getvalue() == EXPECTED_BUNDLE diff --git a/stix2/test/v21/test_bundle.py b/stix2/test/v21/test_bundle.py index b7d0946..008fcf1 100644 --- a/stix2/test/v21/test_bundle.py +++ b/stix2/test/v21/test_bundle.py @@ -130,7 +130,6 @@ def test_create_bundle_fp_serialize_pretty(indicator, malware, relationship): bundle.fp_serialize(buffer, pretty=True) - assert str(bundle) == EXPECTED_BUNDLE assert bundle.serialize(pretty=True) == EXPECTED_BUNDLE assert buffer.getvalue() == EXPECTED_BUNDLE From d703833f6f5be17fe37294f3061dade861bd5ade Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Fri, 2 Apr 2021 11:18:08 -0400 Subject: [PATCH 35/56] Update README.rst Demonstrate the use of ``serialize`` --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index f8f8304..47e0fef 100644 --- a/README.rst +++ b/README.rst @@ -39,7 +39,7 @@ be set automatically if not provided as keyword arguments. pattern_type="stix", pattern="[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']") -To parse a STIX JSON string into a Python STIX object, use ``parse()``: +To parse a STIX JSON string into a Python STIX object, use ``parse()``. To serialize a STIX object, use ``serialize()``: .. code-block:: python @@ -61,7 +61,7 @@ To parse a STIX JSON string into a Python STIX object, use ``parse()``: "valid_from": "2017-09-26T23:33:39.829952Z" }""") - print(indicator) + print(indicator.serialize(pretty=true)) For more in-depth documentation, please see `https://stix2.readthedocs.io/ `__. From bd90e219adcd13b9ca562478549cdd6432b7c9ec Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Fri, 2 Apr 2021 11:18:46 -0400 Subject: [PATCH 36/56] Update README.rst --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 47e0fef..447eb5c 100644 --- a/README.rst +++ b/README.rst @@ -61,7 +61,7 @@ To parse a STIX JSON string into a Python STIX object, use ``parse()``. To seria "valid_from": "2017-09-26T23:33:39.829952Z" }""") - print(indicator.serialize(pretty=true)) + print(indicator.serialize(pretty=True)) For more in-depth documentation, please see `https://stix2.readthedocs.io/ `__. From 3cd05e6bdc184e912cd344104207761e414e8f62 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Fri, 2 Apr 2021 13:04:08 -0400 Subject: [PATCH 37/56] Update contributing.rst remove Travis CI mention... --- docs/contributing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index a904c31..c676311 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -107,7 +107,7 @@ run: then look at the resulting report in ``htmlcov/index.html``. All commits pushed to the ``master`` branch or submitted as a pull request are -tested with `Travis-CI `_ +tested with `GitHub Actions `_ automatically. Adding a dependency From 9f96d7631e319669cc5164f03cc9d6e54fedf8ad Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Fri, 9 Apr 2021 09:26:35 -0400 Subject: [PATCH 38/56] Update TAXII example It was set up for an old version of the default test data in medallion. --- examples/taxii_example.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/taxii_example.py b/examples/taxii_example.py index 51ba821..4bee3c0 100644 --- a/examples/taxii_example.py +++ b/examples/taxii_example.py @@ -8,7 +8,7 @@ import stix2 def main(): collection = Collection( - "http://127.0.0.1:5000/trustgroup1/collections/52892447-4d7e-4f70-b94d-d7f22742ff63/", + "http://127.0.0.1:5000/trustgroup1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/", user="admin", password="Password0", ) @@ -16,12 +16,12 @@ def main(): taxii = stix2.TAXIICollectionSource(collection) # get (url watch indicator) - indicator_fw = taxii.get("indicator--00000000-0000-4000-8000-000000000001") + indicator_fw = taxii.get("indicator--6770298f-0fd8-471a-ab8c-1c658a46574e") print("\n\n-------Queried for Indicator - got:") print(indicator_fw.serialize(indent=4)) # all versions (url watch indicator - currently two) - indicator_fw_versions = taxii.all_versions("indicator--00000000-0000-4000-8000-000000000001") + indicator_fw_versions = taxii.all_versions("indicator--6770298f-0fd8-471a-ab8c-1c658a46574e") print("\n\n------Queried for indicator (all_versions()) - got:") for indicator in indicator_fw_versions: print(indicator.serialize(indent=4)) From a002c1736d2fdf54064823ab3fccf261157e4ac0 Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Wed, 14 Apr 2021 16:39:56 -0400 Subject: [PATCH 39/56] Update guide docs, remove outdated comments --- docs/guide/extensions.ipynb | 6 +- docs/guide/markings.ipynb | 571 ++++++++++++++++++++--------------- docs/guide/patterns.ipynb | 4 +- docs/guide/serializing.ipynb | 2 +- stix2/base.py | 4 +- stix2/parsing.py | 9 +- stix2/registry.py | 4 - 7 files changed, 340 insertions(+), 260 deletions(-) diff --git a/docs/guide/extensions.ipynb b/docs/guide/extensions.ipynb index 817b08a..9dcbdb2 100644 --- a/docs/guide/extensions.ipynb +++ b/docs/guide/extensions.ipynb @@ -59,7 +59,7 @@ "source": [ "## STIX Extensions\n", "\n", - "This page is specific for the STIX Extensions mechanism defined in STIX 2.1 CS 02, for the deprecated STIX Customization mechanisms see the [Custom](custom.ipynb) section.\n", + "This page is specific for the STIX Extensions mechanism defined in STIX 2.1 CS 02. For the deprecated STIX Customization mechanisms see the [Custom](custom.ipynb) section.\n", "\n", "### Top Level Property Extensions\n", "\n", @@ -336,7 +336,7 @@ "source": [ "### Using CustomExtension decorator\n", "\n", - "However, in order to prevent repetitive instantiation of the same extension, the `@CustomExtension` when used in a class for registering the `extension-definition` with stix2, it allows passing the id. Use the `extension_type` class variable to define what kind of extension it is." + "However, in order to prevent repetitive instantiation of the same extension, the `@CustomExtension` decorator can be used to register the `extension-definition` with stix2. Use the `extension_type` class variable to define what kind of extension it is. Then its id can be passed into objects that use this extension." ] }, { @@ -922,7 +922,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.7" + "version": "3.9.2" } }, "nbformat": 4, diff --git a/docs/guide/markings.ipynb b/docs/guide/markings.ipynb index 942f629..5bc1215 100644 --- a/docs/guide/markings.ipynb +++ b/docs/guide/markings.ipynb @@ -77,8 +77,13 @@ { "data": { "text/html": [ - "
{\n",
        "    "type": "indicator",\n",
        "    "spec_version": "2.1",\n",
-       "    "id": "indicator--46498844-7689-4e7b-be25-b119d8401159",\n",
-       "    "created": "2020-06-24T20:55:56.088861Z",\n",
-       "    "modified": "2020-06-24T20:55:56.088861Z",\n",
+       "    "id": "indicator--a315ce0b-1211-478e-812a-cd6d3eecc3c1",\n",
+       "    "created": "2021-04-09T13:59:48.911595Z",\n",
+       "    "modified": "2021-04-09T13:59:48.911595Z",\n",
        "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
        "    "pattern_type": "stix",\n",
        "    "pattern_version": "2.1",\n",
-       "    "valid_from": "2020-06-24T20:55:56.088861Z",\n",
+       "    "valid_from": "2021-04-09T13:59:48.911595Z",\n",
        "    "object_marking_refs": [\n",
        "        "marking-definition--f88d31f6-486f-44da-b317-01333bde0b82"\n",
        "    ]\n",
@@ -194,8 +199,13 @@
     {
      "data": {
       "text/html": [
-       "
{\n",
        "    "type": "marking-definition",\n",
        "    "spec_version": "2.1",\n",
-       "    "id": "marking-definition--9a4efc30-a7ac-42d0-8776-16f390a0fd44",\n",
-       "    "created": "2020-06-24T20:56:06.779241Z",\n",
+       "    "id": "marking-definition--4b8e86b5-d505-46a4-91b4-a8db17f4ff4d",\n",
+       "    "created": "2021-04-09T13:59:50.587649Z",\n",
        "    "definition_type": "statement",\n",
        "    "definition": {\n",
        "        "statement": "Copyright 2017, Example Corp"\n",
@@ -308,8 +318,13 @@
     {
      "data": {
       "text/html": [
-       "
{\n",
        "    "type": "indicator",\n",
        "    "spec_version": "2.1",\n",
-       "    "id": "indicator--75d66696-9960-4229-ba89-2caac50891b3",\n",
-       "    "created": "2020-06-24T20:56:29.80259Z",\n",
-       "    "modified": "2020-06-24T20:56:29.80259Z",\n",
+       "    "id": "indicator--91ed23a6-c5f0-4b16-8369-64cf39f974bf",\n",
+       "    "created": "2021-04-09T13:59:52.602254Z",\n",
+       "    "modified": "2021-04-09T13:59:52.602254Z",\n",
        "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
        "    "pattern_type": "stix",\n",
        "    "pattern_version": "2.1",\n",
-       "    "valid_from": "2020-06-24T20:56:29.80259Z",\n",
+       "    "valid_from": "2021-04-09T13:59:52.602254Z",\n",
        "    "object_marking_refs": [\n",
-       "        "marking-definition--9a4efc30-a7ac-42d0-8776-16f390a0fd44"\n",
+       "        "marking-definition--4b8e86b5-d505-46a4-91b4-a8db17f4ff4d"\n",
        "    ]\n",
        "}\n",
        "
\n"
@@ -416,8 +431,13 @@
     {
      "data": {
       "text/html": [
-       "
{\n",
        "    "type": "indicator",\n",
        "    "spec_version": "2.1",\n",
-       "    "id": "indicator--757ea853-138c-44e2-bb00-e78eebfaa378",\n",
-       "    "created": "2020-06-24T20:56:43.703563Z",\n",
-       "    "modified": "2020-06-24T20:56:43.703563Z",\n",
+       "    "id": "indicator--42ae262e-4839-4c1a-a50a-3a6690623a9d",\n",
+       "    "created": "2021-04-09T13:59:54.207797Z",\n",
+       "    "modified": "2021-04-09T13:59:54.207797Z",\n",
        "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
        "    "pattern_type": "stix",\n",
        "    "pattern_version": "2.1",\n",
-       "    "valid_from": "2020-06-24T20:56:43.703563Z",\n",
+       "    "valid_from": "2021-04-09T13:59:54.207797Z",\n",
        "    "object_marking_refs": [\n",
        "        "marking-definition--f88d31f6-486f-44da-b317-01333bde0b82"\n",
        "    ]\n",
@@ -525,14 +545,19 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": 7,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/html": [
-       "
{\n",
        "    "type": "malware",\n",
        "    "spec_version": "2.1",\n",
-       "    "id": "malware--1752bbec-765a-4711-a304-f0e92ca902ae",\n",
-       "    "created": "2020-06-24T21:21:07.148194Z",\n",
-       "    "modified": "2020-06-24T21:21:07.148194Z",\n",
+       "    "id": "malware--2658ac6a-44e9-44ea-8c8a-d67abae4d0d5",\n",
+       "    "created": "2021-04-09T13:59:56.556801Z",\n",
+       "    "modified": "2021-04-09T13:59:56.556801Z",\n",
        "    "name": "Poison Ivy",\n",
        "    "description": "A ransomware related to ...",\n",
        "    "is_family": false,\n",
        "    "granular_markings": [\n",
        "        {\n",
-       "            "marking_ref": "marking-definition--9a4efc30-a7ac-42d0-8776-16f390a0fd44",\n",
+       "            "marking_ref": "marking-definition--4b8e86b5-d505-46a4-91b4-a8db17f4ff4d",\n",
        "            "selectors": [\n",
        "                "description"\n",
        "            ]\n",
@@ -629,7 +654,7 @@
        ""
       ]
      },
-     "execution_count": 8,
+     "execution_count": 7,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -662,7 +687,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": 8,
    "metadata": {},
    "outputs": [
     {
@@ -706,14 +731,19 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": 9,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/html": [
-       "
{\n",
        "    "type": "indicator",\n",
        "    "spec_version": "2.1",\n",
-       "    "id": "indicator--46498844-7689-4e7b-be25-b119d8401159",\n",
-       "    "created": "2020-06-24T20:55:56.088861Z",\n",
-       "    "modified": "2020-06-24T21:21:39.898475Z",\n",
+       "    "id": "indicator--a315ce0b-1211-478e-812a-cd6d3eecc3c1",\n",
+       "    "created": "2021-04-09T13:59:48.911595Z",\n",
+       "    "modified": "2021-04-09T14:00:01.165749Z",\n",
        "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
        "    "pattern_type": "stix",\n",
        "    "pattern_version": "2.1",\n",
-       "    "valid_from": "2020-06-24T20:55:56.088861Z",\n",
+       "    "valid_from": "2021-04-09T13:59:48.911595Z",\n",
        "    "object_marking_refs": [\n",
-       "        "marking-definition--f88d31f6-486f-44da-b317-01333bde0b82",\n",
-       "        "marking-definition--9a4efc30-a7ac-42d0-8776-16f390a0fd44"\n",
+       "        "marking-definition--4b8e86b5-d505-46a4-91b4-a8db17f4ff4d",\n",
+       "        "marking-definition--f88d31f6-486f-44da-b317-01333bde0b82"\n",
        "    ]\n",
        "}\n",
        "
\n"
@@ -801,7 +831,7 @@
        ""
       ]
      },
-     "execution_count": 10,
+     "execution_count": 9,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -820,14 +850,19 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 11,
+   "execution_count": 10,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/html": [
-       "
{\n",
        "    "type": "indicator",\n",
        "    "spec_version": "2.1",\n",
-       "    "id": "indicator--46498844-7689-4e7b-be25-b119d8401159",\n",
-       "    "created": "2020-06-24T20:55:56.088861Z",\n",
-       "    "modified": "2020-06-24T21:21:43.529702Z",\n",
+       "    "id": "indicator--a315ce0b-1211-478e-812a-cd6d3eecc3c1",\n",
+       "    "created": "2021-04-09T13:59:48.911595Z",\n",
+       "    "modified": "2021-04-09T14:00:03.00911Z",\n",
        "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
        "    "pattern_type": "stix",\n",
        "    "pattern_version": "2.1",\n",
-       "    "valid_from": "2020-06-24T20:55:56.088861Z",\n",
+       "    "valid_from": "2021-04-09T13:59:48.911595Z",\n",
        "    "object_marking_refs": [\n",
        "        "marking-definition--f88d31f6-486f-44da-b317-01333bde0b82"\n",
        "    ]\n",
@@ -914,21 +949,142 @@
        ""
       ]
      },
+     "execution_count": 10,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "indicator5 = indicator4.remove_markings(marking_definition)\n",
+    "print(indicator5.serialize(pretty=True))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "The markings on an object can be replaced with a different set of markings:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "
{\n",
+       "    "type": "indicator",\n",
+       "    "spec_version": "2.1",\n",
+       "    "id": "indicator--a315ce0b-1211-478e-812a-cd6d3eecc3c1",\n",
+       "    "created": "2021-04-09T13:59:48.911595Z",\n",
+       "    "modified": "2021-04-09T14:00:04.531083Z",\n",
+       "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
+       "    "pattern_type": "stix",\n",
+       "    "pattern_version": "2.1",\n",
+       "    "valid_from": "2021-04-09T13:59:48.911595Z",\n",
+       "    "object_marking_refs": [\n",
+       "        "marking-definition--34098fce-860f-48ae-8e50-ebd3cc5e41da",\n",
+       "        "marking-definition--4b8e86b5-d505-46a4-91b4-a8db17f4ff4d"\n",
+       "    ]\n",
+       "}\n",
+       "
\n"
+      ],
+      "text/plain": [
+       ""
+      ]
+     },
      "execution_count": 11,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
-    "indicator5 = indicator4.remove_markings(marking_definition)\n",
-    "print(indicator5)"
+    "from stix2 import TLP_GREEN\n",
+    "\n",
+    "indicator6 = indicator5.set_markings([TLP_GREEN, marking_definition])\n",
+    "print(indicator6.serialize(pretty=True))"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "The markings on an object can be replaced with a different set of markings:"
+    "STIX objects can also be cleared of all markings with [clear_markings()](../api/stix2.markings.rst#stix2.markings.clear_markings):"
    ]
   },
   {
@@ -939,8 +1095,13 @@
     {
      "data": {
       "text/html": [
-       "
{\n",
        "    "type": "indicator",\n",
        "    "spec_version": "2.1",\n",
-       "    "id": "indicator--46498844-7689-4e7b-be25-b119d8401159",\n",
-       "    "created": "2020-06-24T20:55:56.088861Z",\n",
-       "    "modified": "2020-06-24T21:21:47.703212Z",\n",
+       "    "id": "indicator--a315ce0b-1211-478e-812a-cd6d3eecc3c1",\n",
+       "    "created": "2021-04-09T13:59:48.911595Z",\n",
+       "    "modified": "2021-04-09T14:00:06.512465Z",\n",
        "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
        "    "pattern_type": "stix",\n",
        "    "pattern_version": "2.1",\n",
-       "    "valid_from": "2020-06-24T20:55:56.088861Z",\n",
-       "    "object_marking_refs": [\n",
-       "        "marking-definition--9a4efc30-a7ac-42d0-8776-16f390a0fd44",\n",
-       "        "marking-definition--34098fce-860f-48ae-8e50-ebd3cc5e41da"\n",
-       "    ]\n",
+       "    "valid_from": "2021-04-09T13:59:48.911595Z"\n",
        "}\n",
        "
\n"
       ],
@@ -1033,121 +1190,9 @@
      "output_type": "execute_result"
     }
    ],
-   "source": [
-    "from stix2 import TLP_GREEN\n",
-    "\n",
-    "indicator6 = indicator5.set_markings([TLP_GREEN, marking_definition])\n",
-    "print(indicator6)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "STIX objects can also be cleared of all markings with [clear_markings()](../api/stix2.markings.rst#stix2.markings.clear_markings):"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 13,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/html": [
-       "
{\n",
-       "    "type": "indicator",\n",
-       "    "spec_version": "2.1",\n",
-       "    "id": "indicator--46498844-7689-4e7b-be25-b119d8401159",\n",
-       "    "created": "2020-06-24T20:55:56.088861Z",\n",
-       "    "modified": "2020-06-24T21:21:53.287178Z",\n",
-       "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
-       "    "pattern_type": "stix",\n",
-       "    "pattern_version": "2.1",\n",
-       "    "valid_from": "2020-06-24T20:55:56.088861Z"\n",
-       "}\n",
-       "
\n"
-      ],
-      "text/plain": [
-       ""
-      ]
-     },
-     "execution_count": 13,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
    "source": [
     "indicator7 = indicator5.clear_markings()\n",
-    "print(indicator7)"
+    "print(indicator7.serialize(pretty=True))"
    ]
   },
   {
@@ -1168,17 +1213,17 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 14,
+   "execution_count": 13,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "['marking-definition--9a4efc30-a7ac-42d0-8776-16f390a0fd44',\n",
-       " 'marking-definition--34098fce-860f-48ae-8e50-ebd3cc5e41da']"
+       "['marking-definition--34098fce-860f-48ae-8e50-ebd3cc5e41da',\n",
+       " 'marking-definition--4b8e86b5-d505-46a4-91b4-a8db17f4ff4d']"
       ]
      },
-     "execution_count": 14,
+     "execution_count": 13,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -1196,7 +1241,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 15,
+   "execution_count": 14,
    "metadata": {},
    "outputs": [
     {
@@ -1205,7 +1250,7 @@
        "['marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9']"
       ]
      },
-     "execution_count": 15,
+     "execution_count": 14,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -1225,7 +1270,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 16,
+   "execution_count": 15,
    "metadata": {},
    "outputs": [
     {
@@ -1234,7 +1279,7 @@
        "['marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9']"
       ]
      },
-     "execution_count": 16,
+     "execution_count": 15,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -1250,6 +1295,26 @@
     "Finally, you may also check if an object is marked by a specific markings. Again, for granular markings, pass in the selector or list of selectors."
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "True"
+      ]
+     },
+     "execution_count": 16,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "indicator.is_marked(TLP_AMBER.id)"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": 17,
@@ -1266,33 +1331,13 @@
      "output_type": "execute_result"
     }
    ],
-   "source": [
-    "indicator.is_marked(TLP_AMBER.id)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 18,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "True"
-      ]
-     },
-     "execution_count": 18,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
    "source": [
     "malware.is_marked(TLP_WHITE.id, 'name')"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 19,
+   "execution_count": 18,
    "metadata": {
     "scrolled": true
    },
@@ -1303,7 +1348,7 @@
        "False"
       ]
      },
-     "execution_count": 19,
+     "execution_count": 18,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -1323,14 +1368,19 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 20,
+   "execution_count": 19,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/html": [
-       "
{\n",
        "    "type": "indicator",\n",
        "    "spec_version": "2.1",\n",
-       "    "id": "indicator--f4004de9-a6d9-4c7b-823e-3d8199173c09",\n",
-       "    "created": "2020-06-24T21:35:08.630228Z",\n",
-       "    "modified": "2020-06-24T21:35:08.630228Z",\n",
+       "    "id": "indicator--a2fd263a-ec46-4fff-84af-27419f0b9f15",\n",
+       "    "created": "2021-04-09T14:02:31.991141Z",\n",
+       "    "modified": "2021-04-09T14:02:31.991141Z",\n",
        "    "description": "Una descripcion sobre este indicador",\n",
        "    "indicator_types": [\n",
        "        "malware"\n",
@@ -1410,7 +1460,7 @@
        "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
        "    "pattern_type": "stix",\n",
        "    "pattern_version": "2.1",\n",
-       "    "valid_from": "2020-06-24T21:35:08.630228Z",\n",
+       "    "valid_from": "2021-04-09T14:02:31.991141Z",\n",
        "    "object_marking_refs": [\n",
        "        "marking-definition--f88d31f6-486f-44da-b317-01333bde0b82"\n",
        "    ],\n",
@@ -1435,15 +1485,20 @@
        ""
       ]
      },
-     "execution_count": 20,
+     "execution_count": 19,
      "metadata": {},
      "output_type": "execute_result"
     },
     {
      "data": {
       "text/html": [
-       "
{\n",
        "    "type": "indicator",\n",
        "    "spec_version": "2.1",\n",
-       "    "id": "indicator--f4004de9-a6d9-4c7b-823e-3d8199173c09",\n",
-       "    "created": "2020-06-24T21:35:08.630228Z",\n",
-       "    "modified": "2020-06-24T21:35:14.54482Z",\n",
+       "    "id": "indicator--a2fd263a-ec46-4fff-84af-27419f0b9f15",\n",
+       "    "created": "2021-04-09T14:02:31.991141Z",\n",
+       "    "modified": "2021-04-09T14:03:11.817032Z",\n",
        "    "description": "Una descripcion sobre este indicador",\n",
        "    "indicator_types": [\n",
        "        "malware"\n",
@@ -1814,7 +1884,7 @@
        "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
        "    "pattern_type": "stix",\n",
        "    "pattern_version": "2.1",\n",
-       "    "valid_from": "2020-06-24T21:35:08.630228Z",\n",
+       "    "valid_from": "2021-04-09T14:02:31.991141Z",\n",
        "    "object_marking_refs": [\n",
        "        "marking-definition--f88d31f6-486f-44da-b317-01333bde0b82"\n",
        "    ]\n",
@@ -1831,7 +1901,8 @@
     }
    ],
    "source": [
-    "print(v21_indicator.clear_markings(\"description\"))  # By default, both types of markings will be removed"
+    "# By default, both types of markings will be removed\n",
+    "print(v21_indicator.clear_markings(\"description\").serialize(pretty=True))"
    ]
   },
   {
@@ -1842,8 +1913,13 @@
     {
      "data": {
       "text/html": [
-       "
{\n",
        "    "type": "indicator",\n",
        "    "spec_version": "2.1",\n",
-       "    "id": "indicator--f4004de9-a6d9-4c7b-823e-3d8199173c09",\n",
-       "    "created": "2020-06-24T21:35:08.630228Z",\n",
-       "    "modified": "2020-06-24T21:35:39.298138Z",\n",
+       "    "id": "indicator--a2fd263a-ec46-4fff-84af-27419f0b9f15",\n",
+       "    "created": "2021-04-09T14:02:31.991141Z",\n",
+       "    "modified": "2021-04-09T14:03:24.701927Z",\n",
        "    "description": "Una descripcion sobre este indicador",\n",
        "    "indicator_types": [\n",
        "        "malware"\n",
@@ -1923,7 +1999,7 @@
        "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
        "    "pattern_type": "stix",\n",
        "    "pattern_version": "2.1",\n",
-       "    "valid_from": "2020-06-24T21:35:08.630228Z",\n",
+       "    "valid_from": "2021-04-09T14:02:31.991141Z",\n",
        "    "object_marking_refs": [\n",
        "        "marking-definition--f88d31f6-486f-44da-b317-01333bde0b82"\n",
        "    ],\n",
@@ -1949,7 +2025,7 @@
    ],
    "source": [
     "# If lang is False, no lang markings will be removed\n",
-    "print(v21_indicator.clear_markings(\"description\", lang=False))"
+    "print(v21_indicator.clear_markings(\"description\", lang=False).serialize(pretty=True))"
    ]
   },
   {
@@ -1960,8 +2036,13 @@
     {
      "data": {
       "text/html": [
-       "
{\n",
        "    "type": "indicator",\n",
        "    "spec_version": "2.1",\n",
-       "    "id": "indicator--f4004de9-a6d9-4c7b-823e-3d8199173c09",\n",
-       "    "created": "2020-06-24T21:35:08.630228Z",\n",
-       "    "modified": "2020-06-24T21:35:42.684794Z",\n",
+       "    "id": "indicator--a2fd263a-ec46-4fff-84af-27419f0b9f15",\n",
+       "    "created": "2021-04-09T14:02:31.991141Z",\n",
+       "    "modified": "2021-04-09T14:03:29.751985Z",\n",
        "    "description": "Una descripcion sobre este indicador",\n",
        "    "indicator_types": [\n",
        "        "malware"\n",
@@ -2041,7 +2122,7 @@
        "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
        "    "pattern_type": "stix",\n",
        "    "pattern_version": "2.1",\n",
-       "    "valid_from": "2020-06-24T21:35:08.630228Z",\n",
+       "    "valid_from": "2021-04-09T14:02:31.991141Z",\n",
        "    "object_marking_refs": [\n",
        "        "marking-definition--f88d31f6-486f-44da-b317-01333bde0b82"\n",
        "    ],\n",
@@ -2067,7 +2148,7 @@
    ],
    "source": [
     "# If marking_ref is False, no marking-definition markings will be removed\n",
-    "print(v21_indicator.clear_markings(\"description\", marking_ref=False))"
+    "print(v21_indicator.clear_markings(\"description\", marking_ref=False).serialize(pretty=True))"
    ]
   }
  ],
@@ -2087,7 +2168,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.0a6"
+   "version": "3.9.2"
   }
  },
  "nbformat": 4,
diff --git a/docs/guide/patterns.ipynb b/docs/guide/patterns.ipynb
index 4dc0175..dfd49f7 100644
--- a/docs/guide/patterns.ipynb
+++ b/docs/guide/patterns.ipynb
@@ -1502,7 +1502,7 @@
     "\n",
     "ece14 = ObservationExpression(EqualityComparisonExpression(ObjectPath(\"file\", [\"name\"]), \"$$t00rzch$$.elf\"))\n",
     "ind = Indicator(name=\"Cryptotorch\", pattern_type=\"stix\", pattern=ece14)\n",
-    "print(ind)"
+    "print(ind.serialize(pretty=True))"
    ]
   }
  ],
@@ -1522,7 +1522,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.0a6"
+   "version": "3.9.2"
   }
  },
  "nbformat": 4,
diff --git a/docs/guide/serializing.ipynb b/docs/guide/serializing.ipynb
index fd91c60..cbbc6f3 100644
--- a/docs/guide/serializing.ipynb
+++ b/docs/guide/serializing.ipynb
@@ -183,7 +183,7 @@
     "---\n",
     "**New in 3.0.0:** \n",
     "\n",
-    "Calling `str()` on a STIX object will call `serialize()` without any formatting options. The change was made to address the performance penalty induced by unknowingly calling with the pretty formatted option. As shown above, to get effect `str()` would in the past versions of the library use the method directly and pass the pretty argument directly `serialize(pretty=True)`.\n",
+    "Calling `str()` on a STIX object will call `serialize()` without any formatting options. The change was made to address the performance penalty induced by unknowingly calling with the pretty formatted option. As shown above, to get the same effect as `str()` had in past versions of the library, use the method directly and pass in the pretty argument `serialize(pretty=True)`.\n",
     "\n",
     "---\n",
     "\n",
diff --git a/stix2/base.py b/stix2/base.py
index c581be8..ac726bd 100644
--- a/stix2/base.py
+++ b/stix2/base.py
@@ -126,7 +126,6 @@ class _STIXBase(collections.abc.Mapping):
             if ext_found is False:
                 raise ExtraPropertiesError(cls, extra_kwargs)
 
-        # because allow_custom is true, any extra kwargs are custom
         if custom_props or extra_kwargs:
             self._allow_custom = True
             if isinstance(self, stix2.v21._STIXBase21):
@@ -153,7 +152,8 @@ class _STIXBase(collections.abc.Mapping):
         if missing_kwargs:
             new_ext_check = (
                 bool(getattr(self, "extension_type", None))
-            ) and issubclass(cls, stix2.v21._Extension)
+                and issubclass(cls, stix2.v21._Extension)
+            )
             if new_ext_check is False:
                 raise MissingPropertiesError(cls, missing_kwargs)
 
diff --git a/stix2/parsing.py b/stix2/parsing.py
index 001f523..8b39f91 100644
--- a/stix2/parsing.py
+++ b/stix2/parsing.py
@@ -87,11 +87,14 @@ def dict_to_stix2(stix_dict, allow_custom=False, version=None):
             # be parsed into STIX object, returned as is
             return stix_dict
         for key_id, ext_def in stix_dict.get('extensions', {}).items():
-            if key_id.startswith('extension-definition--') and ext_def.get('extension_type', None):
+            if (
+                key_id.startswith('extension-definition--') and
+                'property-extension' not in ext_def.get('extension_type', '')
+            ):
                 # prevents ParseError for unregistered objects when
-                # 'is_new_object' or 'is_extension_so' are set to True and allow_custom=False
+                # allow_custom=False and the extension defines a new object
                 return stix_dict
-        raise ParseError("Can't parse unknown object type '%s'! For custom types, use the CustomObject decorator." % stix_dict['type'])
+        raise ParseError("Can't parse unknown object type '%s'! For custom types, use the CustomObject decorator." % obj_type)
 
     return obj_class(allow_custom=allow_custom, **stix_dict)
 
diff --git a/stix2/registry.py b/stix2/registry.py
index 7cfc9ef..5ce7e24 100644
--- a/stix2/registry.py
+++ b/stix2/registry.py
@@ -76,8 +76,4 @@ def class_for_type(stix_type, stix_version, category=None):
                 cat_map["extensions"].get(stix_type)
             )
 
-    # Left "observable-extensions" out; it has a different
-    # substructure.  A version->category->type lookup would result
-    # in another map, not a class.  So it doesn't fit the pattern.
-
     return cls

From 547273208a5f51a97d61719f79afaa86d1cb28fb Mon Sep 17 00:00:00 2001
From: Emmanuelle Vargas-Gonzalez 
Date: Thu, 15 Apr 2021 10:27:00 -0400
Subject: [PATCH 40/56] be explicit on TAXII client version being used

---
 examples/taxii_example.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/taxii_example.py b/examples/taxii_example.py
index 4bee3c0..27b83d2 100644
--- a/examples/taxii_example.py
+++ b/examples/taxii_example.py
@@ -1,4 +1,4 @@
-from taxii2client import Collection
+from taxii2client.v21 import Collection
 
 import stix2
 

From 5d83d44d0e6102768b934fedf68929f31aa268ab Mon Sep 17 00:00:00 2001
From: Emmanuelle Vargas-Gonzalez 
Date: Thu, 15 Apr 2021 12:28:58 -0400
Subject: [PATCH 41/56] adding documentation on specific lines, typos, and a
 bit of extra code

---
 stix2/base.py       | 12 +++++++++++-
 stix2/properties.py |  2 +-
 2 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/stix2/base.py b/stix2/base.py
index ac726bd..3e0d670 100644
--- a/stix2/base.py
+++ b/stix2/base.py
@@ -119,8 +119,14 @@ class _STIXBase(collections.abc.Mapping):
 
         if extra_kwargs and not self._allow_custom:
             ext_found = False
+            # This section performs a check on top-level objects that support extensions.
+            # If extra_kwargs is not empty, allow_custom False, and the extension_type is not
+            # toplevel then we raise the ExtraPropertiesError regardless.
             for key_id, ext_def in kwargs.get('extensions', {}).items():
-                if key_id.startswith('extension-definition--'):
+                if (
+                    key_id.startswith('extension-definition--') and
+                    ext_def.get('extension_type', '') == 'toplevel-property-extension'
+                ):
                     ext_found = True
                     break
             if ext_found is False:
@@ -150,6 +156,10 @@ class _STIXBase(collections.abc.Mapping):
         required_properties = set(get_required_properties(self._properties))
         missing_kwargs = required_properties - set(setting_kwargs)
         if missing_kwargs:
+            # In this scenario, we are inside within the scope of the extension.
+            # It is possible to check if this is a new Extension Class by
+            # querying "extension_type". Note: There is an API limitation currently
+            # because a toplevel-property-extension cannot validate its parent properties
             new_ext_check = (
                 bool(getattr(self, "extension_type", None))
                 and issubclass(cls, stix2.v21._Extension)
diff --git a/stix2/properties.py b/stix2/properties.py
index 1ec1311..1c0a9d8 100644
--- a/stix2/properties.py
+++ b/stix2/properties.py
@@ -117,7 +117,7 @@ class Property(object):
             creating an object with that property. No default value exists for
             these properties. (Default: ``False``)
         fixed: This provides a constant default value. Users are free to
-            provide this value explicity when constructing an object (which
+            provide this value explicitly when constructing an object (which
             allows you to copy **all** values from an existing object to a new
             object), but if the user provides a value other than the ``fixed``
             value, it will raise an error. This is semantically equivalent to

From 94859a0dafe76320fd004e6877714fbe6890f079 Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Mon, 14 Jun 2021 17:34:06 -0400
Subject: [PATCH 42/56] Fix tests, add a loophole to allow custom content

The "loophole" occurs when an object contains an unregistered top-level
extension. Since it's unregistered we can't tell if any custom
properties on the object were defined in that extension so we assume
that they are.
---
 stix2/base.py                     | 24 ++++++++++++++++++------
 stix2/test/v20/test_properties.py | 14 --------------
 stix2/test/v21/test_properties.py | 14 --------------
 3 files changed, 18 insertions(+), 34 deletions(-)

diff --git a/stix2/base.py b/stix2/base.py
index 1b3227d..4a28d8f 100644
--- a/stix2/base.py
+++ b/stix2/base.py
@@ -69,7 +69,7 @@ class _STIXBase(collections.abc.Mapping):
 
         return has_custom
 
-    # interproperty constraint methods
+    # inter-property constraint methods
 
     def _check_mutually_exclusive_properties(self, list_of_properties, at_least_one=True):
         current_properties = self.properties_populated()
@@ -119,7 +119,7 @@ class _STIXBase(collections.abc.Mapping):
 
         extra_kwargs = kwargs.keys() - self._properties.keys()
         if extra_kwargs and issubclass(cls, stix2.v21._Extension):
-            extra_kwargs = [prop for prop in extra_kwargs if prop != 'extension_type']
+            extra_kwargs = {prop for prop in extra_kwargs if prop != 'extension_type'}
 
         if extra_kwargs and not allow_custom:
             ext_found = False
@@ -136,12 +136,24 @@ class _STIXBase(collections.abc.Mapping):
             if ext_found is False:
                 raise ExtraPropertiesError(cls, extra_kwargs)
 
-        if custom_props:
-            # loophole for custom_properties...
+        extension_toplevel_properties = set()
+        unregistered_top_level_extension = False
+        if 'extensions' in kwargs and not isinstance(self, stix2.v20._STIXBase20):
+            for ext_name, ext in kwargs['extensions'].items():
+                if ext.get('extension_type', '') == 'toplevel-property-extension':
+                    registered_extensions = stix2.registry.STIX2_OBJ_MAPS['2.1'].get('extensions', {})
+                    if ext_name in registered_extensions:
+                        registered_ext_properties = registered_extensions[ext_name]._properties.keys()
+                        extension_toplevel_properties.update(registered_ext_properties)
+                    else:
+                        unregistered_top_level_extension = True
+
+        if custom_props or unregistered_top_level_extension:
+            # loophole for custom_properties and unregistered top-level extensions...
             allow_custom = True
 
-        all_custom_prop_names = extra_kwargs | custom_props.keys() - \
-            self._properties.keys()
+        all_custom_prop_names = (extra_kwargs | custom_props.keys()) - \
+            self._properties.keys() - extension_toplevel_properties
         if all_custom_prop_names:
             if not isinstance(self, stix2.v20._STIXBase20):
                 for prop_name in all_custom_prop_names:
diff --git a/stix2/test/v20/test_properties.py b/stix2/test/v20/test_properties.py
index 2429a7b..fba9203 100644
--- a/stix2/test/v20/test_properties.py
+++ b/stix2/test/v20/test_properties.py
@@ -537,20 +537,6 @@ def test_extension_property_invalid3():
     assert result[1]
 
 
-def test_extension_property_invalid_type():
-    ext_prop = ExtensionsProperty(spec_version="2.0")
-    with pytest.raises(CustomContentError) as excinfo:
-        ext_prop.clean(
-            {
-                'windows-pebinary-ext': {
-                    'pe_type': 'exe',
-                },
-            },
-            False,
-        )
-    assert "Can't parse unknown extension" in str(excinfo.value)
-
-
 def test_extension_at_least_one_property_constraint():
     with pytest.raises(AtLeastOnePropertyError):
         stix2.v20.TCPExt()
diff --git a/stix2/test/v21/test_properties.py b/stix2/test/v21/test_properties.py
index 936b7c1..4fb84ec 100644
--- a/stix2/test/v21/test_properties.py
+++ b/stix2/test/v21/test_properties.py
@@ -571,20 +571,6 @@ def test_extension_property_invalid3():
     assert result[1]
 
 
-def test_extension_property_invalid_type():
-    ext_prop = ExtensionsProperty(spec_version='2.1', enclosing_type='indicator')
-    with pytest.raises(CustomContentError) as excinfo:
-        ext_prop.clean(
-            {
-                'windows-pebinary-ext': {
-                    'pe_type': 'exe',
-                },
-            },
-            False,
-        )
-    assert "Can't parse unknown extension" in str(excinfo.value)
-
-
 def test_extension_at_least_one_property_constraint():
     with pytest.raises(AtLeastOnePropertyError):
         stix2.v21.TCPExt()

From e17f3e6c1caa2ae110eae953c7056451e760cbc7 Mon Sep 17 00:00:00 2001
From: Michael Chisholm 
Date: Fri, 25 Jun 2021 22:04:59 -0400
Subject: [PATCH 43/56] Add constants to the v21 vocab module for
 extension-type-enum

---
 stix2/v21/vocab.py | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/stix2/v21/vocab.py b/stix2/v21/vocab.py
index dc790d7..9443d0e 100644
--- a/stix2/v21/vocab.py
+++ b/stix2/v21/vocab.py
@@ -87,6 +87,22 @@ ENCRYPTION_ALGORITHM = [
 ]
 
 
+EXTENSION_TYPE_NEW_SDO = "new-sdo"
+EXTENSION_TYPE_NEW_SCO = "new-sco"
+EXTENSION_TYPE_NEW_SRO = "new-sro"
+EXTENSION_TYPE_PROPERTY_EXTENSION = "property-extension"
+EXTENSION_TYPE_TOPLEVEL_PROPERTY_EXTENSION = "toplevel-property-extension"
+
+
+EXTENSION_TYPE = [
+    EXTENSION_TYPE_NEW_SDO,
+    EXTENSION_TYPE_NEW_SCO,
+    EXTENSION_TYPE_NEW_SRO,
+    EXTENSION_TYPE_PROPERTY_EXTENSION,
+    EXTENSION_TYPE_TOPLEVEL_PROPERTY_EXTENSION
+]
+
+
 GROUPING_CONTEXT_SUSPICIOUS_ACTIVITY = "suspicious-activity"
 GROUPING_CONTEXT_MALWARE_ANALYSIS = "malware-analysis"
 GROUPING_CONTEXT_UNSPECIFIED = "unspecified"

From 47b864cb4d0b3dc00f0be487cbb0e2fc6f17f341 Mon Sep 17 00:00:00 2001
From: Michael Chisholm 
Date: Fri, 25 Jun 2021 22:06:52 -0400
Subject: [PATCH 44/56] Fix v21 ExtensionDefinition class to use the enum
 constant from the vocab module for extension-type-enum, instead of
 replicating that list.

---
 stix2/v21/common.py | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/stix2/v21/common.py b/stix2/v21/common.py
index b512c88..7c9cd70 100644
--- a/stix2/v21/common.py
+++ b/stix2/v21/common.py
@@ -14,7 +14,7 @@ from ..properties import (
 )
 from ..utils import NOW, _get_dict
 from .base import _STIXBase21
-from .vocab import HASHING_ALGORITHM
+from .vocab import HASHING_ALGORITHM, EXTENSION_TYPE
 
 
 class ExternalReference(_STIXBase21):
@@ -126,13 +126,7 @@ class ExtensionDefinition(_STIXBase21):
         (
             'extension_types', ListProperty(
                 EnumProperty(
-                    allowed=[
-                            'new-sdo',
-                            'new-sco',
-                            'new-sro',
-                            'property-extension',
-                            'toplevel-property-extension',
-                    ],
+                    allowed=EXTENSION_TYPE,
                 ), required=True,
             ),
         ),

From 70718063d3a85cbae0664316b4f46f6bb7af0233 Mon Sep 17 00:00:00 2001
From: Michael Chisholm 
Date: Fri, 25 Jun 2021 22:10:31 -0400
Subject: [PATCH 45/56] Remove relics of the old way customization was handled
 and enforced, from _STIXBase20 and _STIXBase21.  That awkwardness is no
 longer necessary.

---
 stix2/v20/base.py | 5 +----
 stix2/v21/base.py | 6 +-----
 2 files changed, 2 insertions(+), 9 deletions(-)

diff --git a/stix2/v20/base.py b/stix2/v20/base.py
index a0d65c6..b5437ca 100644
--- a/stix2/v20/base.py
+++ b/stix2/v20/base.py
@@ -10,10 +10,7 @@ class _STIXBase20(_STIXBase):
 
 
 class _Observable(_Observable, _STIXBase20):
-
-    def __init__(self, **kwargs):
-        self._properties['extensions'].allow_custom = kwargs.get('allow_custom', False)
-        super(_Observable, self).__init__(**kwargs)
+    pass
 
 
 class _Extension(_Extension, _STIXBase20):
diff --git a/stix2/v21/base.py b/stix2/v21/base.py
index 8598a83..90b9fd2 100644
--- a/stix2/v21/base.py
+++ b/stix2/v21/base.py
@@ -6,11 +6,7 @@ from ..base import (
 
 
 class _STIXBase21(_STIXBase):
-
-    def __init__(self, **kwargs):
-        if 'extensions' in self._properties:
-            self._properties['extensions'].allow_custom = kwargs.get('allow_custom', False)
-        super(_STIXBase21, self).__init__(**kwargs)
+    pass
 
 
 class _Observable(_Observable, _STIXBase21):

From c7b48402329d10b25522c95fa7b717ee55b81fff Mon Sep 17 00:00:00 2001
From: Michael Chisholm 
Date: Fri, 25 Jun 2021 22:20:02 -0400
Subject: [PATCH 46/56] Code style and behavioral improvements in
 ExtensionsProperty:

- Changed an error message "Cannot determine extension type."
At that point in the code, we in fact have a registered
extension type and the class for it, so it didn't seem to make
any sense.  I changed it to say that an extension of that type
couldn't be created from type of value given.  Because this is
really about typing (in both the old and new code), I also
changed the exception class to TypeError.

- Changed customization behavior: unregistered
"extension-definition--" style extensions are no longer
considered custom.  Their mere presence can no longer be
considered a customization, since it doesn't fit the
criteria, and in fact the extension mechanism is supposed to
supplant the old customization system anyway.  Unregistered
extensions of other types are still considered custom.

- Fixed up unit tests according to the exception message
change.

- Added a unit test for unregistered "extension-definition--"
style extensions.
---
 stix2/properties.py           | 39 +++++++++++++++++++++++------------
 stix2/test/v20/test_custom.py |  2 +-
 stix2/test/v21/test_custom.py | 37 ++++++++++++++++++++++++++++++++-
 3 files changed, 63 insertions(+), 15 deletions(-)

diff --git a/stix2/properties.py b/stix2/properties.py
index 0f68f6a..e63206a 100644
--- a/stix2/properties.py
+++ b/stix2/properties.py
@@ -14,7 +14,7 @@ import stix2.hashes
 from .base import _STIXBase
 from .exceptions import CustomContentError, DictionaryKeyError, STIXError
 from .parsing import parse, parse_observable
-from .registry import STIX2_OBJ_MAPS
+from .registry import class_for_type
 from .utils import (
     STIXTypeClass, _get_dict, get_class_hierarchy_names, get_type_from_id,
     is_object, is_stix_type, parse_into_datetime, to_enum,
@@ -781,17 +781,21 @@ class ExtensionsProperty(DictionaryProperty):
             raise ValueError("The extensions property must contain a dictionary")
 
         has_custom = False
-        extension_type_map = STIX2_OBJ_MAPS[self.spec_version].get('extensions', {})
         for key, subvalue in dictified.items():
-            if key in extension_type_map:
-                cls = extension_type_map[key]
-                if type(subvalue) is dict:
+            cls = class_for_type(key, self.spec_version, "extensions")
+            if cls:
+                if isinstance(subvalue, dict):
                     ext = cls(allow_custom=allow_custom, **subvalue)
-                elif type(subvalue) is cls:
-                    # If already an instance of an _Extension class, assume it's valid
+                elif isinstance(subvalue, cls):
+                    # If already an instance of the registered class, assume
+                    # it's valid
                     ext = subvalue
                 else:
-                    raise ValueError("Cannot determine extension type.")
+                    raise TypeError(
+                        "Can't create extension '{}' from {}.".format(
+                            key, type(subvalue)
+                        )
+                    )
 
                 has_custom = has_custom or ext.has_custom
 
@@ -805,15 +809,24 @@ class ExtensionsProperty(DictionaryProperty):
                 dictified[key] = ext
 
             else:
-                if allow_custom:
+                # If an unregistered "extension-definition--" style extension,
+                # we don't know what's supposed to be in it, so we can't
+                # determine whether there's anything custom.  So, assume there
+                # are no customizations.  If it's a different type of extension,
+                # non-registration implies customization (since all spec-defined
+                # extensions should be pre-registered with the library).
+
+                if key.startswith('extension-definition--'):
+                    _validate_id(
+                        key, self.spec_version, 'extension-definition--'
+                    )
+                elif allow_custom:
                     has_custom = True
-                    dictified[key] = subvalue
-                elif key.startswith('extension-definition--'):
-                    _validate_id(key, '2.1', 'extension-definition')
-                    dictified[key] = subvalue
                 else:
                     raise CustomContentError("Can't parse unknown extension type: {}".format(key))
 
+                dictified[key] = subvalue
+
         return dictified, has_custom
 
 
diff --git a/stix2/test/v20/test_custom.py b/stix2/test/v20/test_custom.py
index fd45ff4..4647167 100644
--- a/stix2/test/v20/test_custom.py
+++ b/stix2/test/v20/test_custom.py
@@ -795,7 +795,7 @@ def test_custom_extension_wrong_observable_type():
             },
         )
 
-    assert 'Cannot determine extension type' in excinfo.value.reason
+    assert "Can't create extension 'ntfs-ext' from" in excinfo.value.reason
 
 
 @pytest.mark.parametrize(
diff --git a/stix2/test/v21/test_custom.py b/stix2/test/v21/test_custom.py
index 4f60129..ff0464b 100644
--- a/stix2/test/v21/test_custom.py
+++ b/stix2/test/v21/test_custom.py
@@ -984,7 +984,7 @@ def test_custom_extension_wrong_observable_type():
             },
         )
 
-    assert 'Cannot determine extension type' in excinfo.value.reason
+    assert "Can't create extension 'ntfs-ext'" in excinfo.value.reason
 
 
 @pytest.mark.parametrize(
@@ -1228,6 +1228,41 @@ def test_parse_observable_with_unregistered_custom_extension(data):
     assert not isinstance(parsed_ob['extensions']['x-foobar-ext'], stix2.base._STIXBase)
 
 
+def test_unregistered_new_style_extension():
+
+    f_dict = {
+        "type": "file",
+        "name": "foo.txt",
+        "extensions": {
+            "extension-definition--31adb724-a9a4-44b6-8ec2-fd4b181c9507": {
+                "extension-type": "property-extension",
+                "a": 1,
+                "b": True
+            }
+        }
+    }
+
+    f = stix2.parse(f_dict, allow_custom=False)
+
+    assert f.extensions[
+        "extension-definition--31adb724-a9a4-44b6-8ec2-fd4b181c9507"
+    ]["a"] == 1
+    assert f.extensions[
+        "extension-definition--31adb724-a9a4-44b6-8ec2-fd4b181c9507"
+    ]["b"]
+    assert not f.has_custom
+
+    f = stix2.parse(f_dict, allow_custom=True)
+
+    assert f.extensions[
+        "extension-definition--31adb724-a9a4-44b6-8ec2-fd4b181c9507"
+    ]["a"] == 1
+    assert f.extensions[
+        "extension-definition--31adb724-a9a4-44b6-8ec2-fd4b181c9507"
+    ]["b"]
+    assert not f.has_custom
+
+
 def test_register_custom_object():
     # Not the way to register custom object.
     class CustomObject2(object):

From d87718c15cabc17a247791625c719a52eed6290b Mon Sep 17 00:00:00 2001
From: Michael Chisholm 
Date: Wed, 30 Jun 2021 17:50:00 -0400
Subject: [PATCH 47/56] Bug fixes, hackage removal, and some pre-commit
 stylistic fixes.

- Fixed bugged logic in _STIXBase._check_at_least_one_property(),
and revamped the code to be simpler and clearer.

- Changed custom extension registration to auto-create an
"extension_type" property based on the attribute of that
name on the custom class, if present.

- The custom extension registration change above uncovered
what seemed like a bug in a unit test: a custom extension
was registered, but it was not given an extension type.  The
test used the extension as extension_type="property-extension";
this now causes a standard error about an extra property.  I
fixed the test to assign the custom extension the proper type.
---
 stix2/base.py                        | 40 ++++++++++++++++++----------
 stix2/properties.py                  |  6 ++---
 stix2/test/v20/test_observed_data.py |  1 +
 stix2/test/v21/test_custom.py        |  8 +++---
 stix2/test/v21/test_observed_data.py |  4 +--
 stix2/v21/base.py                    | 11 --------
 stix2/v21/common.py                  |  2 +-
 stix2/v21/observables.py             | 19 ++++++++++++-
 stix2/v21/vocab.py                   |  2 +-
 9 files changed, 56 insertions(+), 37 deletions(-)

diff --git a/stix2/base.py b/stix2/base.py
index 4a28d8f..5b88998 100644
--- a/stix2/base.py
+++ b/stix2/base.py
@@ -78,20 +78,34 @@ class _STIXBase(collections.abc.Mapping):
         if count > 1 or (at_least_one and count == 0):
             raise MutuallyExclusivePropertiesError(self.__class__, list_of_properties)
 
-    def _check_at_least_one_property(self, list_of_properties=None):
-        if not list_of_properties:
-            list_of_properties = sorted(self.__class__._properties.keys())
+    def _check_at_least_one_property(self, properties_checked=None):
+        """
+        Check whether one or more of the given properties are present.
+
+        :param properties_checked: An iterable of the names of the properties
+            of interest, or None to check against a default list.  The default
+            list includes all properties defined on the object, with some
+            hard-coded exceptions.
+        :raises AtLeastOnePropertyError: If none of the given properties are
+            present.
+        """
+        if properties_checked is None:
+            property_exceptions = {"extensions", "type"}
             if isinstance(self, _Observable):
-                props_to_remove = {"type", "id", "defanged", "spec_version"}
-            else:
-                props_to_remove = {"type"}
+                property_exceptions |= {"id", "defanged", "spec_version"}
 
-            list_of_properties = [prop for prop in list_of_properties if prop not in props_to_remove]
-        current_properties = self.properties_populated()
-        list_of_properties_populated = set(list_of_properties).intersection(current_properties)
+            properties_checked = self._properties.keys() - property_exceptions
 
-        if list_of_properties and (not list_of_properties_populated or list_of_properties_populated == {'extensions'}):
-            raise AtLeastOnePropertyError(self.__class__, list_of_properties)
+        elif not isinstance(properties_checked, set):
+            properties_checked = set(properties_checked)
+
+        if properties_checked:
+            properties_checked_assigned = properties_checked & self.keys()
+
+            if not properties_checked_assigned:
+                raise AtLeastOnePropertyError(
+                    self.__class__, properties_checked
+                )
 
     def _check_properties_dependency(self, list_of_properties, list_of_dependent_properties):
         failed_dependency_pairs = []
@@ -118,8 +132,6 @@ class _STIXBase(collections.abc.Mapping):
             raise ValueError("'custom_properties' must be a dictionary")
 
         extra_kwargs = kwargs.keys() - self._properties.keys()
-        if extra_kwargs and issubclass(cls, stix2.v21._Extension):
-            extra_kwargs = {prop for prop in extra_kwargs if prop != 'extension_type'}
 
         if extra_kwargs and not allow_custom:
             ext_found = False
@@ -129,7 +141,7 @@ class _STIXBase(collections.abc.Mapping):
             for key_id, ext_def in kwargs.get('extensions', {}).items():
                 if (
                     key_id.startswith('extension-definition--') and
-                    ext_def.get('extension_type', '') == 'toplevel-property-extension'
+                    ext_def.get('extension_type') == 'toplevel-property-extension'
                 ):
                     ext_found = True
                     break
diff --git a/stix2/properties.py b/stix2/properties.py
index e63206a..be1c4b6 100644
--- a/stix2/properties.py
+++ b/stix2/properties.py
@@ -793,8 +793,8 @@ class ExtensionsProperty(DictionaryProperty):
                 else:
                     raise TypeError(
                         "Can't create extension '{}' from {}.".format(
-                            key, type(subvalue)
-                        )
+                            key, type(subvalue),
+                        ),
                     )
 
                 has_custom = has_custom or ext.has_custom
@@ -818,7 +818,7 @@ class ExtensionsProperty(DictionaryProperty):
 
                 if key.startswith('extension-definition--'):
                     _validate_id(
-                        key, self.spec_version, 'extension-definition--'
+                        key, self.spec_version, 'extension-definition--',
                     )
                 elif allow_custom:
                     has_custom = True
diff --git a/stix2/test/v20/test_observed_data.py b/stix2/test/v20/test_observed_data.py
index 79ed6c4..196fb46 100644
--- a/stix2/test/v20/test_observed_data.py
+++ b/stix2/test/v20/test_observed_data.py
@@ -1117,6 +1117,7 @@ def test_process_example_empty_error():
     assert excinfo.value.cls == stix2.v20.Process
     properties_of_process = list(stix2.v20.Process._properties.keys())
     properties_of_process.remove("type")
+    properties_of_process.remove("extensions")
     assert excinfo.value.properties == sorted(properties_of_process)
     msg = "At least one of the ({1}) properties for {0} must be populated."
     msg = msg.format(
diff --git a/stix2/test/v21/test_custom.py b/stix2/test/v21/test_custom.py
index ff0464b..1b3166f 100644
--- a/stix2/test/v21/test_custom.py
+++ b/stix2/test/v21/test_custom.py
@@ -1237,9 +1237,9 @@ def test_unregistered_new_style_extension():
             "extension-definition--31adb724-a9a4-44b6-8ec2-fd4b181c9507": {
                 "extension-type": "property-extension",
                 "a": 1,
-                "b": True
-            }
-        }
+                "b": True,
+            },
+        },
     }
 
     f = stix2.parse(f_dict, allow_custom=False)
@@ -1508,7 +1508,7 @@ def test_registered_embedded_extension_passes_with_allow_custom_false():
         ],
     )
     class ExtensionFoo1:
-        pass
+        extension_type = "property-extension"
 
     indicator = stix2.v21.Indicator(
         id='indicator--e97bfccf-8970-4a3c-9cd1-5b5b97ed5d0c',
diff --git a/stix2/test/v21/test_observed_data.py b/stix2/test/v21/test_observed_data.py
index 28c40c0..d2ccec4 100644
--- a/stix2/test/v21/test_observed_data.py
+++ b/stix2/test/v21/test_observed_data.py
@@ -1218,8 +1218,8 @@ def test_process_example_empty_error():
         stix2.v21.Process()
 
     assert excinfo.value.cls == stix2.v21.Process
-    properties_of_process = list(stix2.v21.Process._properties.keys())
-    properties_of_process = [prop for prop in properties_of_process if prop not in ["type", "id", "defanged", "spec_version"]]
+    properties_of_process = stix2.v21.Process._properties.keys()
+    properties_of_process -= {"type", "id", "defanged", "spec_version", "extensions"}
     assert excinfo.value.properties == sorted(properties_of_process)
     msg = "At least one of the ({1}) properties for {0} must be populated."
     msg = msg.format(
diff --git a/stix2/v21/base.py b/stix2/v21/base.py
index 90b9fd2..3878b79 100644
--- a/stix2/v21/base.py
+++ b/stix2/v21/base.py
@@ -29,17 +29,6 @@ class _Observable(_Observable, _STIXBase21):
 class _Extension(_Extension, _STIXBase21):
     extension_type = None
 
-    def __init__(self, **kwargs):
-        super(_Extension, self).__init__(**kwargs)
-        if getattr(self, "extension_type", None):
-            self._inner["extension_type"] = self.extension_type
-
-    def _check_at_least_one_property(self, list_of_properties=None):
-        new_ext_check = getattr(self, "extension_type", None)
-
-        if new_ext_check is None:
-            super(_Extension, self)._check_at_least_one_property(list_of_properties=list_of_properties)
-
 
 class _DomainObject(_DomainObject, _STIXBase21):
     pass
diff --git a/stix2/v21/common.py b/stix2/v21/common.py
index 7c9cd70..e62bdae 100644
--- a/stix2/v21/common.py
+++ b/stix2/v21/common.py
@@ -14,7 +14,7 @@ from ..properties import (
 )
 from ..utils import NOW, _get_dict
 from .base import _STIXBase21
-from .vocab import HASHING_ALGORITHM, EXTENSION_TYPE
+from .vocab import EXTENSION_TYPE, HASHING_ALGORITHM
 
 
 class ExternalReference(_STIXBase21):
diff --git a/stix2/v21/observables.py b/stix2/v21/observables.py
index 4e5876d..44b346c 100644
--- a/stix2/v21/observables.py
+++ b/stix2/v21/observables.py
@@ -6,6 +6,7 @@ _Observable and do not have a ``_type`` attribute.
 """
 
 from collections import OrderedDict
+from collections.abc import Mapping
 import itertools
 
 from ..custom import _custom_extension_builder, _custom_observable_builder
@@ -20,7 +21,7 @@ from ..properties import (
 from .base import _Extension, _Observable, _STIXBase21
 from .common import GranularMarking
 from .vocab import (
-    ACCOUNT_TYPE, ENCRYPTION_ALGORITHM, HASHING_ALGORITHM,
+    ACCOUNT_TYPE, ENCRYPTION_ALGORITHM, EXTENSION_TYPE, HASHING_ALGORITHM,
     NETWORK_SOCKET_ADDRESS_FAMILY, NETWORK_SOCKET_TYPE,
     WINDOWS_INTEGRITY_LEVEL, WINDOWS_PEBINARY_TYPE, WINDOWS_REGISTRY_DATATYPE,
     WINDOWS_SERVICE_START_TYPE, WINDOWS_SERVICE_STATUS, WINDOWS_SERVICE_TYPE,
@@ -901,5 +902,21 @@ def CustomExtension(type='x-custom-observable-ext', properties=None):
     """Custom STIX Object Extension decorator.
     """
     def wrapper(cls):
+
+        # Auto-create an "extension_type" property from the class attribute, if
+        # it exists.
+        extension_type = getattr(cls, "extension_type", None)
+        if extension_type:
+            extension_type_prop = EnumProperty(
+                EXTENSION_TYPE,
+                required=False,
+                fixed=extension_type,
+            )
+
+            if isinstance(properties, Mapping):
+                properties["extension_type"] = extension_type_prop
+            else:
+                properties.append(("extension_type", extension_type_prop))
+
         return _custom_extension_builder(cls, type, properties, '2.1', _Extension)
     return wrapper
diff --git a/stix2/v21/vocab.py b/stix2/v21/vocab.py
index 9443d0e..303692e 100644
--- a/stix2/v21/vocab.py
+++ b/stix2/v21/vocab.py
@@ -99,7 +99,7 @@ EXTENSION_TYPE = [
     EXTENSION_TYPE_NEW_SCO,
     EXTENSION_TYPE_NEW_SRO,
     EXTENSION_TYPE_PROPERTY_EXTENSION,
-    EXTENSION_TYPE_TOPLEVEL_PROPERTY_EXTENSION
+    EXTENSION_TYPE_TOPLEVEL_PROPERTY_EXTENSION,
 ]
 
 

From ac8e46f491f490ec3bf79c3660fffba400d793a1 Mon Sep 17 00:00:00 2001
From: Michael Chisholm 
Date: Wed, 30 Jun 2021 20:20:29 -0400
Subject: [PATCH 48/56] Improve customization detection in the face of
 toplevel-property-extension style extensions.  If an unregistered extension
 of that type is encountered, all unrecognized toplevel props will now be
 considered extension properties (not custom). It will no longer turn on the
 allow_custom flag, which would allow customizations everywhere.

Also, if all extensions of the aforementioned type are registered,
their properties are now used to properly distinguish between
extension and custom properties.  There need not be any ambiguity
in that case.
---
 stix2/base.py     | 62 ++++++++++++++++++++++++++---------------------
 stix2/registry.py |  3 ++-
 2 files changed, 36 insertions(+), 29 deletions(-)

diff --git a/stix2/base.py b/stix2/base.py
index 5b88998..05b0354 100644
--- a/stix2/base.py
+++ b/stix2/base.py
@@ -18,6 +18,7 @@ from .exceptions import (
 )
 from .markings import _MarkingsMixin
 from .markings.utils import validate
+from .registry import class_for_type
 from .serialization import STIXJSONEncoder, fp_serialize, serialize
 from .utils import NOW, PREFIX_21_REGEX, get_timestamp
 from .versioning import new_version as _new_version
@@ -126,46 +127,51 @@ class _STIXBase(collections.abc.Mapping):
         # Use the same timestamp for any auto-generated datetimes
         self.__now = get_timestamp()
 
-        # Detect any keyword arguments not allowed for a specific type
         custom_props = kwargs.pop('custom_properties', {})
         if custom_props and not isinstance(custom_props, dict):
             raise ValueError("'custom_properties' must be a dictionary")
 
+        # Detect any keyword arguments not allowed for a specific type.
+        # In STIX 2.1, this is complicated by "toplevel-property-extension"
+        # type extensions, which can add extra properties which are *not*
+        # considered custom.
         extra_kwargs = kwargs.keys() - self._properties.keys()
 
-        if extra_kwargs and not allow_custom:
-            ext_found = False
-            # This section performs a check on top-level objects that support extensions.
-            # If extra_kwargs is not empty, allow_custom False, and the extension_type is not
-            # toplevel then we raise the ExtraPropertiesError regardless.
-            for key_id, ext_def in kwargs.get('extensions', {}).items():
-                if (
-                    key_id.startswith('extension-definition--') and
-                    ext_def.get('extension_type') == 'toplevel-property-extension'
-                ):
-                    ext_found = True
-                    break
-            if ext_found is False:
-                raise ExtraPropertiesError(cls, extra_kwargs)
+        extensions = kwargs.get("extensions")
+        if extensions:
+            has_unregistered_toplevel_extension = False
+            registered_toplevel_extension_props = set()
 
-        extension_toplevel_properties = set()
-        unregistered_top_level_extension = False
-        if 'extensions' in kwargs and not isinstance(self, stix2.v20._STIXBase20):
-            for ext_name, ext in kwargs['extensions'].items():
-                if ext.get('extension_type', '') == 'toplevel-property-extension':
-                    registered_extensions = stix2.registry.STIX2_OBJ_MAPS['2.1'].get('extensions', {})
-                    if ext_name in registered_extensions:
-                        registered_ext_properties = registered_extensions[ext_name]._properties.keys()
-                        extension_toplevel_properties.update(registered_ext_properties)
+            for ext_id, ext in extensions.items():
+                if ext.get("extension_type") == "toplevel-property-extension":
+                    registered_ext_class = class_for_type(
+                        ext_id, "2.1", "extensions"
+                    )
+                    if registered_ext_class:
+                        registered_toplevel_extension_props |= \
+                            registered_ext_class._properties.keys()
                     else:
-                        unregistered_top_level_extension = True
+                        has_unregistered_toplevel_extension = True
 
-        if custom_props or unregistered_top_level_extension:
-            # loophole for custom_properties and unregistered top-level extensions...
+            if has_unregistered_toplevel_extension:
+                # Must assume all extras are extension properties, not custom.
+                extra_kwargs.clear()
+
+            else:
+                # All toplevel property extensions (if any) have been
+                # registered.  So we can tell what their properties are and
+                # treat only those as not custom.
+                extra_kwargs -= registered_toplevel_extension_props
+
+        if extra_kwargs and not allow_custom:
+            raise ExtraPropertiesError(cls, extra_kwargs)
+
+        if custom_props:
+            # loophole for custom_properties...
             allow_custom = True
 
         all_custom_prop_names = (extra_kwargs | custom_props.keys()) - \
-            self._properties.keys() - extension_toplevel_properties
+            self._properties.keys()
         if all_custom_prop_names:
             if not isinstance(self, stix2.v20._STIXBase20):
                 for prop_name in all_custom_prop_names:
diff --git a/stix2/registry.py b/stix2/registry.py
index 5ce7e24..7086497 100644
--- a/stix2/registry.py
+++ b/stix2/registry.py
@@ -49,7 +49,8 @@ def class_for_type(stix_type, stix_version, category=None):
     Get the registered class which implements a particular STIX type for a
     particular STIX version.
 
-    :param stix_type: A STIX type as a string
+    :param stix_type: A STIX type as a string, or for extension-definition
+        style extensions, the STIX ID of the definition.
     :param stix_version: A STIX version as a string, e.g. "2.1"
     :param category: An optional "category" value, which is just used directly
         as a second key after the STIX version, and depends on how the types

From 6fc39a70eaad0a69fe31874a07921d47191d5cb3 Mon Sep 17 00:00:00 2001
From: Michael Chisholm 
Date: Thu, 1 Jul 2021 20:08:09 -0400
Subject: [PATCH 49/56] pre-commit stylistic fixes

---
 stix2/base.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/stix2/base.py b/stix2/base.py
index 05b0354..7c68002 100644
--- a/stix2/base.py
+++ b/stix2/base.py
@@ -105,7 +105,7 @@ class _STIXBase(collections.abc.Mapping):
 
             if not properties_checked_assigned:
                 raise AtLeastOnePropertyError(
-                    self.__class__, properties_checked
+                    self.__class__, properties_checked,
                 )
 
     def _check_properties_dependency(self, list_of_properties, list_of_dependent_properties):
@@ -145,7 +145,7 @@ class _STIXBase(collections.abc.Mapping):
             for ext_id, ext in extensions.items():
                 if ext.get("extension_type") == "toplevel-property-extension":
                     registered_ext_class = class_for_type(
-                        ext_id, "2.1", "extensions"
+                        ext_id, "2.1", "extensions",
                     )
                     if registered_ext_class:
                         registered_toplevel_extension_props |= \

From b9eba77008aedffbf91dff829ad3da95526760ee Mon Sep 17 00:00:00 2001
From: Michael Chisholm 
Date: Fri, 2 Jul 2021 20:54:54 -0400
Subject: [PATCH 50/56] Move the CustomExtension decorator from the
 v21.observables module to v21.common.  Custom extensions are not specific to
 SCOs, so I don't know why it was in that module.  Now, ExtensionDefinition
 and CustomExtension are together in the same module, just like
 MarkingDefinition and CustomMarking are together.  Made sense to me.

---
 stix2/v21/__init__.py    |  4 ++--
 stix2/v21/common.py      | 30 ++++++++++++++++++++++++++++--
 stix2/v21/observables.py | 29 +++--------------------------
 stix2/v21/sdo.py         |  7 ++++---
 4 files changed, 37 insertions(+), 33 deletions(-)

diff --git a/stix2/v21/__init__.py b/stix2/v21/__init__.py
index 977fdf7..ece8b36 100644
--- a/stix2/v21/__init__.py
+++ b/stix2/v21/__init__.py
@@ -19,13 +19,13 @@ from .base import (
 )
 from .bundle import Bundle
 from .common import (
-    TLP_AMBER, TLP_GREEN, TLP_RED, TLP_WHITE, CustomMarking,
+    TLP_AMBER, TLP_GREEN, TLP_RED, TLP_WHITE, CustomExtension, CustomMarking,
     ExtensionDefinition, ExternalReference, GranularMarking, KillChainPhase,
     LanguageContent, MarkingDefinition, StatementMarking, TLPMarking,
 )
 from .observables import (
     URL, AlternateDataStream, ArchiveExt, Artifact, AutonomousSystem,
-    CustomExtension, CustomObservable, Directory, DomainName, EmailAddress,
+    CustomObservable, Directory, DomainName, EmailAddress,
     EmailMessage, EmailMIMEComponent, File, HTTPRequestExt, ICMPExt,
     IPv4Address, IPv6Address, MACAddress, Mutex, NetworkTraffic, NTFSExt,
     PDFExt, Process, RasterImageExt, SocketExt, Software, TCPExt,
diff --git a/stix2/v21/common.py b/stix2/v21/common.py
index e62bdae..6b2922b 100644
--- a/stix2/v21/common.py
+++ b/stix2/v21/common.py
@@ -1,8 +1,10 @@
 """STIX 2.1 Common Data Types and Properties."""
 
 from collections import OrderedDict
+from collections.abc import Mapping
 
-from ..custom import _custom_marking_builder
+from . import _Extension
+from ..custom import _custom_marking_builder, _custom_extension_builder
 from ..exceptions import InvalidValueError, PropertyPresenceError
 from ..markings import _MarkingsMixin
 from ..markings.utils import check_tlp_marking
@@ -139,6 +141,30 @@ class ExtensionDefinition(_STIXBase21):
     ])
 
 
+def CustomExtension(type='x-custom-ext', properties=None):
+    """Custom STIX Object Extension decorator.
+    """
+    def wrapper(cls):
+
+        # Auto-create an "extension_type" property from the class attribute, if
+        # it exists.
+        extension_type = getattr(cls, "extension_type", None)
+        if extension_type:
+            extension_type_prop = EnumProperty(
+                EXTENSION_TYPE,
+                required=False,
+                fixed=extension_type,
+            )
+
+            if isinstance(properties, Mapping):
+                properties["extension_type"] = extension_type_prop
+            else:
+                properties.append(("extension_type", extension_type_prop))
+
+        return _custom_extension_builder(cls, type, properties, '2.1', _Extension)
+    return wrapper
+
+
 class TLPMarking(_STIXBase21):
     """For more detailed information on this object's properties, see
     `the STIX 2.1 specification `__.
@@ -260,7 +286,7 @@ def CustomMarking(type='x-custom-marking', properties=None, extension_name=None)
         if extension_name:
             from . import observables
 
-            @observables.CustomExtension(type=extension_name, properties=properties)
+            @CustomExtension(type=extension_name, properties=properties)
             class NameExtension:
                 extension_type = 'property-extension'
 
diff --git a/stix2/v21/observables.py b/stix2/v21/observables.py
index 44b346c..943e59f 100644
--- a/stix2/v21/observables.py
+++ b/stix2/v21/observables.py
@@ -6,10 +6,9 @@ _Observable and do not have a ``_type`` attribute.
 """
 
 from collections import OrderedDict
-from collections.abc import Mapping
 import itertools
 
-from ..custom import _custom_extension_builder, _custom_observable_builder
+from ..custom import _custom_observable_builder
 from ..exceptions import AtLeastOnePropertyError, DependentPropertiesError
 from ..properties import (
     BinaryProperty, BooleanProperty, DictionaryProperty,
@@ -19,9 +18,9 @@ from ..properties import (
     TypeProperty,
 )
 from .base import _Extension, _Observable, _STIXBase21
-from .common import GranularMarking
+from .common import GranularMarking, CustomExtension
 from .vocab import (
-    ACCOUNT_TYPE, ENCRYPTION_ALGORITHM, EXTENSION_TYPE, HASHING_ALGORITHM,
+    ACCOUNT_TYPE, ENCRYPTION_ALGORITHM, HASHING_ALGORITHM,
     NETWORK_SOCKET_ADDRESS_FAMILY, NETWORK_SOCKET_TYPE,
     WINDOWS_INTEGRITY_LEVEL, WINDOWS_PEBINARY_TYPE, WINDOWS_REGISTRY_DATATYPE,
     WINDOWS_SERVICE_START_TYPE, WINDOWS_SERVICE_STATUS, WINDOWS_SERVICE_TYPE,
@@ -898,25 +897,3 @@ def CustomObservable(type='x-custom-observable', properties=None, id_contrib_pro
     return wrapper
 
 
-def CustomExtension(type='x-custom-observable-ext', properties=None):
-    """Custom STIX Object Extension decorator.
-    """
-    def wrapper(cls):
-
-        # Auto-create an "extension_type" property from the class attribute, if
-        # it exists.
-        extension_type = getattr(cls, "extension_type", None)
-        if extension_type:
-            extension_type_prop = EnumProperty(
-                EXTENSION_TYPE,
-                required=False,
-                fixed=extension_type,
-            )
-
-            if isinstance(properties, Mapping):
-                properties["extension_type"] = extension_type_prop
-            else:
-                properties.append(("extension_type", extension_type_prop))
-
-        return _custom_extension_builder(cls, type, properties, '2.1', _Extension)
-    return wrapper
diff --git a/stix2/v21/sdo.py b/stix2/v21/sdo.py
index bfa741f..d4ff34e 100644
--- a/stix2/v21/sdo.py
+++ b/stix2/v21/sdo.py
@@ -7,7 +7,6 @@ import warnings
 
 from stix2patterns.validator import run_validator
 
-from . import observables
 from ..custom import _custom_object_builder
 from ..exceptions import (
     InvalidValueError, PropertyPresenceError, STIXDeprecationWarning,
@@ -20,7 +19,9 @@ from ..properties import (
 )
 from ..utils import NOW
 from .base import _DomainObject
-from .common import ExternalReference, GranularMarking, KillChainPhase
+from .common import (
+    CustomExtension, ExternalReference, GranularMarking, KillChainPhase
+)
 from .vocab import (
     ATTACK_MOTIVATION, ATTACK_RESOURCE_LEVEL, GROUPING_CONTEXT, IDENTITY_CLASS,
     IMPLEMENTATION_LANGUAGE, INDICATOR_TYPE, INDUSTRY_SECTOR,
@@ -858,7 +859,7 @@ def CustomObject(type='x-custom-type', properties=None, extension_name=None, is_
             ]),
         )
         if extension_name:
-            @observables.CustomExtension(type=extension_name, properties=extension_properties)
+            @CustomExtension(type=extension_name, properties=extension_properties)
             class NameExtension:
                 if is_sdo:
                     extension_type = 'new-sdo'

From 93d2524d45b964ec4d16d76bd7d3c7a10604a513 Mon Sep 17 00:00:00 2001
From: Michael Chisholm 
Date: Fri, 2 Jul 2021 21:10:52 -0400
Subject: [PATCH 51/56] Remove excessive nested lists from CusomObservable
 decorator. Remove iterable chaining from CustomObject decorator.  If all
 values are guaranteed lists now, it no longer makes sense to use it.  Simpler
 and clearer to use plain old list concatenation.

---
 stix2/v21/observables.py | 22 +++++++++++--------
 stix2/v21/sdo.py         | 46 +++++++++++++++++++---------------------
 2 files changed, 35 insertions(+), 33 deletions(-)

diff --git a/stix2/v21/observables.py b/stix2/v21/observables.py
index 943e59f..414a0b0 100644
--- a/stix2/v21/observables.py
+++ b/stix2/v21/observables.py
@@ -873,16 +873,20 @@ def CustomObservable(type='x-custom-observable', properties=None, id_contrib_pro
     """
     def wrapper(cls):
         _properties = list(
-            itertools.chain.from_iterable([
-                [('type', TypeProperty(type, spec_version='2.1'))],
-                [('spec_version', StringProperty(fixed='2.1'))],
-                [('id', IDProperty(type, spec_version='2.1'))],
+            itertools.chain(
+                [
+                    ('type', TypeProperty(type, spec_version='2.1')),
+                    ('spec_version', StringProperty(fixed='2.1')),
+                    ('id', IDProperty(type, spec_version='2.1'))
+                ],
                 properties,
-                [('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1')))],
-                [('granular_markings', ListProperty(GranularMarking))],
-                [('defanged', BooleanProperty(default=lambda: False))],
-                [('extensions', ExtensionsProperty(spec_version='2.1'))],
-            ]),
+                [
+                    ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))),
+                    ('granular_markings', ListProperty(GranularMarking)),
+                    ('defanged', BooleanProperty(default=lambda: False)),
+                    ('extensions', ExtensionsProperty(spec_version='2.1')),
+                ],
+            ),
         )
         if extension_name:
             @CustomExtension(type=extension_name, properties=properties)
diff --git a/stix2/v21/sdo.py b/stix2/v21/sdo.py
index d4ff34e..5782252 100644
--- a/stix2/v21/sdo.py
+++ b/stix2/v21/sdo.py
@@ -1,7 +1,6 @@
 """STIX 2.1 Domain Objects."""
 
 from collections import OrderedDict
-import itertools
 from urllib.parse import quote_plus
 import warnings
 
@@ -834,30 +833,29 @@ def CustomObject(type='x-custom-type', properties=None, extension_name=None, is_
     """
     def wrapper(cls):
         extension_properties = [x for x in properties if not x[0].startswith('x_')]
-        _properties = list(
-            itertools.chain.from_iterable([
-                [
-                    ('type', TypeProperty(type, spec_version='2.1')),
-                    ('spec_version', StringProperty(fixed='2.1')),
-                    ('id', IDProperty(type, spec_version='2.1')),
-                    ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')),
-                    ('created', TimestampProperty(default=lambda: NOW, precision='millisecond', precision_constraint='min')),
-                    ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond', precision_constraint='min')),
-                ],
-                extension_properties,
-                [
-                    ('revoked', BooleanProperty(default=lambda: False)),
-                    ('labels', ListProperty(StringProperty)),
-                    ('confidence', IntegerProperty()),
-                    ('lang', StringProperty()),
-                    ('external_references', ListProperty(ExternalReference)),
-                    ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))),
-                    ('granular_markings', ListProperty(GranularMarking)),
-                    ('extensions', ExtensionsProperty(spec_version='2.1')),
-                ],
-                sorted([x for x in properties if x[0].startswith('x_')], key=lambda x: x[0]),
-            ]),
+        _properties = (
+            [
+                ('type', TypeProperty(type, spec_version='2.1')),
+                ('spec_version', StringProperty(fixed='2.1')),
+                ('id', IDProperty(type, spec_version='2.1')),
+                ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')),
+                ('created', TimestampProperty(default=lambda: NOW, precision='millisecond', precision_constraint='min')),
+                ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond', precision_constraint='min')),
+            ]
+            + extension_properties
+            + [
+                ('revoked', BooleanProperty(default=lambda: False)),
+                ('labels', ListProperty(StringProperty)),
+                ('confidence', IntegerProperty()),
+                ('lang', StringProperty()),
+                ('external_references', ListProperty(ExternalReference)),
+                ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))),
+                ('granular_markings', ListProperty(GranularMarking)),
+                ('extensions', ExtensionsProperty(spec_version='2.1')),
+            ]
+            + sorted((x for x in properties if x[0].startswith('x_')), key=lambda x: x[0])
         )
+
         if extension_name:
             @CustomExtension(type=extension_name, properties=extension_properties)
             class NameExtension:

From 8bbf5fa4616efafb93bc089c8f1d9c8c0e6c2043 Mon Sep 17 00:00:00 2001
From: Michael Chisholm 
Date: Tue, 6 Jul 2021 14:27:40 -0400
Subject: [PATCH 52/56] Make extension instances work the same as other
 objects, with respect to properties.  Before, properties were declared on
 toplevel-property-extension extensions as if they were going to be used in
 the normal way (as actual properties on instances of the extension), but they
 were not used that way, and there was some ugly hackage to make it work. 
 Despite the fact that property instances were given during extension
 registration, they were not used to typecheck, set defaults, etc on toplevel
 property extension properties.

I changed how registration and object initialization works with
respect to properties associated with extensions.  Now,
extensions work the same as any other object and code is
cleaner.  Property instances associated with registered toplevel
extensions are used to enforce requirements like any other
object.

Added some unit tests specifically for property cleaning for
extensions.

Property order (for those contexts where it matters) is updated
to be spec-defined, toplevel extension, custom.
---
 stix2/base.py                 | 104 +++++++++++-------
 stix2/custom.py               |  39 ++++++-
 stix2/test/v21/test_custom.py | 193 +++++++++++++++++++++++++++++++++-
 stix2/v21/common.py           |  18 +---
 stix2/v21/observables.py      |   4 +-
 stix2/v21/sdo.py              |   2 +-
 6 files changed, 297 insertions(+), 63 deletions(-)

diff --git a/stix2/base.py b/stix2/base.py
index 7c68002..69d21b3 100644
--- a/stix2/base.py
+++ b/stix2/base.py
@@ -1,5 +1,6 @@
 """Base classes for type definitions in the STIX2 library."""
 
+import collections
 import collections.abc
 import copy
 import itertools
@@ -36,14 +37,32 @@ class _STIXBase(collections.abc.Mapping):
     """Base class for STIX object types"""
 
     def object_properties(self):
-        props = set(self._properties.keys())
-        custom_props = list(set(self._inner.keys()) - props)
-        custom_props.sort()
+        """
+        Get a list of property names in a particular order: spec order for
+        spec defined properties, followed by toplevel-property-extension
+        properties (any order), followed by custom properties (any order).
 
-        all_properties = list(self._properties.keys())
-        all_properties.extend(custom_props)  # Any custom properties to the bottom
+        The returned list doesn't include only defined+extension properties,
+        nor does it include only assigned properties (i.e. those this object
+        actually possesses).  It's a mix of both: the spec defined property
+        group and extension group include all of them, regardless of whether
+        they're present on this object; the custom group include only names of
+        properties present on this object.
 
-        return all_properties
+        :return: A list of property names
+        """
+        if self.__property_order is None:
+            custom_props = sorted(
+                self.keys() - self._properties.keys()
+                - self.__ext_property_names
+            )
+
+            # Any custom properties to the bottom
+            self.__property_order = list(self._properties) \
+                + list(self.__ext_property_names) \
+                + custom_props
+
+        return self.__property_order
 
     def _check_property(self, prop_name, prop, kwargs, allow_custom):
         if prop_name not in kwargs:
@@ -131,46 +150,45 @@ class _STIXBase(collections.abc.Mapping):
         if custom_props and not isinstance(custom_props, dict):
             raise ValueError("'custom_properties' must be a dictionary")
 
-        # Detect any keyword arguments not allowed for a specific type.
+        # Detect any keyword arguments representing customization.
         # In STIX 2.1, this is complicated by "toplevel-property-extension"
         # type extensions, which can add extra properties which are *not*
         # considered custom.
-        extra_kwargs = kwargs.keys() - self._properties.keys()
-
         extensions = kwargs.get("extensions")
+        registered_toplevel_extension_props = {}
+        has_unregistered_toplevel_extension = False
         if extensions:
-            has_unregistered_toplevel_extension = False
-            registered_toplevel_extension_props = set()
-
             for ext_id, ext in extensions.items():
                 if ext.get("extension_type") == "toplevel-property-extension":
                     registered_ext_class = class_for_type(
                         ext_id, "2.1", "extensions",
                     )
                     if registered_ext_class:
-                        registered_toplevel_extension_props |= \
-                            registered_ext_class._properties.keys()
+                        registered_toplevel_extension_props.update(
+                            registered_ext_class._toplevel_properties
+                        )
                     else:
                         has_unregistered_toplevel_extension = True
 
-            if has_unregistered_toplevel_extension:
-                # Must assume all extras are extension properties, not custom.
-                extra_kwargs.clear()
+        if has_unregistered_toplevel_extension:
+            # Must assume all extras are extension properties, not custom.
+            custom_kwargs = set()
 
-            else:
-                # All toplevel property extensions (if any) have been
-                # registered.  So we can tell what their properties are and
-                # treat only those as not custom.
-                extra_kwargs -= registered_toplevel_extension_props
+        else:
+            # All toplevel property extensions (if any) have been
+            # registered.  So we can tell what their properties are and
+            # treat only those as not custom.
+            custom_kwargs = kwargs.keys() - self._properties.keys() \
+                - registered_toplevel_extension_props.keys()
 
-        if extra_kwargs and not allow_custom:
-            raise ExtraPropertiesError(cls, extra_kwargs)
+        if custom_kwargs and not allow_custom:
+            raise ExtraPropertiesError(cls, custom_kwargs)
 
         if custom_props:
             # loophole for custom_properties...
             allow_custom = True
 
-        all_custom_prop_names = (extra_kwargs | custom_props.keys()) - \
+        all_custom_prop_names = (custom_kwargs | custom_props.keys()) - \
             self._properties.keys()
         if all_custom_prop_names:
             if not isinstance(self, stix2.v20._STIXBase20):
@@ -181,6 +199,21 @@ class _STIXBase(collections.abc.Mapping):
                             reason="Property name '%s' must begin with an alpha character." % prop_name,
                         )
 
+        # defined_properties = all properties defined on this type, plus all
+        # properties defined on this instance as a result of toplevel property
+        # extensions.
+        defined_properties = collections.ChainMap(
+            self._properties, registered_toplevel_extension_props
+        )
+
+        # object_properties() needs this; cache it here to avoid needing to
+        # recompute.
+        self.__ext_property_names = set(registered_toplevel_extension_props)
+        # object_properties() will compute this on first call, based on
+        # __ext_property_names above.  Maybe it makes sense to not compute this
+        # unless really necessary.
+        self.__property_order = None
+
         # Remove any keyword arguments whose value is None or [] (i.e. empty list)
         setting_kwargs = {
             k: v
@@ -189,22 +222,15 @@ class _STIXBase(collections.abc.Mapping):
         }
 
         # Detect any missing required properties
-        required_properties = set(get_required_properties(self._properties))
-        missing_kwargs = required_properties - set(setting_kwargs)
+        required_properties = set(
+            get_required_properties(defined_properties)
+        )
+        missing_kwargs = required_properties - setting_kwargs.keys()
         if missing_kwargs:
-            # In this scenario, we are inside within the scope of the extension.
-            # It is possible to check if this is a new Extension Class by
-            # querying "extension_type". Note: There is an API limitation currently
-            # because a toplevel-property-extension cannot validate its parent properties
-            new_ext_check = (
-                bool(getattr(self, "extension_type", None))
-                and issubclass(cls, stix2.v21._Extension)
-            )
-            if new_ext_check is False:
-                raise MissingPropertiesError(cls, missing_kwargs)
+            raise MissingPropertiesError(cls, missing_kwargs)
 
         has_custom = bool(all_custom_prop_names)
-        for prop_name, prop_metadata in self._properties.items():
+        for prop_name, prop_metadata in defined_properties.items():
             temp_custom = self._check_property(
                 prop_name, prop_metadata, setting_kwargs, allow_custom,
             )
@@ -213,7 +239,7 @@ class _STIXBase(collections.abc.Mapping):
 
         # Cache defaulted optional properties for serialization
         defaulted = []
-        for name, prop in self._properties.items():
+        for name, prop in defined_properties.items():
             try:
                 if (
                     not prop.required and not hasattr(prop, '_fixed_value') and
diff --git a/stix2/custom.py b/stix2/custom.py
index 9b1654e..7806179 100644
--- a/stix2/custom.py
+++ b/stix2/custom.py
@@ -1,6 +1,7 @@
 from collections import OrderedDict
 
 from .base import _cls_init
+from .properties import EnumProperty
 from .registration import (
     _get_extension_class, _register_extension, _register_marking,
     _register_object, _register_observable,
@@ -93,12 +94,46 @@ def _custom_observable_builder(cls, type, properties, version, base_class, id_co
 
 
 def _custom_extension_builder(cls, type, properties, version, base_class):
-    prop_dict = _get_properties_dict(properties)
+
+    properties = _get_properties_dict(properties)
+    toplevel_properties = None
+
+    # Auto-create an "extension_type" property from the class attribute, if
+    # it exists.  How to treat the other properties which were given depends on
+    # the extension type.
+    extension_type = getattr(cls, "extension_type", None)
+    if extension_type:
+        # I suppose I could also go with a plain string property, since the
+        # value is fixed... but an enum property seems more true to the
+        # property's semantics.  Also, I can't import a vocab module for the
+        # enum values without circular import errors. :(
+        extension_type_prop = EnumProperty(
+            [
+                "new-sdo", "new-sco", "new-sro", "property-extension",
+                "toplevel-property-extension"
+            ],
+            required=False,
+            fixed=extension_type,
+        )
+
+        nested_properties = {
+            "extension_type": extension_type_prop
+        }
+
+        if extension_type == "toplevel-property-extension":
+            toplevel_properties = properties
+        else:
+            nested_properties.update(properties)
+
+    else:
+        nested_properties = properties
 
     class _CustomExtension(cls, base_class):
 
         _type = type
-        _properties = prop_dict
+        _properties = nested_properties
+        if extension_type == "toplevel-property-extension":
+            _toplevel_properties = toplevel_properties
 
         def __init__(self, **kwargs):
             base_class.__init__(self, **kwargs)
diff --git a/stix2/test/v21/test_custom.py b/stix2/test/v21/test_custom.py
index 1b3166f..7aed1b7 100644
--- a/stix2/test/v21/test_custom.py
+++ b/stix2/test/v21/test_custom.py
@@ -1,3 +1,4 @@
+import contextlib
 import uuid
 
 import pytest
@@ -8,7 +9,9 @@ import stix2.registration
 import stix2.registry
 import stix2.v21
 
-from ...exceptions import DuplicateRegistrationError, InvalidValueError
+from ...exceptions import (
+    DuplicateRegistrationError, InvalidValueError, MissingPropertiesError
+)
 from .constants import FAKE_TIME, IDENTITY_ID, MARKING_DEFINITION_ID
 
 # Custom Properties in SDOs
@@ -1675,6 +1678,194 @@ def test_registered_new_extension_marking_allow_custom_false():
            '{"extension_type": "property-extension", "some_marking_field": "value"}}' in marking_serialized
 
 
+@contextlib.contextmanager
+def _register_extension(ext, props):
+
+    ext_def_id = "extension-definition--" + str(uuid.uuid4())
+
+    stix2.v21.CustomExtension(
+        ext_def_id,
+        props
+    )(ext)
+
+    try:
+        yield ext_def_id
+    finally:
+        # "unregister" the extension
+        del stix2.registry.STIX2_OBJ_MAPS["2.1"]["extensions"][ext_def_id]
+
+
+def test_nested_ext_prop_meta():
+
+    class TestExt:
+        extension_type = "property-extension"
+
+    props = {
+        "intprop": stix2.properties.IntegerProperty(required=True),
+        "strprop": stix2.properties.StringProperty(
+            required=False, default=lambda: "foo"
+        )
+    }
+
+    with _register_extension(TestExt, props) as ext_def_id:
+
+        obj = stix2.v21.Identity(
+            name="test",
+            extensions={
+                ext_def_id: {
+                    "extension_type": "property-extension",
+                    "intprop": "1",
+                    "strprop": 2
+                }
+            }
+        )
+
+        assert obj.extensions[ext_def_id].extension_type == "property-extension"
+        assert obj.extensions[ext_def_id].intprop == 1
+        assert obj.extensions[ext_def_id].strprop == "2"
+
+        obj = stix2.v21.Identity(
+            name="test",
+            extensions={
+                ext_def_id: {
+                    "extension_type": "property-extension",
+                    "intprop": "1",
+                }
+            }
+        )
+
+        # Ensure default kicked in
+        assert obj.extensions[ext_def_id].strprop == "foo"
+
+        with pytest.raises(InvalidValueError):
+            stix2.v21.Identity(
+                name="test",
+                extensions={
+                    ext_def_id: {
+                        "extension_type": "property-extension",
+                        # wrong value type
+                        "intprop": "foo"
+                    }
+                }
+            )
+
+        with pytest.raises(InvalidValueError):
+            stix2.v21.Identity(
+                name="test",
+                extensions={
+                    ext_def_id: {
+                        "extension_type": "property-extension",
+                        # missing required property
+                        "strprop": "foo"
+                    }
+                }
+            )
+
+        with pytest.raises(InvalidValueError):
+            stix2.v21.Identity(
+                name="test",
+                extensions={
+                    ext_def_id: {
+                        "extension_type": "property-extension",
+                        "intprop": 1,
+                        # Use of undefined property
+                        "foo": False,
+                    }
+                }
+            )
+
+        with pytest.raises(InvalidValueError):
+            stix2.v21.Identity(
+                name="test",
+                extensions={
+                    ext_def_id: {
+                        # extension_type doesn't match with registration
+                        "extension_type": "new-sdo",
+                        "intprop": 1,
+                        "strprop": "foo",
+                    }
+                }
+            )
+
+
+def test_toplevel_ext_prop_meta():
+
+    class TestExt:
+        extension_type = "toplevel-property-extension"
+
+    props = {
+        "intprop": stix2.properties.IntegerProperty(required=True),
+        "strprop": stix2.properties.StringProperty(
+            required=False, default=lambda: "foo"
+        )
+    }
+
+    with _register_extension(TestExt, props) as ext_def_id:
+
+        obj = stix2.v21.Identity(
+            name="test",
+            intprop="1",
+            strprop=2,
+            extensions={
+                ext_def_id: {
+                    "extension_type": "toplevel-property-extension"
+                }
+            }
+        )
+
+        assert obj.extensions[ext_def_id].extension_type == "toplevel-property-extension"
+        assert obj.intprop == 1
+        assert obj.strprop == "2"
+
+        obj = stix2.v21.Identity(
+            name="test",
+            intprop=1,
+            extensions={
+                ext_def_id: {
+                    "extension_type": "toplevel-property-extension"
+                }
+            }
+        )
+
+        # Ensure default kicked in
+        assert obj.strprop == "foo"
+
+        with pytest.raises(InvalidValueError):
+            stix2.v21.Identity(
+                name="test",
+                intprop="foo",  # wrong value type
+                extensions={
+                    ext_def_id: {
+                        "extension_type": "toplevel-property-extension"
+                    }
+                }
+            )
+
+        with pytest.raises(InvalidValueError):
+            stix2.v21.Identity(
+                name="test",
+                intprop=1,
+                extensions={
+                    ext_def_id: {
+                        "extension_type": "toplevel-property-extension",
+                        # Use of undefined property
+                        "foo": False,
+                    }
+                }
+            )
+
+        with pytest.raises(MissingPropertiesError):
+            stix2.v21.Identity(
+                name="test",
+                strprop="foo",  # missing required property
+                extensions={
+                    ext_def_id: {
+                        "extension_type": "toplevel-property-extension"
+                    }
+                }
+            )
+
+
 def test_allow_custom_propagation():
     obj_dict = {
         "type": "bundle",
diff --git a/stix2/v21/common.py b/stix2/v21/common.py
index 6b2922b..9ba1096 100644
--- a/stix2/v21/common.py
+++ b/stix2/v21/common.py
@@ -1,7 +1,6 @@
 """STIX 2.1 Common Data Types and Properties."""
 
 from collections import OrderedDict
-from collections.abc import Mapping
 
 from . import _Extension
 from ..custom import _custom_marking_builder, _custom_extension_builder
@@ -145,23 +144,8 @@ def CustomExtension(type='x-custom-ext', properties=None):
     """Custom STIX Object Extension decorator.
     """
     def wrapper(cls):
-
-        # Auto-create an "extension_type" property from the class attribute, if
-        # it exists.
-        extension_type = getattr(cls, "extension_type", None)
-        if extension_type:
-            extension_type_prop = EnumProperty(
-                EXTENSION_TYPE,
-                required=False,
-                fixed=extension_type,
-            )
-
-            if isinstance(properties, Mapping):
-                properties["extension_type"] = extension_type_prop
-            else:
-                properties.append(("extension_type", extension_type_prop))
-
         return _custom_extension_builder(cls, type, properties, '2.1', _Extension)
+
     return wrapper
 
 
diff --git a/stix2/v21/observables.py b/stix2/v21/observables.py
index 414a0b0..3b0d35a 100644
--- a/stix2/v21/observables.py
+++ b/stix2/v21/observables.py
@@ -889,7 +889,7 @@ def CustomObservable(type='x-custom-observable', properties=None, id_contrib_pro
             ),
         )
         if extension_name:
-            @CustomExtension(type=extension_name, properties=properties)
+            @CustomExtension(type=extension_name, properties={})
             class NameExtension:
                 extension_type = 'new-sco'
 
@@ -899,5 +899,3 @@ def CustomObservable(type='x-custom-observable', properties=None, id_contrib_pro
             cls.with_extension = extension_name
         return _custom_observable_builder(cls, type, _properties, '2.1', _Observable, id_contrib_props)
     return wrapper
-
-
diff --git a/stix2/v21/sdo.py b/stix2/v21/sdo.py
index 5782252..4a37e21 100644
--- a/stix2/v21/sdo.py
+++ b/stix2/v21/sdo.py
@@ -857,7 +857,7 @@ def CustomObject(type='x-custom-type', properties=None, extension_name=None, is_
         )
 
         if extension_name:
-            @CustomExtension(type=extension_name, properties=extension_properties)
+            @CustomExtension(type=extension_name, properties={})
             class NameExtension:
                 if is_sdo:
                     extension_type = 'new-sdo'

From 2cda97cf5e51403c43a96ceb9073afab7706638a Mon Sep 17 00:00:00 2001
From: Michael Chisholm 
Date: Tue, 6 Jul 2021 20:32:58 -0400
Subject: [PATCH 53/56] Changed STIX object initialization to formulate a
 property order and process properties in that order.  This establishes
 iteration order on object properties, making the object_properties() method
 unnecessary.  So the latter method has been deleted.  All uses of that method
 have been removed.

Removed unnecessary deepcopy() in STIXJSONEncoder, to improve
efficiency.  This uncovered a bug which had been affecting
STIXdatetime instances.  Not deepcopying doesn't trip the bug,
which can change serialization format.  This caused a unit
test to fail, which was checking serialization format.  I fixed
the unit test.

Fixed a bug in _STIXBase.__repr__ which caused it to omit all
properties with falsey values.  This caused several unit tests
to break, since they were written against the old buggy repr
format.  Notably, 'revoked=False' was never included in reprs
before, but it is now.
---
 stix2/base.py                               | 77 ++++++++-------------
 stix2/serialization.py                      |  4 +-
 stix2/test/v20/test_identity.py             |  2 +-
 stix2/test/v20/test_indicator.py            |  1 +
 stix2/test/v20/test_markings.py             |  2 +-
 stix2/test/v21/test_extension_definition.py |  1 -
 stix2/test/v21/test_identity.py             |  1 -
 stix2/test/v21/test_incident.py             |  1 -
 stix2/test/v21/test_indicator.py            |  3 +-
 stix2/test/v21/test_location.py             |  6 +-
 stix2/test/v21/test_note.py                 |  1 +
 stix2/test/v21/test_opinion.py              |  3 +-
 12 files changed, 41 insertions(+), 61 deletions(-)

diff --git a/stix2/base.py b/stix2/base.py
index 69d21b3..24176f4 100644
--- a/stix2/base.py
+++ b/stix2/base.py
@@ -36,34 +36,6 @@ def get_required_properties(properties):
 class _STIXBase(collections.abc.Mapping):
     """Base class for STIX object types"""
 
-    def object_properties(self):
-        """
-        Get a list of property names in a particular order: spec order for
-        spec defined properties, followed by toplevel-property-extension
-        properties (any order), followed by custom properties (any order).
-
-        The returned list doesn't include only defined+extension properties,
-        nor does it include only assigned properties (i.e. those this object
-        actually possesses).  It's a mix of both: the spec defined property
-        group and extension group include all of them, regardless of whether
-        they're present on this object; the custom group include only names of
-        properties present on this object.
-
-        :return: A list of property names
-        """
-        if self.__property_order is None:
-            custom_props = sorted(
-                self.keys() - self._properties.keys()
-                - self.__ext_property_names
-            )
-
-            # Any custom properties to the bottom
-            self.__property_order = list(self._properties) \
-                + list(self.__ext_property_names) \
-                + custom_props
-
-        return self.__property_order
-
     def _check_property(self, prop_name, prop, kwargs, allow_custom):
         if prop_name not in kwargs:
             if hasattr(prop, 'default'):
@@ -206,20 +178,33 @@ class _STIXBase(collections.abc.Mapping):
             self._properties, registered_toplevel_extension_props
         )
 
-        # object_properties() needs this; cache it here to avoid needing to
-        # recompute.
-        self.__ext_property_names = set(registered_toplevel_extension_props)
-        # object_properties() will compute this on first call, based on
-        # __ext_property_names above.  Maybe it makes sense to not compute this
-        # unless really necessary.
-        self.__property_order = None
+        assigned_properties = collections.ChainMap(kwargs, custom_props)
 
-        # Remove any keyword arguments whose value is None or [] (i.e. empty list)
-        setting_kwargs = {
-            k: v
-            for k, v in itertools.chain(kwargs.items(), custom_props.items())
-            if v is not None and v != []
-        }
+        # Establish property order: spec-defined, toplevel extension, custom.
+        toplevel_extension_props = registered_toplevel_extension_props.keys() \
+            | (kwargs.keys() - self._properties.keys() - custom_kwargs)
+        property_order = itertools.chain(
+            self._properties,
+            toplevel_extension_props,
+            sorted(all_custom_prop_names)
+        )
+
+        setting_kwargs = {}
+
+        has_custom = bool(all_custom_prop_names)
+        for prop_name in property_order:
+
+            prop_val = assigned_properties.get(prop_name)
+            if prop_val not in (None, []):
+                setting_kwargs[prop_name] = prop_val
+
+            prop = defined_properties.get(prop_name)
+            if prop:
+                temp_custom = self._check_property(
+                    prop_name, prop, setting_kwargs, allow_custom,
+                )
+
+                has_custom = has_custom or temp_custom
 
         # Detect any missing required properties
         required_properties = set(
@@ -229,14 +214,6 @@ class _STIXBase(collections.abc.Mapping):
         if missing_kwargs:
             raise MissingPropertiesError(cls, missing_kwargs)
 
-        has_custom = bool(all_custom_prop_names)
-        for prop_name, prop_metadata in defined_properties.items():
-            temp_custom = self._check_property(
-                prop_name, prop_metadata, setting_kwargs, allow_custom,
-            )
-
-            has_custom = has_custom or temp_custom
-
         # Cache defaulted optional properties for serialization
         defaulted = []
         for name, prop in defined_properties.items():
@@ -304,7 +281,7 @@ class _STIXBase(collections.abc.Mapping):
         return self.serialize()
 
     def __repr__(self):
-        props = ', '.join([f"{k}={self[k]!r}" for k in self.object_properties() if self.get(k)])
+        props = ', '.join([f"{k}={self[k]!r}" for k in self])
         return f'{self.__class__.__name__}({props})'
 
     def __deepcopy__(self, memo):
diff --git a/stix2/serialization.py b/stix2/serialization.py
index 2784d39..236b987 100644
--- a/stix2/serialization.py
+++ b/stix2/serialization.py
@@ -24,7 +24,7 @@ class STIXJSONEncoder(json.JSONEncoder):
         if isinstance(obj, (dt.date, dt.datetime)):
             return format_datetime(obj)
         elif isinstance(obj, stix2.base._STIXBase):
-            tmp_obj = dict(copy.deepcopy(obj))
+            tmp_obj = dict(obj)
             for prop_name in obj._defaulted_optional_properties:
                 del tmp_obj[prop_name]
             return tmp_obj
@@ -177,7 +177,7 @@ def find_property_index(obj, search_key, search_value):
 
     if isinstance(obj, stix2.base._STIXBase):
         if search_key in obj and obj[search_key] == search_value:
-            idx = _find(obj.object_properties(), search_key)
+            idx = _find(list(obj), search_key)
         else:
             idx = _find_property_in_seq(obj.values(), search_key, search_value)
     elif isinstance(obj, dict):
diff --git a/stix2/test/v20/test_identity.py b/stix2/test/v20/test_identity.py
index c62da46..93787b3 100644
--- a/stix2/test/v20/test_identity.py
+++ b/stix2/test/v20/test_identity.py
@@ -74,6 +74,6 @@ def test_identity_with_custom():
     )
 
     assert identity.x_foo == "bar"
-    assert "x_foo" in identity.object_properties()
+    assert "x_foo" in identity
 
 # TODO: Add other examples
diff --git a/stix2/test/v20/test_indicator.py b/stix2/test/v20/test_indicator.py
index 47f4812..10a7015 100644
--- a/stix2/test/v20/test_indicator.py
+++ b/stix2/test/v20/test_indicator.py
@@ -28,6 +28,7 @@ EXPECTED_INDICATOR_REPR = "Indicator(" + " ".join(
     modified='2017-01-01T00:00:01.000Z',
     pattern="[file:hashes.MD5 = 'd41d8cd98f00b204e9800998ecf8427e']",
     valid_from='1970-01-01T00:00:01Z',
+    revoked=False,
     labels=['malicious-activity']
 """.split(),
 ) + ")"
diff --git a/stix2/test/v20/test_markings.py b/stix2/test/v20/test_markings.py
index f7d15fa..70b46a6 100644
--- a/stix2/test/v20/test_markings.py
+++ b/stix2/test/v20/test_markings.py
@@ -21,7 +21,7 @@ EXPECTED_TLP_MARKING_DEFINITION = """{
 EXPECTED_STATEMENT_MARKING_DEFINITION = """{
     "type": "marking-definition",
     "id": "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9",
-    "created": "2017-01-20T00:00:00Z",
+    "created": "2017-01-20T00:00:00.000Z",
     "definition_type": "statement",
     "definition": {
         "statement": "Copyright 2016, Example Corp"
diff --git a/stix2/test/v21/test_extension_definition.py b/stix2/test/v21/test_extension_definition.py
index 97513c4..af13425 100644
--- a/stix2/test/v21/test_extension_definition.py
+++ b/stix2/test/v21/test_extension_definition.py
@@ -105,4 +105,3 @@ def test_extension_definition_with_custom():
     )
 
     assert extension_definition.x_foo == "bar"
-    assert "x_foo" in extension_definition.object_properties()
diff --git a/stix2/test/v21/test_identity.py b/stix2/test/v21/test_identity.py
index c235b4d..2f8747c 100644
--- a/stix2/test/v21/test_identity.py
+++ b/stix2/test/v21/test_identity.py
@@ -77,6 +77,5 @@ def test_identity_with_custom():
     )
 
     assert identity.x_foo == "bar"
-    assert "x_foo" in identity.object_properties()
 
 # TODO: Add other examples
diff --git a/stix2/test/v21/test_incident.py b/stix2/test/v21/test_incident.py
index 27bc254..fac9238 100644
--- a/stix2/test/v21/test_incident.py
+++ b/stix2/test/v21/test_incident.py
@@ -78,4 +78,3 @@ def test_incident_with_custom():
     )
 
     assert incident.x_foo == "bar"
-    assert "x_foo" in incident.object_properties()
diff --git a/stix2/test/v21/test_indicator.py b/stix2/test/v21/test_indicator.py
index e42ffba..6175948 100644
--- a/stix2/test/v21/test_indicator.py
+++ b/stix2/test/v21/test_indicator.py
@@ -30,7 +30,8 @@ EXPECTED_INDICATOR_REPR = "Indicator(" + " ".join(
     pattern="[file:hashes.MD5 = 'd41d8cd98f00b204e9800998ecf8427e']",
     pattern_type='stix',
     pattern_version='2.1',
-    valid_from='1970-01-01T00:00:01Z'
+    valid_from='1970-01-01T00:00:01Z',
+    revoked=False
 """.split(),
 ) + ")"
 
diff --git a/stix2/test/v21/test_location.py b/stix2/test/v21/test_location.py
index c2df5d3..5d3eab8 100644
--- a/stix2/test/v21/test_location.py
+++ b/stix2/test/v21/test_location.py
@@ -27,7 +27,8 @@ EXPECTED_LOCATION_1_REPR = "Location(" + " ".join(
     created='2016-04-06T20:03:00.000Z',
     modified='2016-04-06T20:03:00.000Z',
     latitude=48.8566,
-    longitude=2.3522""".split(),
+    longitude=2.3522,
+    revoked=False""".split(),
 ) + ")"
 
 EXPECTED_LOCATION_2 = """{
@@ -47,7 +48,8 @@ EXPECTED_LOCATION_2_REPR = "Location(" + " ".join(
     id='location--a6e9345f-5a15-4c29-8bb3-7dcc5d168d64',
     created='2016-04-06T20:03:00.000Z',
     modified='2016-04-06T20:03:00.000Z',
-    region='northern-america'""".split(),
+    region='northern-america',
+    revoked=False""".split(),
 ) + ")"
 
 
diff --git a/stix2/test/v21/test_note.py b/stix2/test/v21/test_note.py
index 2f20c8d..ca1fc6d 100644
--- a/stix2/test/v21/test_note.py
+++ b/stix2/test/v21/test_note.py
@@ -48,6 +48,7 @@ EXPECTED_OPINION_REPR = "Note(" + " ".join((
     content='%s',
     authors=['John Doe'],
     object_refs=['campaign--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f'],
+    revoked=False,
     external_references=[ExternalReference(source_name='job-tracker', external_id='job-id-1234')]
 """ % CONTENT
 ).split()) + ")"
diff --git a/stix2/test/v21/test_opinion.py b/stix2/test/v21/test_opinion.py
index 21c96d7..31cd0af 100644
--- a/stix2/test/v21/test_opinion.py
+++ b/stix2/test/v21/test_opinion.py
@@ -38,7 +38,8 @@ EXPECTED_OPINION_REPR = "Opinion(" + " ".join((
     modified='2016-05-12T08:17:27.000Z',
     explanation="%s",
     opinion='strongly-disagree',
-    object_refs=['relationship--16d2358f-3b0d-4c88-b047-0da2f7ed4471']
+    object_refs=['relationship--16d2358f-3b0d-4c88-b047-0da2f7ed4471'],
+    revoked=False
     """ % EXPLANATION
 ).split()) + ")"
 

From 99a8ade4cd3ebc69d62b688da5c21e56c93106f7 Mon Sep 17 00:00:00 2001
From: Michael Chisholm 
Date: Tue, 6 Jul 2021 20:40:50 -0400
Subject: [PATCH 54/56] pre-commit stylistic fixes

---
 stix2/base.py                 |  8 ++--
 stix2/custom.py               |  4 +-
 stix2/serialization.py        |  1 -
 stix2/test/v21/test_custom.py | 70 +++++++++++++++++------------------
 stix2/v21/__init__.py         | 15 ++++----
 stix2/v21/common.py           |  4 +-
 stix2/v21/observables.py      |  4 +-
 stix2/v21/sdo.py              |  2 +-
 8 files changed, 52 insertions(+), 56 deletions(-)

diff --git a/stix2/base.py b/stix2/base.py
index 24176f4..3ff01d9 100644
--- a/stix2/base.py
+++ b/stix2/base.py
@@ -137,7 +137,7 @@ class _STIXBase(collections.abc.Mapping):
                     )
                     if registered_ext_class:
                         registered_toplevel_extension_props.update(
-                            registered_ext_class._toplevel_properties
+                            registered_ext_class._toplevel_properties,
                         )
                     else:
                         has_unregistered_toplevel_extension = True
@@ -175,7 +175,7 @@ class _STIXBase(collections.abc.Mapping):
         # properties defined on this instance as a result of toplevel property
         # extensions.
         defined_properties = collections.ChainMap(
-            self._properties, registered_toplevel_extension_props
+            self._properties, registered_toplevel_extension_props,
         )
 
         assigned_properties = collections.ChainMap(kwargs, custom_props)
@@ -186,7 +186,7 @@ class _STIXBase(collections.abc.Mapping):
         property_order = itertools.chain(
             self._properties,
             toplevel_extension_props,
-            sorted(all_custom_prop_names)
+            sorted(all_custom_prop_names),
         )
 
         setting_kwargs = {}
@@ -208,7 +208,7 @@ class _STIXBase(collections.abc.Mapping):
 
         # Detect any missing required properties
         required_properties = set(
-            get_required_properties(defined_properties)
+            get_required_properties(defined_properties),
         )
         missing_kwargs = required_properties - setting_kwargs.keys()
         if missing_kwargs:
diff --git a/stix2/custom.py b/stix2/custom.py
index 7806179..adef768 100644
--- a/stix2/custom.py
+++ b/stix2/custom.py
@@ -110,14 +110,14 @@ def _custom_extension_builder(cls, type, properties, version, base_class):
         extension_type_prop = EnumProperty(
             [
                 "new-sdo", "new-sco", "new-sro", "property-extension",
-                "toplevel-property-extension"
+                "toplevel-property-extension",
             ],
             required=False,
             fixed=extension_type,
         )
 
         nested_properties = {
-            "extension_type": extension_type_prop
+            "extension_type": extension_type_prop,
         }
 
         if extension_type == "toplevel-property-extension":
diff --git a/stix2/serialization.py b/stix2/serialization.py
index 236b987..7510b37 100644
--- a/stix2/serialization.py
+++ b/stix2/serialization.py
@@ -1,6 +1,5 @@
 """STIX2 core serialization methods."""
 
-import copy
 import datetime as dt
 import io
 
diff --git a/stix2/test/v21/test_custom.py b/stix2/test/v21/test_custom.py
index 7aed1b7..f35b02d 100644
--- a/stix2/test/v21/test_custom.py
+++ b/stix2/test/v21/test_custom.py
@@ -10,7 +10,7 @@ import stix2.registry
 import stix2.v21
 
 from ...exceptions import (
-    DuplicateRegistrationError, InvalidValueError, MissingPropertiesError
+    DuplicateRegistrationError, InvalidValueError, MissingPropertiesError,
 )
 from .constants import FAKE_TIME, IDENTITY_ID, MARKING_DEFINITION_ID
 
@@ -1685,7 +1685,7 @@ def _register_extension(ext, props):
 
     stix2.v21.CustomExtension(
         ext_def_id,
-        props
+        props,
     )(ext)
 
     try:
@@ -1703,8 +1703,8 @@ def test_nested_ext_prop_meta():
     props = {
         "intprop": stix2.properties.IntegerProperty(required=True),
         "strprop": stix2.properties.StringProperty(
-            required=False, default=lambda: "foo"
-        )
+            required=False, default=lambda: "foo",
+        ),
     }
 
     with _register_extension(TestExt, props) as ext_def_id:
@@ -1715,9 +1715,9 @@ def test_nested_ext_prop_meta():
                 ext_def_id: {
                     "extension_type": "property-extension",
                     "intprop": "1",
-                    "strprop": 2
-                }
-            }
+                    "strprop": 2,
+                },
+            },
         )
 
         assert obj.extensions[ext_def_id].extension_type == "property-extension"
@@ -1730,8 +1730,8 @@ def test_nested_ext_prop_meta():
                 ext_def_id: {
                     "extension_type": "property-extension",
                     "intprop": "1",
-                }
-            }
+                },
+            },
         )
 
         # Ensure default kicked in
@@ -1744,9 +1744,9 @@ def test_nested_ext_prop_meta():
                     ext_def_id: {
                         "extension_type": "property-extension",
                         # wrong value type
-                        "intprop": "foo"
-                    }
-                }
+                        "intprop": "foo",
+                    },
+                },
             )
 
         with pytest.raises(InvalidValueError):
@@ -1756,9 +1756,9 @@ def test_nested_ext_prop_meta():
                     ext_def_id: {
                         "extension_type": "property-extension",
                         # missing required property
-                        "strprop": "foo"
-                    }
-                }
+                        "strprop": "foo",
+                    },
+                },
             )
 
         with pytest.raises(InvalidValueError):
@@ -1770,8 +1770,8 @@ def test_nested_ext_prop_meta():
                         "intprop": 1,
                         # Use of undefined property
                         "foo": False,
-                    }
-                }
+                    },
+                },
             )
 
         with pytest.raises(InvalidValueError):
@@ -1783,8 +1783,8 @@ def test_nested_ext_prop_meta():
                         "extension_type": "new-sdo",
                         "intprop": 1,
                         "strprop": "foo",
-                    }
-                }
+                    },
+                },
             )
 
 
@@ -1796,8 +1796,8 @@ def test_toplevel_ext_prop_meta():
     props = {
         "intprop": stix2.properties.IntegerProperty(required=True),
         "strprop": stix2.properties.StringProperty(
-            required=False, default=lambda: "foo"
-        )
+            required=False, default=lambda: "foo",
+        ),
     }
 
     with _register_extension(TestExt, props) as ext_def_id:
@@ -1808,9 +1808,9 @@ def test_toplevel_ext_prop_meta():
             strprop=2,
             extensions={
                 ext_def_id: {
-                    "extension_type": "toplevel-property-extension"
-                }
-            }
+                    "extension_type": "toplevel-property-extension",
+                },
+            },
         )
 
         assert obj.extensions[ext_def_id].extension_type == "toplevel-property-extension"
@@ -1822,9 +1822,9 @@ def test_toplevel_ext_prop_meta():
             intprop=1,
             extensions={
                 ext_def_id: {
-                    "extension_type": "toplevel-property-extension"
-                }
-            }
+                    "extension_type": "toplevel-property-extension",
+                },
+            },
         )
 
         # Ensure default kicked in
@@ -1836,9 +1836,9 @@ def test_toplevel_ext_prop_meta():
                 intprop="foo",  # wrong value type
                 extensions={
                     ext_def_id: {
-                        "extension_type": "toplevel-property-extension"
-                    }
-                }
+                        "extension_type": "toplevel-property-extension",
+                    },
+                },
             )
 
         with pytest.raises(InvalidValueError):
@@ -1850,8 +1850,8 @@ def test_toplevel_ext_prop_meta():
                         "extension_type": "toplevel-property-extension",
                         # Use of undefined property
                         "foo": False,
-                    }
-                }
+                    },
+                },
             )
 
         with pytest.raises(MissingPropertiesError):
@@ -1860,9 +1860,9 @@ def test_toplevel_ext_prop_meta():
                 strprop="foo",  # missing required property
                 extensions={
                     ext_def_id: {
-                        "extension_type": "toplevel-property-extension"
-                    }
-                }
+                        "extension_type": "toplevel-property-extension",
+                    },
+                },
             )
 
 
diff --git a/stix2/v21/__init__.py b/stix2/v21/__init__.py
index ece8b36..2d29322 100644
--- a/stix2/v21/__init__.py
+++ b/stix2/v21/__init__.py
@@ -25,14 +25,13 @@ from .common import (
 )
 from .observables import (
     URL, AlternateDataStream, ArchiveExt, Artifact, AutonomousSystem,
-    CustomObservable, Directory, DomainName, EmailAddress,
-    EmailMessage, EmailMIMEComponent, File, HTTPRequestExt, ICMPExt,
-    IPv4Address, IPv6Address, MACAddress, Mutex, NetworkTraffic, NTFSExt,
-    PDFExt, Process, RasterImageExt, SocketExt, Software, TCPExt,
-    UNIXAccountExt, UserAccount, WindowsPEBinaryExt,
-    WindowsPEOptionalHeaderType, WindowsPESection, WindowsProcessExt,
-    WindowsRegistryKey, WindowsRegistryValueType, WindowsServiceExt,
-    X509Certificate, X509V3ExtensionsType,
+    CustomObservable, Directory, DomainName, EmailAddress, EmailMessage,
+    EmailMIMEComponent, File, HTTPRequestExt, ICMPExt, IPv4Address,
+    IPv6Address, MACAddress, Mutex, NetworkTraffic, NTFSExt, PDFExt, Process,
+    RasterImageExt, SocketExt, Software, TCPExt, UNIXAccountExt, UserAccount,
+    WindowsPEBinaryExt, WindowsPEOptionalHeaderType, WindowsPESection,
+    WindowsProcessExt, WindowsRegistryKey, WindowsRegistryValueType,
+    WindowsServiceExt, X509Certificate, X509V3ExtensionsType,
 )
 from .sdo import (
     AttackPattern, Campaign, CourseOfAction, CustomObject, Grouping, Identity,
diff --git a/stix2/v21/common.py b/stix2/v21/common.py
index 9ba1096..b5f9c62 100644
--- a/stix2/v21/common.py
+++ b/stix2/v21/common.py
@@ -3,7 +3,7 @@
 from collections import OrderedDict
 
 from . import _Extension
-from ..custom import _custom_marking_builder, _custom_extension_builder
+from ..custom import _custom_extension_builder, _custom_marking_builder
 from ..exceptions import InvalidValueError, PropertyPresenceError
 from ..markings import _MarkingsMixin
 from ..markings.utils import check_tlp_marking
@@ -268,8 +268,6 @@ def CustomMarking(type='x-custom-marking', properties=None, extension_name=None)
     """
     def wrapper(cls):
         if extension_name:
-            from . import observables
-
             @CustomExtension(type=extension_name, properties=properties)
             class NameExtension:
                 extension_type = 'property-extension'
diff --git a/stix2/v21/observables.py b/stix2/v21/observables.py
index 3b0d35a..4398772 100644
--- a/stix2/v21/observables.py
+++ b/stix2/v21/observables.py
@@ -18,7 +18,7 @@ from ..properties import (
     TypeProperty,
 )
 from .base import _Extension, _Observable, _STIXBase21
-from .common import GranularMarking, CustomExtension
+from .common import CustomExtension, GranularMarking
 from .vocab import (
     ACCOUNT_TYPE, ENCRYPTION_ALGORITHM, HASHING_ALGORITHM,
     NETWORK_SOCKET_ADDRESS_FAMILY, NETWORK_SOCKET_TYPE,
@@ -877,7 +877,7 @@ def CustomObservable(type='x-custom-observable', properties=None, id_contrib_pro
                 [
                     ('type', TypeProperty(type, spec_version='2.1')),
                     ('spec_version', StringProperty(fixed='2.1')),
-                    ('id', IDProperty(type, spec_version='2.1'))
+                    ('id', IDProperty(type, spec_version='2.1')),
                 ],
                 properties,
                 [
diff --git a/stix2/v21/sdo.py b/stix2/v21/sdo.py
index 4a37e21..3fd4e04 100644
--- a/stix2/v21/sdo.py
+++ b/stix2/v21/sdo.py
@@ -19,7 +19,7 @@ from ..properties import (
 from ..utils import NOW
 from .base import _DomainObject
 from .common import (
-    CustomExtension, ExternalReference, GranularMarking, KillChainPhase
+    CustomExtension, ExternalReference, GranularMarking, KillChainPhase,
 )
 from .vocab import (
     ATTACK_MOTIVATION, ATTACK_RESOURCE_LEVEL, GROUPING_CONTEXT, IDENTITY_CLASS,

From 638689c4816bf610c5149a1f4a2968ed440d486b Mon Sep 17 00:00:00 2001
From: Michael Chisholm 
Date: Wed, 7 Jul 2021 13:03:13 -0400
Subject: [PATCH 55/56] Add another check to the test_toplevel_ext_prop_meta()
 unit test.

---
 stix2/test/v21/test_custom.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/stix2/test/v21/test_custom.py b/stix2/test/v21/test_custom.py
index f35b02d..fcb573b 100644
--- a/stix2/test/v21/test_custom.py
+++ b/stix2/test/v21/test_custom.py
@@ -1854,6 +1854,20 @@ def test_toplevel_ext_prop_meta():
                 },
             )
 
+        with pytest.raises(InvalidValueError):
+            stix2.v21.Identity(
+                name="test",
+                intprop=1,
+                extensions={
+                    ext_def_id: {
+                        "extension_type": "toplevel-property-extension",
+                        # Use of a defined property, but intended for the
+                        # top level.  This should still error out.
+                        "intprop": 1,
+                    },
+                },
+            )
+
         with pytest.raises(MissingPropertiesError):
             stix2.v21.Identity(
                 name="test",

From bb164ad1ae701e1b7269bf754459f415f96015e7 Mon Sep 17 00:00:00 2001
From: Michael Chisholm 
Date: Wed, 7 Jul 2021 16:06:04 -0400
Subject: [PATCH 56/56] Small change to a unit test

---
 stix2/test/v21/test_custom.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/stix2/test/v21/test_custom.py b/stix2/test/v21/test_custom.py
index fcb573b..637c79f 100644
--- a/stix2/test/v21/test_custom.py
+++ b/stix2/test/v21/test_custom.py
@@ -1863,7 +1863,7 @@ def test_toplevel_ext_prop_meta():
                         "extension_type": "toplevel-property-extension",
                         # Use of a defined property, but intended for the
                         # top level.  This should still error out.
-                        "intprop": 1,
+                        "strprop": 1,
                     },
                 },
             )