From b2ee33208f3e78b9a8043d3bbc61bb51fe44fa79 Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Fri, 12 Jun 2020 14:31:01 -0400 Subject: [PATCH 01/30] Set STIX 2.1 as default, fix indicator.valid_from --- README.rst | 3 ++- stix2/__init__.py | 4 ++-- stix2/properties.py | 7 ++++++- stix2/test/v21/test_indicator.py | 6 +++--- stix2/v21/sdo.py | 2 +- 5 files changed, 14 insertions(+), 8 deletions(-) diff --git a/README.rst b/README.rst index 0bf30fd..37890da 100644 --- a/README.rst +++ b/README.rst @@ -33,7 +33,8 @@ be set automatically if not provided as keyword arguments. from stix2 import Indicator indicator = Indicator(name="File hash for malware variant", - labels=["malicious-activity"], + indicator_types=["malicious-activity"], + pattern_type="stix", pattern="[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']") To parse a STIX JSON string into a Python STIX object, use ``parse()``: diff --git a/stix2/__init__.py b/stix2/__init__.py index 26eed6f..10384b1 100644 --- a/stix2/__init__.py +++ b/stix2/__init__.py @@ -20,7 +20,7 @@ # flake8: noqa -DEFAULT_VERSION = '2.0' # Default version will always be the latest STIX 2.X version +DEFAULT_VERSION = '2.1' # Default version will always be the latest STIX 2.X version from .confidence import scales from .datastore import CompositeDataSource @@ -53,7 +53,7 @@ from .patterns import ( RepeatQualifier, StartStopQualifier, StringConstant, TimestampConstant, WithinQualifier, ) -from .v20 import * # This import will always be the latest STIX 2.X version +from .v21 import * # This import will always be the latest STIX 2.X version from .version import __version__ from .versioning import new_version, revoke diff --git a/stix2/properties.py b/stix2/properties.py index 8bb0636..9da5c29 100644 --- a/stix2/properties.py +++ b/stix2/properties.py @@ -14,7 +14,7 @@ import stix2 from .base import _STIXBase from .exceptions import ( CustomContentError, DictionaryKeyError, MissingPropertiesError, - MutuallyExclusivePropertiesError, + MutuallyExclusivePropertiesError, STIXError, ) from .parsing import STIX2_OBJ_MAPS, parse, parse_observable from .utils import _get_dict, get_class_hierarchy_names, parse_into_datetime @@ -168,6 +168,11 @@ class Property(object): def __init__(self, required=False, fixed=None, default=None): self.required = required + + if required and default: + raise STIXError("Cant't use 'required' and 'default' together. 'required'" + "really means 'the user must provide this.'") + if fixed: self._fixed_value = fixed self.clean = self._default_clean diff --git a/stix2/test/v21/test_indicator.py b/stix2/test/v21/test_indicator.py index 76fe86b..8452f70 100644 --- a/stix2/test/v21/test_indicator.py +++ b/stix2/test/v21/test_indicator.py @@ -98,8 +98,8 @@ def test_indicator_required_properties(): stix2.v21.Indicator() assert excinfo.value.cls == stix2.v21.Indicator - assert excinfo.value.properties == ["pattern", "pattern_type", "valid_from"] - assert str(excinfo.value) == "No values for required properties for Indicator: (pattern, pattern_type, valid_from)." + assert excinfo.value.properties == ["pattern", "pattern_type"] + assert str(excinfo.value) == "No values for required properties for Indicator: (pattern, pattern_type)." def test_indicator_required_property_pattern(): @@ -107,7 +107,7 @@ def test_indicator_required_property_pattern(): stix2.v21.Indicator(indicator_types=['malicious-activity']) assert excinfo.value.cls == stix2.v21.Indicator - assert excinfo.value.properties == ["pattern", "pattern_type", "valid_from"] + assert excinfo.value.properties == ["pattern", "pattern_type"] def test_indicator_created_ref_invalid_format(): diff --git a/stix2/v21/sdo.py b/stix2/v21/sdo.py index 6961423..9418c40 100644 --- a/stix2/v21/sdo.py +++ b/stix2/v21/sdo.py @@ -187,7 +187,7 @@ class Indicator(_DomainObject): ('pattern', PatternProperty(required=True)), ('pattern_type', StringProperty(required=True)), ('pattern_version', StringProperty()), - ('valid_from', TimestampProperty(default=lambda: NOW, required=True)), + ('valid_from', TimestampProperty(default=lambda: NOW)), ('valid_until', TimestampProperty()), ('kill_chain_phases', ListProperty(KillChainPhase)), ('revoked', BooleanProperty(default=lambda: False)), From b42b1cafa7f8ecb1ade7b2ff9953415cf129364e Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Thu, 18 Jun 2020 08:58:00 -0400 Subject: [PATCH 02/30] Move a couple tests for organization --- stix2/test/v20/test_custom.py | 24 ++++++++++++------------ stix2/test/v21/test_custom.py | 24 ++++++++++++------------ 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/stix2/test/v20/test_custom.py b/stix2/test/v20/test_custom.py index 57f8aac..33be3e2 100644 --- a/stix2/test/v20/test_custom.py +++ b/stix2/test/v20/test_custom.py @@ -288,6 +288,18 @@ def test_custom_marking_no_init_2(): assert no2.property1 == 'something' +def test_register_duplicate_marking(): + with pytest.raises(DuplicateRegistrationError) as excinfo: + @stix2.v20.CustomMarking( + 'x-new-obj2', [ + ('property1', stix2.properties.StringProperty(required=True)), + ], + ) + class NewObj2(): + pass + assert "cannot be registered again" in str(excinfo.value) + + @stix2.v20.CustomObject( 'x-new-type', [ ('property1', stix2.properties.StringProperty(required=True)), @@ -1119,15 +1131,3 @@ def test_register_duplicate_observable_extension(): class NewExtension2(): pass assert "cannot be registered again" in str(excinfo.value) - - -def test_register_duplicate_marking(): - with pytest.raises(DuplicateRegistrationError) as excinfo: - @stix2.v20.CustomMarking( - 'x-new-obj-2', [ - ('property1', stix2.properties.StringProperty(required=True)), - ], - ) - class NewObj2(): - pass - assert "cannot be registered again" in str(excinfo.value) diff --git a/stix2/test/v21/test_custom.py b/stix2/test/v21/test_custom.py index 3646f11..146abcd 100644 --- a/stix2/test/v21/test_custom.py +++ b/stix2/test/v21/test_custom.py @@ -397,6 +397,18 @@ def test_custom_marking_invalid_type_name(): pass # pragma: no cover assert "Invalid type name '7x-new-marking':" in str(excinfo.value) + +def test_register_duplicate_marking(): + with pytest.raises(DuplicateRegistrationError) as excinfo: + @stix2.v21.CustomMarking( + 'x-new-obj', [ + ('property1', stix2.properties.StringProperty(required=True)), + ], + ) + class NewObj2(): + pass + assert "cannot be registered again" in str(excinfo.value) + # Custom Objects @@ -1326,15 +1338,3 @@ def test_register_duplicate_observable_extension(): class NewExtension2(): pass assert "cannot be registered again" in str(excinfo.value) - - -def test_register_duplicate_marking(): - with pytest.raises(DuplicateRegistrationError) as excinfo: - @stix2.v21.CustomMarking( - 'x-new-obj', [ - ('property1', stix2.properties.StringProperty(required=True)), - ], - ) - class NewObj2(): - pass - assert "cannot be registered again" in str(excinfo.value) From f78b396648f7cfdc09e926c8128e667da9c157c4 Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Thu, 18 Jun 2020 10:23:08 -0400 Subject: [PATCH 03/30] Fix parsing tests --- stix2/test/v20/test_parsing.py | 12 +++++++++--- stix2/test/v21/test_parsing.py | 26 ++++++++++++++++++-------- 2 files changed, 27 insertions(+), 11 deletions(-) diff --git a/stix2/test/v20/test_parsing.py b/stix2/test/v20/test_parsing.py index 32c3e8b..6eb554e 100644 --- a/stix2/test/v20/test_parsing.py +++ b/stix2/test/v20/test_parsing.py @@ -1,3 +1,5 @@ +from collections import OrderedDict + import pytest import stix2 @@ -67,8 +69,12 @@ def test_parse_observable_with_no_version(): def test_register_marking_with_version(): - parsing._register_marking(stix2.v20.TLP_WHITE.__class__, version='2.0') + class NewMarking1: + _type = 'x-new-marking1' + _properties = OrderedDict() + + parsing._register_marking(NewMarking1, version='2.0') v = 'v20' - assert stix2.v20.TLP_WHITE.definition._type in parsing.STIX2_OBJ_MAPS[v]['markings'] - assert v in str(stix2.v20.TLP_WHITE.__class__) + assert NewMarking1._type in parsing.STIX2_OBJ_MAPS[v]['markings'] + assert v in str(parsing.STIX2_OBJ_MAPS[v]['markings'][NewMarking1._type]) diff --git a/stix2/test/v21/test_parsing.py b/stix2/test/v21/test_parsing.py index 1d930e6..53b53b2 100644 --- a/stix2/test/v21/test_parsing.py +++ b/stix2/test/v21/test_parsing.py @@ -1,3 +1,5 @@ +from collections import OrderedDict + import pytest import stix2 @@ -64,7 +66,7 @@ def test_parse_observable_with_version(): @pytest.mark.xfail(reason="The default version is not 2.1", condition=stix2.DEFAULT_VERSION != "2.1") def test_parse_observable_with_no_version(): - observable = {"type": "file", "name": "foo.exe"} + observable = {"type": "file", "name": "foo.exe", "spec_version": "2.1"} obs_obj = parsing.parse_observable(observable) v = 'v21' @@ -72,18 +74,26 @@ def test_parse_observable_with_no_version(): def test_register_marking_with_version(): - parsing._register_marking(stix2.v21.TLP_WHITE.__class__, version='2.1') + class NewMarking1: + _type = 'x-new-marking1' + _properties = OrderedDict() + + parsing._register_marking(NewMarking1, version='2.1') v = 'v21' - assert stix2.v21.TLP_WHITE.definition._type in parsing.STIX2_OBJ_MAPS[v]['markings'] - assert v in str(stix2.v21.TLP_WHITE.__class__) + assert NewMarking1._type in parsing.STIX2_OBJ_MAPS[v]['markings'] + assert v in str(parsing.STIX2_OBJ_MAPS[v]['markings'][NewMarking1._type]) @pytest.mark.xfail(reason="The default version is not 2.1", condition=stix2.DEFAULT_VERSION != "2.1") def test_register_marking_with_no_version(): - # Uses default version (2.0 in this case) - parsing._register_marking(stix2.v21.TLP_WHITE.__class__) + # Uses default version (2.1 in this case) + class NewMarking2: + _type = 'x-new-marking2' + _properties = OrderedDict() + + parsing._register_marking(NewMarking2) v = 'v21' - assert stix2.v21.TLP_WHITE.definition._type in parsing.STIX2_OBJ_MAPS[v]['markings'] - assert v in str(stix2.v21.TLP_WHITE.__class__) + assert NewMarking2._type in parsing.STIX2_OBJ_MAPS[v]['markings'] + assert v in str(parsing.STIX2_OBJ_MAPS[v]['markings'][NewMarking2._type]) From e80d3bad2cc380cb9dccac5bc89399cf55c045a6 Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Thu, 18 Jun 2020 10:35:15 -0400 Subject: [PATCH 04/30] Fix 2.0 HashesProperties --- stix2/v20/common.py | 2 +- stix2/v20/observables.py | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/stix2/v20/common.py b/stix2/v20/common.py index a92f81d..f2a371e 100644 --- a/stix2/v20/common.py +++ b/stix2/v20/common.py @@ -40,7 +40,7 @@ class ExternalReference(_STIXBase20): ('source_name', StringProperty(required=True)), ('description', StringProperty()), ('url', StringProperty()), - ('hashes', HashesProperty()), + ('hashes', HashesProperty(spec_version='2.0')), ('external_id', StringProperty()), ]) diff --git a/stix2/v20/observables.py b/stix2/v20/observables.py index 3491c47..79d4e79 100644 --- a/stix2/v20/observables.py +++ b/stix2/v20/observables.py @@ -30,7 +30,7 @@ class Artifact(_Observable): ('mime_type', StringProperty()), ('payload_bin', BinaryProperty()), ('url', StringProperty()), - ('hashes', HashesProperty()), + ('hashes', HashesProperty(spec_version='2.0')), ('extensions', ExtensionsProperty(spec_version="2.0", enclosing_type=_type)), ]) @@ -173,7 +173,7 @@ class AlternateDataStream(_STIXBase20): _properties = OrderedDict([ ('name', StringProperty(required=True)), - ('hashes', HashesProperty()), + ('hashes', HashesProperty(spec_version='2.0')), ('size', IntegerProperty()), ]) @@ -256,7 +256,7 @@ class WindowsPEOptionalHeaderType(_STIXBase20): ('size_of_heap_commit', IntegerProperty()), ('loader_flags_hex', HexProperty()), ('number_of_rva_and_sizes', IntegerProperty()), - ('hashes', HashesProperty()), + ('hashes', HashesProperty(spec_version='2.0')), ]) def _check_object_constraints(self): @@ -273,7 +273,7 @@ class WindowsPESection(_STIXBase20): ('name', StringProperty(required=True)), ('size', IntegerProperty()), ('entropy', FloatProperty()), - ('hashes', HashesProperty()), + ('hashes', HashesProperty(spec_version='2.0')), ]) @@ -293,7 +293,7 @@ class WindowsPEBinaryExt(_Extension): ('number_of_symbols', IntegerProperty()), ('size_of_optional_header', IntegerProperty()), ('characteristics_hex', HexProperty()), - ('file_header_hashes', HashesProperty()), + ('file_header_hashes', HashesProperty(spec_version='2.0')), ('optional_header', EmbeddedObjectProperty(type=WindowsPEOptionalHeaderType)), ('sections', ListProperty(EmbeddedObjectProperty(type=WindowsPESection))), ]) @@ -307,7 +307,7 @@ class File(_Observable): _type = 'file' _properties = OrderedDict([ ('type', TypeProperty(_type, spec_version='2.0')), - ('hashes', HashesProperty()), + ('hashes', HashesProperty(spec_version='2.0')), ('size', IntegerProperty()), ('name', StringProperty()), ('name_enc', StringProperty()), @@ -759,7 +759,7 @@ class X509Certificate(_Observable): _properties = OrderedDict([ ('type', TypeProperty(_type, spec_version='2.0')), ('is_self_signed', BooleanProperty()), - ('hashes', HashesProperty()), + ('hashes', HashesProperty(spec_version='2.0')), ('version', StringProperty()), ('serial_number', StringProperty()), ('signature_algorithm', StringProperty()), From d62f5ee1419888fb1dee4d2381634266e73f313c Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Thu, 18 Jun 2020 11:36:18 -0400 Subject: [PATCH 05/30] Remove 'encoding' param from json.loads It's ignored since Python 3.1 and we've dropped 2.7. --- stix2/test/v20/test_datastore_taxii.py | 2 +- stix2/test/v21/test_datastore_taxii.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/stix2/test/v20/test_datastore_taxii.py b/stix2/test/v20/test_datastore_taxii.py index b0c6b77..0b21981 100644 --- a/stix2/test/v20/test_datastore_taxii.py +++ b/stix2/test/v20/test_datastore_taxii.py @@ -28,7 +28,7 @@ class MockTAXIICollectionEndpoint(Collection): def add_objects(self, bundle): self._verify_can_write() if isinstance(bundle, six.string_types): - bundle = json.loads(bundle, encoding='utf-8') + bundle = json.loads(bundle) for object in bundle.get("objects", []): self.objects.append(object) self.manifests.append( diff --git a/stix2/test/v21/test_datastore_taxii.py b/stix2/test/v21/test_datastore_taxii.py index 28a7368..92ae6dc 100644 --- a/stix2/test/v21/test_datastore_taxii.py +++ b/stix2/test/v21/test_datastore_taxii.py @@ -28,7 +28,7 @@ class MockTAXIICollectionEndpoint(Collection): def add_objects(self, bundle): self._verify_can_write() if isinstance(bundle, six.string_types): - bundle = json.loads(bundle, encoding='utf-8') + bundle = json.loads(bundle) for object in bundle.get("objects", []): self.objects.append(object) self.manifests.append( From 3ef63d5e175481ddf9860f759f2f48148ed3ea09 Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Thu, 18 Jun 2020 11:37:52 -0400 Subject: [PATCH 06/30] Update Workbench for 2.1 --- stix2/test/test_workbench.py | 67 +++++++++++++++++++++++-- stix2/test/v21/constants.py | 20 ++++++++ stix2/workbench.py | 94 ++++++++++++++++++++++++++++++++++-- 3 files changed, 172 insertions(+), 9 deletions(-) diff --git a/stix2/test/test_workbench.py b/stix2/test/test_workbench.py index d946547..433bf81 100644 --- a/stix2/test/test_workbench.py +++ b/stix2/test/test_workbench.py @@ -4,12 +4,14 @@ import os import stix2 from stix2.workbench import ( _STIX_VID, AttackPattern, Bundle, Campaign, CourseOfAction, - ExternalReference, File, FileSystemSource, Filter, Identity, Indicator, - IntrusionSet, Malware, MarkingDefinition, NTFSExt, ObservedData, + ExternalReference, File, FileSystemSource, Filter, Grouping, Identity, + Indicator, Infrastructure, IntrusionSet, Location, Malware, + MalwareAnalysis, MarkingDefinition, Note, NTFSExt, ObservedData, Opinion, Relationship, Report, StatementMarking, ThreatActor, Tool, Vulnerability, add_data_source, all_versions, attack_patterns, campaigns, - courses_of_action, create, get, identities, indicators, intrusion_sets, - malware, observed_data, query, reports, save, set_default_created, + courses_of_action, create, get, groupings, identities, indicators, + infrastructures, intrusion_sets, locations, malware, malware_analyses, + notes, observed_data, opinions, query, reports, save, set_default_created, set_default_creator, set_default_external_refs, set_default_object_marking_refs, threat_actors, tools, vulnerabilities, ) @@ -35,7 +37,7 @@ def test_workbench_environment(): save(ind) resp = get(constants.INDICATOR_ID) - assert resp['labels'][0] == 'malicious-activity' + assert resp['indicator_types'][0] == 'malicious-activity' resp = all_versions(constants.INDICATOR_ID) assert len(resp) == 1 @@ -77,6 +79,15 @@ def test_workbench_get_all_courses_of_action(): assert resp[0].id == constants.COURSE_OF_ACTION_ID +def test_workbench_get_all_groupings(): + grup = Grouping(id=constants.GROUPING_ID, **constants.GROUPING_KWARGS) + save(grup) + + resp = groupings() + assert len(resp) == 1 + assert resp[0].id == constants.GROUPING_ID + + def test_workbench_get_all_identities(): idty = Identity(id=constants.IDENTITY_ID, **constants.IDENTITY_KWARGS) save(idty) @@ -92,6 +103,15 @@ def test_workbench_get_all_indicators(): assert resp[0].id == constants.INDICATOR_ID +def test_workbench_get_all_infrastructures(): + inf = Infrastructure(id=constants.INFRASTRUCTURE_ID, **constants.INFRASTRUCTURE_KWARGS) + save(inf) + + resp = infrastructures() + assert len(resp) == 1 + assert resp[0].id == constants.INFRASTRUCTURE_ID + + def test_workbench_get_all_intrusion_sets(): ins = IntrusionSet( id=constants.INTRUSION_SET_ID, **constants.INTRUSION_SET_KWARGS @@ -103,6 +123,15 @@ def test_workbench_get_all_intrusion_sets(): assert resp[0].id == constants.INTRUSION_SET_ID +def test_workbench_get_all_locations(): + loc = Location(id=constants.LOCATION_ID, **constants.LOCATION_KWARGS) + save(loc) + + resp = locations() + assert len(resp) == 1 + assert resp[0].id == constants.LOCATION_ID + + def test_workbench_get_all_malware(): mal = Malware(id=constants.MALWARE_ID, **constants.MALWARE_KWARGS) save(mal) @@ -112,6 +141,24 @@ def test_workbench_get_all_malware(): assert resp[0].id == constants.MALWARE_ID +def test_workbench_get_all_malware_analyses(): + mal = MalwareAnalysis(id=constants.MALWARE_ANALYSIS_ID, **constants.MALWARE_ANALYSIS_KWARGS) + save(mal) + + resp = malware_analyses() + assert len(resp) == 1 + assert resp[0].id == constants.MALWARE_ANALYSIS_ID + + +def test_workbench_get_all_notes(): + note = Note(id=constants.NOTE_ID, **constants.NOTE_KWARGS) + save(note) + + resp = notes() + assert len(resp) == 1 + assert resp[0].id == constants.NOTE_ID + + def test_workbench_get_all_observed_data(): od = ObservedData( id=constants.OBSERVED_DATA_ID, **constants.OBSERVED_DATA_KWARGS @@ -123,6 +170,15 @@ def test_workbench_get_all_observed_data(): assert resp[0].id == constants.OBSERVED_DATA_ID +def test_workbench_get_all_opinions(): + op = Opinion(id=constants.OPINION_ID, **constants.OPINION_KWARGS) + save(op) + + resp = opinions() + assert len(resp) == 1 + assert resp[0].id == constants.OPINION_ID + + def test_workbench_get_all_reports(): rep = Report(id=constants.REPORT_ID, **constants.REPORT_KWARGS) save(rep) @@ -210,6 +266,7 @@ def test_workbench_related(): def test_workbench_related_with_filters(): malware = Malware( labels=["ransomware"], name="CryptorBit", created_by_ref=constants.IDENTITY_ID, + is_family=False, ) rel = Relationship(malware.id, 'variant-of', constants.MALWARE_ID) save([malware, rel]) diff --git a/stix2/test/v21/constants.py b/stix2/test/v21/constants.py index c3ce3c0..d5bef24 100644 --- a/stix2/test/v21/constants.py +++ b/stix2/test/v21/constants.py @@ -14,6 +14,7 @@ INFRASTRUCTURE_ID = "infrastructure--3000ae1b-784c-f03d-8abc-0a625b2ff018" INTRUSION_SET_ID = "intrusion-set--4e78f46f-a023-4e5f-bc24-71b3ca22ec29" LOCATION_ID = "location--a6e9345f-5a15-4c29-8bb3-7dcc5d168d64" MALWARE_ID = "malware--9c4638ec-f1de-4ddb-abf4-1b760417654e" +MALWARE_ANALYSIS_ID = "malware-analysis--b46ee0ad-9443-41c5-a8e3-0fa053262805" MARKING_DEFINITION_ID = "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9" NOTE_ID = "note--0c7b5b88-8ff7-4a4d-aa9d-feb398cd0061" OBSERVED_DATA_ID = "observed-data--b67d30ff-02ac-498a-92f9-32f845f448cf" @@ -102,6 +103,10 @@ INTRUSION_SET_KWARGS = dict( name="Bobcat Breakin", ) +LOCATION_KWARGS = dict( + region="africa", +) + MALWARE_KWARGS = dict( malware_types=['ransomware'], name="Cryptolocker", @@ -119,6 +124,16 @@ MALWARE_MORE_KWARGS = dict( is_family=False, ) +MALWARE_ANALYSIS_KWARGS = dict( + product="microsoft", + result="malicious", +) + +NOTE_KWARGS = dict( + content="Heartbleed", + object_refs=[CAMPAIGN_ID] +) + OBSERVED_DATA_KWARGS = dict( first_observed=FAKE_TIME, last_observed=FAKE_TIME, @@ -131,6 +146,11 @@ OBSERVED_DATA_KWARGS = dict( }, ) +OPINION_KWARGS = dict( + opinion="agree", + object_refs=[CAMPAIGN_ID] +) + REPORT_KWARGS = dict( report_types=["campaign"], name="Bad Cybercrime", diff --git a/stix2/workbench.py b/stix2/workbench.py index 57474a9..7b3f46a 100644 --- a/stix2/workbench.py +++ b/stix2/workbench.py @@ -25,11 +25,17 @@ import stix2 from . import AttackPattern as _AttackPattern from . import Campaign as _Campaign from . import CourseOfAction as _CourseOfAction +from . import Grouping as _Grouping from . import Identity as _Identity from . import Indicator as _Indicator +from . import Infrastructure as _Infrastructure from . import IntrusionSet as _IntrusionSet +from . import Location as _Location from . import Malware as _Malware +from . import MalwareAnalysis as _MalwareAnalysis +from . import Note as _Note from . import ObservedData as _ObservedData +from . import Opinion as _Opinion from . import Report as _Report from . import ThreatActor as _ThreatActor from . import Tool as _Tool @@ -40,7 +46,7 @@ from . import ( # noqa: F401 Directory, DomainName, EmailAddress, EmailMessage, EmailMIMEComponent, Environment, ExternalReference, File, FileSystemSource, Filter, GranularMarking, HTTPRequestExt, - ICMPExt, IPv4Address, IPv6Address, KillChainPhase, MACAddress, + ICMPExt, IPv4Address, IPv6Address, KillChainPhase, LanguageContent, MACAddress, MarkingDefinition, MemoryStore, Mutex, NetworkTraffic, NTFSExt, parse_observable, PDFExt, Process, RasterImageExt, Relationship, Sighting, SocketExt, Software, StatementMarking, @@ -56,6 +62,7 @@ from .datastore.filters import FilterSet # Enable some adaptation to the current default supported STIX version. _STIX_VID = "v" + stix2.DEFAULT_VERSION.replace(".", "") +print(_STIX_VID) # Use an implicit MemoryStore @@ -84,12 +91,13 @@ add_data_sources = _environ.source.add_data_sources STIX_OBJS = [ - _AttackPattern, _Campaign, _CourseOfAction, _Identity, - _Indicator, _IntrusionSet, _Malware, _ObservedData, _Report, + _AttackPattern, _Campaign, _CourseOfAction, _Identity, _Grouping, + _Indicator, _Infrastructure, _IntrusionSet, _Location, _Malware, + _MalwareAnalysis, _Note, _ObservedData, _Opinion, _Report, _ThreatActor, _Tool, _Vulnerability, ] -STIX_OBJ_DOCS = """ +STIX_OBJ_DOCS = """s .. method:: created_by(*args, **kwargs) @@ -202,6 +210,19 @@ def courses_of_action(filters=None): return query(filter_list) +def groupings(filters=None): + """Retrieve all Grouping objects. + + Args: + filters (list, optional): A list of additional filters to apply to + the query. + + """ + filter_list = FilterSet(filters) + filter_list.add(Filter('type', '=', 'grouping')) + return query(filter_list) + + def identities(filters=None): """Retrieve all Identity objects. @@ -228,6 +249,19 @@ def indicators(filters=None): return query(filter_list) +def infrastructures(filters=None): + """Retrieve all Infrastructure objects. + + Args: + filters (list, optional): A list of additional filters to apply to + the query. + + """ + filter_list = FilterSet(filters) + filter_list.add(Filter('type', '=', 'infrastructure')) + return query(filter_list) + + def intrusion_sets(filters=None): """Retrieve all Intrusion Set objects. @@ -241,6 +275,19 @@ def intrusion_sets(filters=None): return query(filter_list) +def locations(filters=None): + """Retrieve all Location objects. + + Args: + filters (list, optional): A list of additional filters to apply to + the query. + + """ + filter_list = FilterSet(filters) + filter_list.add(Filter('type', '=', 'location')) + return query(filter_list) + + def malware(filters=None): """Retrieve all Malware objects. @@ -254,6 +301,32 @@ def malware(filters=None): return query(filter_list) +def malware_analyses(filters=None): + """Retrieve all Malware Analysis objects. + + Args: + filters (list, optional): A list of additional filters to apply to + the query. + + """ + filter_list = FilterSet(filters) + filter_list.add(Filter('type', '=', 'malware-analysis')) + return query(filter_list) + + +def notes(filters=None): + """Retrieve all Note objects. + + Args: + filters (list, optional): A list of additional filters to apply to + the query. + + """ + filter_list = FilterSet(filters) + filter_list.add(Filter('type', '=', 'note')) + return query(filter_list) + + def observed_data(filters=None): """Retrieve all Observed Data objects. @@ -267,6 +340,19 @@ def observed_data(filters=None): return query(filter_list) +def opinions(filters=None): + """Retrieve all Opinion objects. + + Args: + filters (list, optional): A list of additional filters to apply to + the query. + + """ + filter_list = FilterSet(filters) + filter_list.add(Filter('type', '=', 'opinion')) + return query(filter_list) + + def reports(filters=None): """Retrieve all Report objects. From 10bdfe9e25a333a34f8ed07f1531182153de6ca9 Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Fri, 26 Jun 2020 16:22:01 -0400 Subject: [PATCH 07/30] Fix KeyError in TAXII DataStore --- stix2/datastore/taxii.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stix2/datastore/taxii.py b/stix2/datastore/taxii.py index b5c0827..063fabd 100644 --- a/stix2/datastore/taxii.py +++ b/stix2/datastore/taxii.py @@ -287,7 +287,7 @@ class TAXIICollectionSource(DataSource): # query TAXII collection try: - all_data = self.collection.get_objects(**taxii_filters_dict)['objects'] + all_data = self.collection.get_objects(**taxii_filters_dict).get('objects', []) # deduplicate data (before filtering as reduces wasted filtering) all_data = deduplicate(all_data) From d5d664ba417b6e7a065c96c825fbf7ea34a80b17 Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Fri, 26 Jun 2020 16:22:50 -0400 Subject: [PATCH 08/30] Update docs for 2.1 as default --- README.rst | 1 + docs/guide/creating.ipynb | 1458 +++++++++++++------------- docs/guide/custom.ipynb | 88 +- docs/guide/datastore.ipynb | 92 +- docs/guide/environment.ipynb | 101 +- docs/guide/equivalence.ipynb | 68 +- docs/guide/filesystem.ipynb | 613 ++++++++++- docs/guide/markings.ipynb | 1923 ++++++++++++++++++++++------------ docs/guide/memory.ipynb | 71 +- docs/guide/parsing.ipynb | 26 +- docs/guide/patterns.ipynb | 1235 ++++++++++++++++++++-- docs/guide/serializing.ipynb | 35 +- docs/guide/taxii.ipynb | 1036 ++++++++++++++++-- docs/guide/ts_support.ipynb | 22 +- docs/guide/versioning.ipynb | 272 ++--- docs/guide/workbench.ipynb | 44 +- stix2/workbench.py | 1 - 17 files changed, 5034 insertions(+), 2052 deletions(-) diff --git a/README.rst b/README.rst index 37890da..0a32af2 100644 --- a/README.rst +++ b/README.rst @@ -54,6 +54,7 @@ To parse a STIX JSON string into a Python STIX object, use ``parse()``: "malicious-activity" ], "pattern_type": "stix", + "pattern_version": "2.1", "pattern": "[file:hashes.md5 ='d41d8cd98f00b204e9800998ecf8427e']", "valid_from": "2017-09-26T23:33:39.829952Z" }""") diff --git a/docs/guide/creating.ipynb b/docs/guide/creating.ipynb index 6544e89..4919452 100644 --- a/docs/guide/creating.ipynb +++ b/docs/guide/creating.ipynb @@ -10,11 +10,13 @@ "source": [ "# Delete this cell to re-enable tracebacks\n", "import sys\n", + "import traceback\n", "ipython = get_ipython()\n", "\n", "def hide_traceback(exc_tuple=None, filename=None, tb_offset=None,\n", " exception_only=False, running_compiled_code=False):\n", - " etype, value, tb = sys.exc_info()\n", + " etype, value, tb = sys.exc_info() \n", + " value.__cause__ = None # suppress chained exceptions\n", " return ipython._showtraceback(etype, value, ipython.InteractiveTB.get_exception_only(etype, value))\n", "\n", "ipython.showtraceback = hide_traceback" @@ -66,689 +68,690 @@ "To create a STIX object, provide keyword arguments to the type's constructor:" ] }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
{\n",
-       "    "type": "indicator",\n",
-       "    "id": "indicator--2f3d4926-163d-4aef-bcd2-19dea96916ae",\n",
-       "    "created": "2019-05-13T13:14:48.509Z",\n",
-       "    "modified": "2019-05-13T13:14:48.509Z",\n",
-       "    "name": "File hash for malware variant",\n",
-       "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
-       "    "valid_from": "2019-05-13T13:14:48.509629Z",\n",
-       "    "labels": [\n",
-       "        "malicious-activity"\n",
-       "    ]\n",
-       "}\n",
-       "
\n" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from stix2 import Indicator\n", - "\n", - "indicator = Indicator(name=\"File hash for malware variant\",\n", - " labels=[\"malicious-activity\"],\n", - " pattern=\"[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']\")\n", - "print(indicator)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Certain required attributes of all objects will be set automatically if not provided as keyword arguments:\n", - "\n", - "- If not provided, ``type`` will be set automatically to the correct type. You can also provide the type explicitly, but this is not necessary:" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "indicator2 = Indicator(type='indicator',\n", - " labels=[\"malicious-activity\"],\n", - " pattern=\"[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Passing a value for ``type`` that does not match the class being constructed will cause an error:" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "ename": "InvalidValueError", - "evalue": "Invalid value for Indicator 'type': must equal 'indicator'.", - "output_type": "error", - "traceback": [ - "\u001b[0;31mInvalidValueError\u001b[0m\u001b[0;31m:\u001b[0m Invalid value for Indicator 'type': must equal 'indicator'.\n" - ] - } - ], - "source": [ - "indicator3 = Indicator(type='xxx',\n", - " labels=[\"malicious-activity\"],\n", - " pattern=\"[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "- If not provided, ``id`` will be generated randomly. If you provide an\n", - " ``id`` argument, it must begin with the correct prefix:" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "ename": "InvalidValueError", - "evalue": "Invalid value for Indicator 'id': must start with 'indicator--'.", - "output_type": "error", - "traceback": [ - "\u001b[0;31mInvalidValueError\u001b[0m\u001b[0;31m:\u001b[0m Invalid value for Indicator 'id': must start with 'indicator--'.\n" - ] - } - ], - "source": [ - "indicator4 = Indicator(id=\"campaign--63ce9068-b5ab-47fa-a2cf-a602ea01f21a\",\n", - " labels=[\"malicious-activity\"],\n", - " pattern=\"[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "For indicators, ``labels`` and ``pattern`` are required and cannot be set automatically. Trying to create an indicator that is missing one of these properties will result in an error:" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "ename": "MissingPropertiesError", - "evalue": "No values for required properties for Indicator: (labels, pattern).", - "output_type": "error", - "traceback": [ - "\u001b[0;31mMissingPropertiesError\u001b[0m\u001b[0;31m:\u001b[0m No values for required properties for Indicator: (labels, pattern).\n" - ] - } - ], - "source": [ - "indicator = Indicator()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "However, the required ``valid_from`` attribute on Indicators will be set to the current time if not provided as a keyword argument.\n", - "\n", - "Once created, the object acts like a frozen dictionary. Properties can be accessed using the standard Python dictionary syntax:" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'File hash for malware variant'" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "indicator['name']" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Or access properties using the standard Python attribute syntax:" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'File hash for malware variant'" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "indicator.name" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "
\n", - "\n", - "**Warning**\n", - "\n", - "Note that there are several attributes on these objects used for method names. Accessing those will return a bound method, not the attribute value.\n", - "\n", - "
\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Attempting to modify any attributes will raise an error:" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "ename": "TypeError", - "evalue": "'Indicator' object does not support item assignment", - "output_type": "error", - "traceback": [ - "\u001b[0;31mTypeError\u001b[0m\u001b[0;31m:\u001b[0m 'Indicator' object does not support item assignment\n" - ] - } - ], - "source": [ - "indicator['name'] = \"This is a revised name\"" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "ename": "ImmutableError", - "evalue": "Cannot modify 'name' property in 'Indicator' after creation.", - "output_type": "error", - "traceback": [ - "\u001b[0;31mImmutableError\u001b[0m\u001b[0;31m:\u001b[0m Cannot modify 'name' property in 'Indicator' after creation.\n" - ] - } - ], - "source": [ - "indicator.name = \"This is a revised name\"" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "To update the properties of an object, see the [Versioning](versioning.ipynb) section." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Creating a Malware object follows the same pattern:" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
{\n",
-       "    "type": "malware",\n",
-       "    "id": "malware--1f2aba70-f0ae-49cd-9267-6fcb1e43be67",\n",
-       "    "created": "2019-05-13T13:15:04.698Z",\n",
-       "    "modified": "2019-05-13T13:15:04.698Z",\n",
-       "    "name": "Poison Ivy",\n",
-       "    "labels": [\n",
-       "        "remote-access-trojan"\n",
-       "    ]\n",
-       "}\n",
-       "
\n" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from stix2 import Malware\n", - "\n", - "malware = Malware(name=\"Poison Ivy\",\n", - " labels=['remote-access-trojan'])\n", - "print(malware)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "As with indicators, the ``type``, ``id``, ``created``, and ``modified`` properties will be set automatically if not provided. For Malware objects, the ``labels`` and ``name`` properties must be provided.\n", - "\n", - "You can see the full list of SDO classes [here](../api/v20/stix2.v20.sdo.rst)." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Creating Relationships\n", - "\n", - "STIX 2 Relationships are separate objects, not properties of the object on either side of the relationship. They are constructed similarly to other STIX objects. The ``type``, ``id``, ``created``, and ``modified`` properties are added automatically if not provided. Callers must provide the ``relationship_type``, ``source_ref``, and ``target_ref`` properties." - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
{\n",
-       "    "type": "relationship",\n",
-       "    "id": "relationship--80c174fa-36d1-47c2-9a9d-ce0c636bedcc",\n",
-       "    "created": "2019-05-13T13:15:13.152Z",\n",
-       "    "modified": "2019-05-13T13:15:13.152Z",\n",
-       "    "relationship_type": "indicates",\n",
-       "    "source_ref": "indicator--2f3d4926-163d-4aef-bcd2-19dea96916ae",\n",
-       "    "target_ref": "malware--1f2aba70-f0ae-49cd-9267-6fcb1e43be67"\n",
-       "}\n",
-       "
\n" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from stix2 import Relationship\n", - "\n", - "relationship = Relationship(relationship_type='indicates',\n", - " source_ref=indicator.id,\n", - " target_ref=malware.id)\n", - "print(relationship)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The ``source_ref`` and ``target_ref`` properties can be either the ID's of other STIX objects, or the STIX objects themselves. For readability, Relationship objects can also be constructed with the ``source_ref``, ``relationship_type``, and ``target_ref`` as positional (non-keyword) arguments:" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
{\n",
-       "    "type": "relationship",\n",
-       "    "id": "relationship--47395d23-dedd-45d4-8db1-c9ffaf44493d",\n",
-       "    "created": "2019-05-13T13:15:16.566Z",\n",
-       "    "modified": "2019-05-13T13:15:16.566Z",\n",
-       "    "relationship_type": "indicates",\n",
-       "    "source_ref": "indicator--2f3d4926-163d-4aef-bcd2-19dea96916ae",\n",
-       "    "target_ref": "malware--1f2aba70-f0ae-49cd-9267-6fcb1e43be67"\n",
-       "}\n",
-       "
\n" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "relationship2 = Relationship(indicator, 'indicates', malware)\n", - "print(relationship2)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Creating Bundles\n", - "\n", - "STIX Bundles can be created by passing objects as arguments to the Bundle constructor. All required properties (``type``, ``id``, and ``spec_version``) will be set automatically if not provided, or can be provided as keyword arguments:" - ] - }, { "cell_type": "code", "execution_count": 15, "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
{\n",
+       "    "type": "indicator",\n",
+       "    "spec_version": "2.1",\n",
+       "    "id": "indicator--a862ff86-68d9-42e5-8095-cd80c040e112",\n",
+       "    "created": "2020-06-24T15:04:40.048932Z",\n",
+       "    "modified": "2020-06-24T15:04:40.048932Z",\n",
+       "    "name": "File hash for malware variant",\n",
+       "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
+       "    "pattern_type": "stix",\n",
+       "    "pattern_version": "2.1",\n",
+       "    "valid_from": "2020-06-24T15:04:40.048932Z"\n",
+       "}\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from stix2 import Indicator\n", + "\n", + "indicator = Indicator(name=\"File hash for malware variant\",\n", + " pattern=\"[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']\",\n", + " pattern_type=\"stix\")\n", + "print(indicator)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Certain required attributes of all objects will be set automatically if not provided as keyword arguments:\n", + "\n", + "- If not provided, ``type`` will be set automatically to the correct type. You can also provide the type explicitly, but this is not necessary:" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [], + "source": [ + "indicator2 = Indicator(type='indicator',\n", + " pattern_type=\"stix\",\n", + " pattern=\"[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Passing a value for ``type`` that does not match the class being constructed will cause an error:" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "ename": "InvalidValueError", + "evalue": "Invalid value for Indicator 'type': must equal 'indicator'.", + "output_type": "error", + "traceback": [ + "\u001b[0;31mInvalidValueError\u001b[0m\u001b[0;31m:\u001b[0m Invalid value for Indicator 'type': must equal 'indicator'.\n" + ] + } + ], + "source": [ + "indicator3 = Indicator(type='xxx',\n", + " pattern_type=\"stix\",\n", + " pattern=\"[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "- If not provided, ``id`` will be generated randomly. If you provide an\n", + " ``id`` argument, it must begin with the correct prefix:" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "ename": "InvalidValueError", + "evalue": "Invalid value for Indicator 'id': must start with 'indicator--'.", + "output_type": "error", + "traceback": [ + "\u001b[0;31mInvalidValueError\u001b[0m\u001b[0;31m:\u001b[0m Invalid value for Indicator 'id': must start with 'indicator--'.\n" + ] + } + ], + "source": [ + "indicator4 = Indicator(id=\"campaign--63ce9068-b5ab-47fa-a2cf-a602ea01f21a\",\n", + " pattern_type=\"stix\",\n", + " pattern=\"[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "For indicators, ``pattern`` and ``pattern_type`` are required and cannot be set automatically. Trying to create an indicator that is missing one of these properties will result in an error:" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "ename": "MissingPropertiesError", + "evalue": "No values for required properties for Indicator: (pattern, pattern_type).", + "output_type": "error", + "traceback": [ + "\u001b[0;31mMissingPropertiesError\u001b[0m\u001b[0;31m:\u001b[0m No values for required properties for Indicator: (pattern, pattern_type).\n" + ] + } + ], + "source": [ + "indicator = Indicator()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "However, the required ``valid_from`` attribute on Indicators will be set to the current time if not provided as a keyword argument.\n", + "\n", + "Once created, the object acts like a frozen dictionary. Properties can be accessed using the standard Python dictionary syntax:" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'File hash for malware variant'" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "indicator['name']" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Or access properties using the standard Python attribute syntax:" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'File hash for malware variant'" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "indicator.name" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "
\n", + "\n", + "**Warning**\n", + "\n", + "Note that there are several attributes on these objects used for method names. Accessing those will return a bound method, not the attribute value.\n", + "\n", + "
\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Attempting to modify any attributes will raise an error:" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "ename": "TypeError", + "evalue": "'Indicator' object does not support item assignment", + "output_type": "error", + "traceback": [ + "\u001b[0;31mTypeError\u001b[0m\u001b[0;31m:\u001b[0m 'Indicator' object does not support item assignment\n" + ] + } + ], + "source": [ + "indicator['name'] = \"This is a revised name\"" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "ename": "ImmutableError", + "evalue": "Cannot modify 'name' property in 'Indicator' after creation.", + "output_type": "error", + "traceback": [ + "\u001b[0;31mImmutableError\u001b[0m\u001b[0;31m:\u001b[0m Cannot modify 'name' property in 'Indicator' after creation.\n" + ] + } + ], + "source": [ + "indicator.name = \"This is a revised name\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To update the properties of an object, see the [Versioning](versioning.ipynb) section." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Creating a Malware object follows the same pattern:" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
{\n",
+       "    "type": "malware",\n",
+       "    "spec_version": "2.1",\n",
+       "    "id": "malware--389c934c-258c-44fb-ae4b-14c6c12270f6",\n",
+       "    "created": "2020-06-24T14:53:20.156644Z",\n",
+       "    "modified": "2020-06-24T14:53:20.156644Z",\n",
+       "    "name": "Poison Ivy",\n",
+       "    "is_family": false\n",
+       "}\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from stix2 import Malware\n", + "\n", + "malware = Malware(name=\"Poison Ivy\",\n", + " is_family=False)\n", + "print(malware)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As with indicators, the ``type``, ``id``, ``created``, and ``modified`` properties will be set automatically if not provided. For Malware objects, the ``is_family`` property must be provided.\n", + "\n", + "You can see the full list of SDO classes [here](../api/v21/stix2.v21.sdo.rst)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Creating Relationships\n", + "\n", + "STIX 2 Relationships are separate objects, not properties of the object on either side of the relationship. They are constructed similarly to other STIX objects. The ``type``, ``id``, ``created``, and ``modified`` properties are added automatically if not provided. Callers must provide the ``relationship_type``, ``source_ref``, and ``target_ref`` properties." + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
{\n",
+       "    "type": "relationship",\n",
+       "    "spec_version": "2.1",\n",
+       "    "id": "relationship--2f6a8785-e27b-487e-b870-b85a2121502d",\n",
+       "    "created": "2020-06-24T15:05:18.250605Z",\n",
+       "    "modified": "2020-06-24T15:05:18.250605Z",\n",
+       "    "relationship_type": "indicates",\n",
+       "    "source_ref": "indicator--a862ff86-68d9-42e5-8095-cd80c040e112",\n",
+       "    "target_ref": "malware--389c934c-258c-44fb-ae4b-14c6c12270f6"\n",
+       "}\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from stix2 import Relationship\n", + "\n", + "relationship = Relationship(relationship_type='indicates',\n", + " source_ref=indicator.id,\n", + " target_ref=malware.id)\n", + "print(relationship)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The ``source_ref`` and ``target_ref`` properties can be either the ID's of other STIX objects, or the STIX objects themselves. For readability, Relationship objects can also be constructed with the ``source_ref``, ``relationship_type``, and ``target_ref`` as positional (non-keyword) arguments:" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
{\n",
+       "    "type": "relationship",\n",
+       "    "spec_version": "2.1",\n",
+       "    "id": "relationship--d43ec245-5496-44f4-8732-3131380435de",\n",
+       "    "created": "2020-06-24T15:05:47.705352Z",\n",
+       "    "modified": "2020-06-24T15:05:47.705352Z",\n",
+       "    "relationship_type": "indicates",\n",
+       "    "source_ref": "indicator--a862ff86-68d9-42e5-8095-cd80c040e112",\n",
+       "    "target_ref": "malware--389c934c-258c-44fb-ae4b-14c6c12270f6"\n",
+       "}\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "relationship2 = Relationship(indicator, 'indicates', malware)\n", + "print(relationship2)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Creating Bundles\n", + "\n", + "STIX Bundles can be created by passing objects as arguments to the Bundle constructor. All required properties (``type``, ``id``, and ``spec_version``) will be set automatically if not provided, or can be provided as keyword arguments:" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, "outputs": [ { "data": { @@ -823,39 +826,38 @@ ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n", ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "bundle",\n",
-       "    "id": "bundle--388c9b2c-936c-420a-baa5-04f48d682a01",\n",
-       "    "spec_version": "2.0",\n",
+       "    "id": "bundle--177c6477-2dee-43d5-b4c9-8b7f3f5ec517",\n",
        "    "objects": [\n",
        "        {\n",
        "            "type": "indicator",\n",
-       "            "id": "indicator--2f3d4926-163d-4aef-bcd2-19dea96916ae",\n",
-       "            "created": "2019-05-13T13:14:48.509Z",\n",
-       "            "modified": "2019-05-13T13:14:48.509Z",\n",
+       "            "spec_version": "2.1",\n",
+       "            "id": "indicator--a862ff86-68d9-42e5-8095-cd80c040e112",\n",
+       "            "created": "2020-06-24T15:04:40.048932Z",\n",
+       "            "modified": "2020-06-24T15:04:40.048932Z",\n",
        "            "name": "File hash for malware variant",\n",
        "            "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
-       "            "valid_from": "2019-05-13T13:14:48.509629Z",\n",
-       "            "labels": [\n",
-       "                "malicious-activity"\n",
-       "            ]\n",
+       "            "pattern_type": "stix",\n",
+       "            "pattern_version": "2.1",\n",
+       "            "valid_from": "2020-06-24T15:04:40.048932Z"\n",
        "        },\n",
        "        {\n",
        "            "type": "malware",\n",
-       "            "id": "malware--1f2aba70-f0ae-49cd-9267-6fcb1e43be67",\n",
-       "            "created": "2019-05-13T13:15:04.698Z",\n",
-       "            "modified": "2019-05-13T13:15:04.698Z",\n",
+       "            "spec_version": "2.1",\n",
+       "            "id": "malware--389c934c-258c-44fb-ae4b-14c6c12270f6",\n",
+       "            "created": "2020-06-24T14:53:20.156644Z",\n",
+       "            "modified": "2020-06-24T14:53:20.156644Z",\n",
        "            "name": "Poison Ivy",\n",
-       "            "labels": [\n",
-       "                "remote-access-trojan"\n",
-       "            ]\n",
+       "            "is_family": false\n",
        "        },\n",
        "        {\n",
        "            "type": "relationship",\n",
-       "            "id": "relationship--80c174fa-36d1-47c2-9a9d-ce0c636bedcc",\n",
-       "            "created": "2019-05-13T13:15:13.152Z",\n",
-       "            "modified": "2019-05-13T13:15:13.152Z",\n",
+       "            "spec_version": "2.1",\n",
+       "            "id": "relationship--2f6a8785-e27b-487e-b870-b85a2121502d",\n",
+       "            "created": "2020-06-24T15:05:18.250605Z",\n",
+       "            "modified": "2020-06-24T15:05:18.250605Z",\n",
        "            "relationship_type": "indicates",\n",
-       "            "source_ref": "indicator--2f3d4926-163d-4aef-bcd2-19dea96916ae",\n",
-       "            "target_ref": "malware--1f2aba70-f0ae-49cd-9267-6fcb1e43be67"\n",
+       "            "source_ref": "indicator--a862ff86-68d9-42e5-8095-cd80c040e112",\n",
+       "            "target_ref": "malware--389c934c-258c-44fb-ae4b-14c6c12270f6"\n",
        "        }\n",
        "    ]\n",
        "}\n",
@@ -865,7 +867,7 @@
        ""
       ]
      },
-     "execution_count": 15,
+     "execution_count": 21,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -882,22 +884,14 @@
    "metadata": {},
    "source": [
     "### Creating Cyber Observable References\n",
-    "Cyber Observable Objects have properties that can reference other Cyber Observable Objects. In order to create those references, use the ``_valid_refs`` property as shown in the following examples. It should be noted that ``_valid_refs`` is necessary when creating references to Cyber Observable Objects since some embedded references can only point to certain types, and ``_valid_refs`` helps ensure consistency. \n",
+    "Cyber Observable Objects have properties that can reference other Cyber Observable Objects. In order to create those references, either supply the ID string of the object being referenced, or pass in the object itself.\n",
     "\n",
-    "There are two cases."
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "#### Case 1: Specifying the type of the Cyber Observable Objects being referenced\n",
-    "In the following example, the IPv4Address object has its ``resolves_to_refs`` property specified. As per the spec, this property's value must be a list of reference(s) to MACAddress objects. In this case, those references are strings that state the type of the Cyber Observable Object being referenced, and are provided in ``_valid_refs``."
+    "For example, the IPv4Address object has a ``resolves_to_refs`` property which must hold a list of references to MACAddress objects. We could specify the id string:"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 16,
+   "execution_count": 22,
    "metadata": {},
    "outputs": [
     {
@@ -973,11 +967,12 @@
        ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n",
        ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "ipv4-addr",\n",
+       "    "id": "ipv4-addr--dc63603e-e634-5357-b239-d4b562bc5445",\n",
        "    "value": "177.60.40.7",\n",
        "    "resolves_to_refs": [\n",
-       "        "1",\n",
-       "        "2"\n",
-       "    ]\n",
+       "        "mac-addr--43f380fd-37c6-476d-8643-60849bf9240e"\n",
+       "    ],\n",
+       "    "spec_version": "2.1"\n",
        "}\n",
        "
\n" ], @@ -985,7 +980,7 @@ "" ] }, - "execution_count": 16, + "execution_count": 22, "metadata": {}, "output_type": "execute_result" } @@ -994,9 +989,8 @@ "from stix2 import IPv4Address\n", "\n", "ip4 = IPv4Address(\n", - " _valid_refs={\"1\": \"mac-addr\", \"2\": \"mac-addr\"},\n", " value=\"177.60.40.7\",\n", - " resolves_to_refs=[\"1\", \"2\"]\n", + " resolves_to_refs=[\"mac-addr--43f380fd-37c6-476d-8643-60849bf9240e\"]\n", ")\n", "\n", "print(ip4)" @@ -1006,13 +1000,12 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "#### Case 2: Specifying the name of the Cyber Observable Objects being referenced\n", - "The following example is just like the one provided in Case 1 above, with one key difference: instead of using strings to specify the type of the Cyber Observable Objects being referenced in ``_valid_refs``, the referenced Cyber Observable Objects are created beforehand and then their names are provided in ``_valid_refs``." + "Or we could create the MACAddress object(s) beforehand and then pass them in:" ] }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 23, "metadata": {}, "outputs": [ { @@ -1088,11 +1081,13 @@ ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n", ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "ipv4-addr",\n",
+       "    "id": "ipv4-addr--dc63603e-e634-5357-b239-d4b562bc5445",\n",
        "    "value": "177.60.40.7",\n",
        "    "resolves_to_refs": [\n",
-       "        "1",\n",
-       "        "2"\n",
-       "    ]\n",
+       "        "mac-addr--f72d7d00-86bd-5cd2-8c86-52f7a83bef62",\n",
+       "        "mac-addr--875ad625-177b-5c2a-9101-d44b0ad55938"\n",
+       "    ],\n",
+       "    "spec_version": "2.1"\n",
        "}\n",
        "
\n" ], @@ -1100,7 +1095,7 @@ "" ] }, - "execution_count": 17, + "execution_count": 23, "metadata": {}, "output_type": "execute_result" } @@ -1112,9 +1107,8 @@ "mac_addr_b = MACAddress(value=\"a7:b8:c9:d0:e1:f2\")\n", "\n", "ip4_valid_refs = IPv4Address(\n", - " _valid_refs={\"1\": mac_addr_a, \"2\": mac_addr_b},\n", " value=\"177.60.40.7\",\n", - " resolves_to_refs=[\"1\", \"2\"]\n", + " resolves_to_refs=[mac_addr_a.id, mac_addr_b.id]\n", ")\n", "\n", "print(ip4_valid_refs)" @@ -1123,21 +1117,21 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 2", + "display_name": "Python 3", "language": "python", - "name": "python2" + "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", - "version": 2 + "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython2", - "version": "2.7.15" + "pygments_lexer": "ipython3", + "version": "3.9.0a6" } }, "nbformat": 4, diff --git a/docs/guide/custom.ipynb b/docs/guide/custom.ipynb index 8185269..3ad105d 100644 --- a/docs/guide/custom.ipynb +++ b/docs/guide/custom.ipynb @@ -15,6 +15,7 @@ "def hide_traceback(exc_tuple=None, filename=None, tb_offset=None,\n", " exception_only=False, running_compiled_code=False):\n", " etype, value, tb = sys.exc_info()\n", + " value.__cause__ = None # suppress chained exceptions\n", " return ipython._showtraceback(etype, value, ipython.InteractiveTB.get_exception_only(etype, value))\n", "\n", "ipython.showtraceback = hide_traceback" @@ -175,9 +176,10 @@ ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n", ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "identity",\n",
-       "    "id": "identity--d6996982-5fb7-4364-b716-b618516989b6",\n",
-       "    "created": "2020-03-05T05:06:27.349Z",\n",
-       "    "modified": "2020-03-05T05:06:27.349Z",\n",
+       "    "spec_version": "2.1",\n",
+       "    "id": "identity--a4c49251-0ad1-44e6-8cfc-3dbd75e73fbd",\n",
+       "    "created": "2020-06-24T18:29:07.107425Z",\n",
+       "    "modified": "2020-06-24T18:29:07.107425Z",\n",
        "    "name": "John Smith",\n",
        "    "identity_class": "individual",\n",
        "    "x_foo": "bar"\n",
@@ -287,9 +289,10 @@
        ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n",
        ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "identity",\n",
-       "    "id": "identity--a167d2de-9fc4-4734-a1ae-57a548aad22a",\n",
-       "    "created": "2020-03-05T05:06:29.180Z",\n",
-       "    "modified": "2020-03-05T05:06:29.180Z",\n",
+       "    "spec_version": "2.1",\n",
+       "    "id": "identity--50c33f36-362b-4815-9f97-f3c7f39aa691",\n",
+       "    "created": "2020-06-24T18:29:15.435425Z",\n",
+       "    "modified": "2020-06-24T18:29:15.435425Z",\n",
        "    "name": "John Smith",\n",
        "    "identity_class": "individual",\n",
        "    "x_foo": "bar"\n",
@@ -428,7 +431,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "To remove a custom properties, use `new_version()` and set it to `None`."
+    "To remove a custom properties, use `new_version()` and set that property to `None`."
    ]
   },
   {
@@ -511,7 +514,7 @@
        "    "type": "identity",\n",
        "    "id": "identity--311b2d2d-f010-4473-83ec-1edf84858f4c",\n",
        "    "created": "2015-12-21T19:59:11.000Z",\n",
-       "    "modified": "2020-03-05T05:06:32.934Z",\n",
+       "    "modified": "2020-06-24T18:29:24.099Z",\n",
        "    "name": "John Smith",\n",
        "    "identity_class": "individual"\n",
        "}\n",
@@ -537,7 +540,7 @@
    "source": [
     "### Custom STIX Object Types\n",
     "\n",
-    "To create a custom STIX object type, define a class with the @[CustomObject](../api/v20/stix2.v20.sdo.rst#stix2.v20.sdo.CustomObject) decorator. It takes the type name and a list of property tuples, each tuple consisting of the property name and a property instance. Any special validation of the properties can be added by supplying an ``__init__`` function.\n",
+    "To create a custom STIX object type, define a class with the @[CustomObject](../api/v21/stix2.v21.sdo.rst#stix2.v21.sdo.CustomObject) decorator. It takes the type name and a list of property tuples, each tuple consisting of the property name and a property instance. Any special validation of the properties can be added by supplying an ``__init__`` function.\n",
     "\n",
     "Let's say zoo animals have become a serious cyber threat and we want to model them in STIX using a custom object type. Let's use a ``species`` property to store the kind of animal, and make that property required. We also want a property to store the class of animal, such as \"mammal\" or \"bird\" but only want to allow specific values in it. We can add some logic to validate this property in ``__init__``."
    ]
@@ -645,9 +648,10 @@
        ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n",
        ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "x-animal",\n",
-       "    "id": "x-animal--1f7ce0ad-fd3a-4cf0-9cd7-13f7bef9ecd4",\n",
-       "    "created": "2020-03-05T05:06:38.010Z",\n",
-       "    "modified": "2020-03-05T05:06:38.010Z",\n",
+       "    "spec_version": "2.1",\n",
+       "    "id": "x-animal--c7dbda16-360a-4622-b9c7-91f0497167cc",\n",
+       "    "created": "2020-06-24T18:33:29.856926Z",\n",
+       "    "modified": "2020-06-24T18:33:29.856926Z",\n",
        "    "species": "lion",\n",
        "    "animal_class": "mammal"\n",
        "}\n",
@@ -703,7 +707,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 11,
+   "execution_count": 12,
    "metadata": {},
    "outputs": [
     {
@@ -784,7 +788,7 @@
        ""
       ]
      },
-     "execution_count": 11,
+     "execution_count": 12,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -795,6 +799,7 @@
     "    \"id\": \"x-animal--941f1471-6815-456b-89b8-7051ddf13e4b\",\n",
     "    \"created\": \"2015-12-21T19:59:11Z\",\n",
     "    \"modified\": \"2015-12-21T19:59:11Z\",\n",
+    "    \"spec_version\": \"2.1\",\n",
     "    \"species\": \"shark\",\n",
     "    \"animal_class\": \"fish\"\n",
     "}\"\"\"\n",
@@ -811,7 +816,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": 13,
    "metadata": {},
    "outputs": [
     {
@@ -841,12 +846,12 @@
    "source": [
     "### Custom Cyber Observable Types\n",
     "\n",
-    "Similar to custom STIX object types, use a decorator to create [custom Cyber Observable](../api/v20/stix2.v20.observables.rst#stix2.v20.observables.CustomObservable) types. Just as before, ``__init__()`` can hold additional validation, but it is not necessary."
+    "Similar to custom STIX object types, use a decorator to create [custom Cyber Observable](../api/v21/stix2.v21.observables.rst#stix2.v21.observables.CustomObservable) types. Just as before, ``__init__()`` can hold additional validation, but it is not necessary."
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 13,
+   "execution_count": 14,
    "metadata": {},
    "outputs": [
     {
@@ -922,6 +927,7 @@
        ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n",
        ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "x-new-observable",\n",
+       "    "id": "x-new-observable--fdb5fd26-533e-44f4-9463-e8ade73e08c0",\n",
        "    "a_property": "something",\n",
        "    "property_2": 10\n",
        "}\n",
@@ -931,7 +937,7 @@
        ""
       ]
      },
-     "execution_count": 13,
+     "execution_count": 14,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -962,7 +968,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 14,
+   "execution_count": 16,
    "metadata": {},
    "outputs": [
     {
@@ -1043,7 +1049,7 @@
        ""
       ]
      },
-     "execution_count": 14,
+     "execution_count": 16,
      "metadata": {},
      "output_type": "execute_result"
     },
@@ -1125,7 +1131,7 @@
        ""
       ]
      },
-     "execution_count": 14,
+     "execution_count": 16,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -1136,6 +1142,7 @@
     "input_string4 = \"\"\"{\n",
     "    \"type\": \"observed-data\",\n",
     "    \"id\": \"observed-data--b67d30ff-02ac-498a-92f9-32f845f448cf\",\n",
+    "    \"spec_version\": \"2.1\",\n",
     "    \"created_by_ref\": \"identity--f431f809-377b-45e0-aa1c-6a4751cae5ff\",\n",
     "    \"created\": \"2016-04-06T19:58:16.000Z\",\n",
     "    \"modified\": \"2016-04-06T19:58:16.000Z\",\n",
@@ -1171,7 +1178,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 15,
+   "execution_count": 17,
    "metadata": {},
    "outputs": [
     {
@@ -1247,7 +1254,7 @@
        ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n",
        ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "x-new-observable-2",\n",
-       "    "id": "x-new-observable-2--6bc655d6-dcb8-52a3-a862-46848c17e599",\n",
+       "    "id": "x-new-observable-2--cafee477-4edc-58fd-81c1-2e23e93f9326",\n",
        "    "a_property": "A property",\n",
        "    "property_2": 2000\n",
        "}\n",
@@ -1257,7 +1264,7 @@
        ""
       ]
      },
-     "execution_count": 15,
+     "execution_count": 17,
      "metadata": {},
      "output_type": "execute_result"
     },
@@ -1334,7 +1341,7 @@
        ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n",
        ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "x-new-observable-2",\n",
-       "    "id": "x-new-observable-2--6bc655d6-dcb8-52a3-a862-46848c17e599",\n",
+       "    "id": "x-new-observable-2--cafee477-4edc-58fd-81c1-2e23e93f9326",\n",
        "    "a_property": "A property",\n",
        "    "property_2": 3000\n",
        "}\n",
@@ -1344,7 +1351,7 @@
        ""
       ]
      },
-     "execution_count": 15,
+     "execution_count": 17,
      "metadata": {},
      "output_type": "execute_result"
     },
@@ -1421,7 +1428,7 @@
        ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n",
        ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "x-new-observable-2",\n",
-       "    "id": "x-new-observable-2--1e56f9c3-a73b-5fbd-b348-83c76523c4df",\n",
+       "    "id": "x-new-observable-2--2945b948-7361-5204-a630-31b828af920c",\n",
        "    "a_property": "A different property",\n",
        "    "property_2": 3000\n",
        "}\n",
@@ -1431,13 +1438,13 @@
        ""
       ]
      },
-     "execution_count": 15,
+     "execution_count": 17,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
-    "from stix2.v21 import CustomObservable # IDs and Deterministic IDs are NOT part of STIX 2.0 Custom Observables\n",
+    "from stix2 import CustomObservable\n",
     "\n",
     "@CustomObservable('x-new-observable-2', [\n",
     "    ('a_property', properties.StringProperty(required=True)),\n",
@@ -1473,12 +1480,12 @@
    "source": [
     "### Custom Cyber Observable Extensions\n",
     "\n",
-    "Finally, custom extensions to existing Cyber Observable types can also be created. Just use the @[CustomExtension](../api/v20/stix2.v20.observables.rst#stix2.v20.observables.CustomExtension) decorator. Note that you must provide the Cyber Observable class to which the extension applies. Again, any extra validation of the properties can be implemented by providing an ``__init__()`` but it is not required. Let's say we want to make an extension to the ``File`` Cyber Observable Object:"
+    "Finally, custom extensions to existing Cyber Observable types can also be created. Just use the @[CustomExtension](../api/v21/stix2.v21.observables.rst#stix2.v21.observables.CustomExtension) decorator. Note that you must provide the Cyber Observable class to which the extension applies. Again, any extra validation of the properties can be implemented by providing an ``__init__()`` but it is not required. Let's say we want to make an extension to the ``File`` Cyber Observable Object:"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 16,
+   "execution_count": 18,
    "metadata": {},
    "outputs": [
     {
@@ -1562,7 +1569,7 @@
        ""
       ]
      },
-     "execution_count": 16,
+     "execution_count": 18,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -1591,7 +1598,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 17,
+   "execution_count": 20,
    "metadata": {},
    "outputs": [
     {
@@ -1672,7 +1679,7 @@
        ""
       ]
      },
-     "execution_count": 17,
+     "execution_count": 20,
      "metadata": {},
      "output_type": "execute_result"
     },
@@ -1754,7 +1761,7 @@
        ""
       ]
      },
-     "execution_count": 17,
+     "execution_count": 20,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -1763,6 +1770,7 @@
     "input_string5 = \"\"\"{\n",
     "    \"type\": \"observed-data\",\n",
     "    \"id\": \"observed-data--b67d30ff-02ac-498a-92f9-32f845f448cf\",\n",
+    "    \"spec_version\": \"2.1\",\n",
     "    \"created_by_ref\": \"identity--f431f809-377b-45e0-aa1c-6a4751cae5ff\",\n",
     "    \"created\": \"2016-04-06T19:58:16.000Z\",\n",
     "    \"modified\": \"2016-04-06T19:58:16.000Z\",\n",
@@ -1793,21 +1801,21 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 2",
+   "display_name": "Python 3",
    "language": "python",
-   "name": "python2"
+   "name": "python3"
   },
   "language_info": {
    "codemirror_mode": {
     "name": "ipython",
-    "version": 2
+    "version": 3
    },
    "file_extension": ".py",
    "mimetype": "text/x-python",
    "name": "python",
    "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython2",
-   "version": "2.7.15+"
+   "pygments_lexer": "ipython3",
+   "version": "3.9.0a6"
   }
  },
  "nbformat": 4,
diff --git a/docs/guide/datastore.ipynb b/docs/guide/datastore.ipynb
index e4aad79..9f8e310 100644
--- a/docs/guide/datastore.ipynb
+++ b/docs/guide/datastore.ipynb
@@ -4,7 +4,6 @@
    "cell_type": "code",
    "execution_count": 1,
    "metadata": {
-    "collapsed": true,
     "nbsphinx": "hidden"
    },
    "outputs": [],
@@ -16,6 +15,7 @@
     "def hide_traceback(exc_tuple=None, filename=None, tb_offset=None,\n",
     "                   exception_only=False, running_compiled_code=False):\n",
     "    etype, value, tb = sys.exc_info()\n",
+    "    value.__cause__ = None  # suppress chained exceptions\n",
     "    return ipython._showtraceback(etype, value, ipython.InteractiveTB.get_exception_only(etype, value))\n",
     "\n",
     "ipython.showtraceback = hide_traceback"
@@ -25,7 +25,6 @@
    "cell_type": "code",
    "execution_count": 2,
    "metadata": {
-    "collapsed": true,
     "nbsphinx": "hidden"
    },
    "outputs": [],
@@ -87,7 +86,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 9,
    "metadata": {},
    "outputs": [
     {
@@ -163,6 +162,7 @@
        ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n",
        ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "intrusion-set",\n",
+       "    "spec_version": "2.1",\n",
        "    "id": "intrusion-set--f3bdec95-3d62-42d9-a840-29630f6cdc1a",\n",
        "    "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5",\n",
        "    "created": "2017-05-31T21:31:53.197Z",\n",
@@ -204,7 +204,7 @@
        ""
       ]
      },
-     "execution_count": 4,
+     "execution_count": 9,
      "metadata": {},
      "output_type": "execute_result"
     },
@@ -281,19 +281,22 @@
        ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n",
        ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "indicator",\n",
-       "    "id": "indicator--02b90f02-a96a-43ee-88f1-1e87297941f2",\n",
+       "    "spec_version": "2.1",\n",
+       "    "id": "indicator--a740531e-63ff-4e49-a9e1-a0a3eed0e3e7",\n",
        "    "created": "2017-11-13T07:00:24.000Z",\n",
        "    "modified": "2017-11-13T07:00:24.000Z",\n",
        "    "name": "Ransomware IP Blocklist",\n",
        "    "description": "IP Blocklist address from abuse.ch",\n",
-       "    "pattern": "[ ipv4-addr:value = '91.237.247.24' ]",\n",
-       "    "valid_from": "2017-11-13T07:00:24Z",\n",
-       "    "labels": [\n",
+       "    "indicator_types": [\n",
        "        "malicious-activity",\n",
        "        "Ransomware",\n",
        "        "Botnet",\n",
        "        "C&C"\n",
        "    ],\n",
+       "    "pattern": "[ ipv4-addr:value = '91.237.247.24' ]",\n",
+       "    "pattern_type": "stix",\n",
+       "    "pattern_version": "2.1",\n",
+       "    "valid_from": "2017-11-13T07:00:24Z",\n",
        "    "external_references": [\n",
        "        {\n",
        "            "source_name": "abuse.ch",\n",
@@ -307,7 +310,7 @@
        ""
       ]
      },
-     "execution_count": 4,
+     "execution_count": 9,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -320,7 +323,7 @@
     "fs = FileSystemSource(\"/tmp/stix2_source\")\n",
     "\n",
     "# create TAXIICollectionSource\n",
-    "colxn = Collection('http://127.0.0.1:5000/trustgroup1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/')\n",
+    "colxn = Collection('http://127.0.0.1:5000/trustgroup1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/', user=\"user1\", password=\"Password1\")\n",
     "ts = TAXIICollectionSource(colxn)\n",
     "\n",
     "# add them both to the CompositeDataSource\n",
@@ -332,7 +335,7 @@
     "print(intrusion_set)\n",
     "\n",
     "# get an object that is only in the TAXII collection\n",
-    "ind = cs.get('indicator--02b90f02-a96a-43ee-88f1-1e87297941f2')\n",
+    "ind = cs.get('indicator--a740531e-63ff-4e49-a9e1-a0a3eed0e3e7')\n",
     "print(ind)"
    ]
   },
@@ -357,6 +360,7 @@
     "\n",
     "* added_after\n",
     "* id\n",
+    "* spec_version\n",
     "* type\n",
     "* version\n",
     "\n",
@@ -386,10 +390,8 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
-   "metadata": {
-    "collapsed": true
-   },
+   "execution_count": 10,
+   "metadata": {},
    "outputs": [],
    "source": [
     "import sys\n",
@@ -420,7 +422,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": 11,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -454,7 +456,8 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "**Note: The `defanged` property is now always included (implicitly) for STIX 2.1 Cyber Observable Objects (SCOs)**\n\n",
+    "**Note: The `defanged` property is now always included (implicitly) for STIX 2.1 Cyber Observable Objects (SCOs)**\n",
+    "\n",
     "This is important to remember if you are writing a filter that involves checking the `objects` property of a STIX 2.1 `ObservedData` object. If any of the objects associated with the `objects` property are STIX 2.1 SCOs, then your filter must include the `defanged` property. For an example, refer to `filters[14]` & `filters[15]` in stix2/test/v21/test_datastore_filters.py "
    ]
   },
@@ -469,10 +472,8 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
-   "metadata": {
-    "collapsed": true
-   },
+   "execution_count": 14,
+   "metadata": {},
    "outputs": [],
    "source": [
     "from stix2 import Campaign, Identity, Indicator, Malware, Relationship\n",
@@ -480,8 +481,8 @@
     "mem = MemoryStore()\n",
     "cam = Campaign(name='Charge', description='Attack!')\n",
     "idy = Identity(name='John Doe', identity_class=\"individual\")\n",
-    "ind = Indicator(labels=['malicious-activity'], pattern=\"[file:hashes.MD5 = 'd41d8cd98f00b204e9800998ecf8427e']\")\n",
-    "mal = Malware(labels=['ransomware'], name=\"Cryptolocker\", created_by_ref=idy)\n",
+    "ind = Indicator(pattern_type='stix', pattern=\"[file:hashes.MD5 = 'd41d8cd98f00b204e9800998ecf8427e']\")\n",
+    "mal = Malware(name=\"Cryptolocker\", is_family=False, created_by_ref=idy)\n",
     "rel1 = Relationship(ind, 'indicates', mal,)\n",
     "rel2 = Relationship(mal, 'targets', idy)\n",
     "rel3 = Relationship(cam, 'uses', mal)\n",
@@ -492,12 +493,12 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "If a STIX object has a `created_by_ref` property, you can use the [creator_of()](../api/stix2.datastore.rst#stix2.datastore.DataSource.creator_of) method to retrieve the [Identity](../api/v20/stix2.v20.sdo.rst#stix2.v20.sdo.Identity) object that created it."
+    "If a STIX object has a `created_by_ref` property, you can use the [creator_of()](../api/stix2.datastore.rst#stix2.datastore.DataSource.creator_of) method to retrieve the [Identity](../api/v21/stix2.v21.sdo.rst#stix2.v21.sdo.Identity) object that created it."
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 11,
+   "execution_count": 15,
    "metadata": {},
    "outputs": [
     {
@@ -573,9 +574,10 @@
        ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n",
        ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "identity",\n",
-       "    "id": "identity--b67cf8d4-cc1a-4bb7-9402-fffcff17c9a9",\n",
-       "    "created": "2018-04-05T20:43:54.117Z",\n",
-       "    "modified": "2018-04-05T20:43:54.117Z",\n",
+       "    "spec_version": "2.1",\n",
+       "    "id": "identity--a2628104-e357-44a0-b16f-d5f36c0fd0ec",\n",
+       "    "created": "2020-06-26T13:59:21.924055Z",\n",
+       "    "modified": "2020-06-26T13:59:21.924055Z",\n",
        "    "name": "John Doe",\n",
        "    "identity_class": "individual"\n",
        "}\n",
@@ -585,7 +587,7 @@
        ""
       ]
      },
-     "execution_count": 11,
+     "execution_count": 15,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -603,7 +605,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": 16,
    "metadata": {},
    "outputs": [
     {
@@ -612,7 +614,7 @@
        "3"
       ]
      },
-     "execution_count": 12,
+     "execution_count": 16,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -631,16 +633,16 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 13,
+   "execution_count": 17,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "[Relationship(type='relationship', id='relationship--3b9cb248-5c2c-425d-85d0-680bfef6e69d', created='2018-04-05T20:43:54.134Z', modified='2018-04-05T20:43:54.134Z', relationship_type='indicates', source_ref='indicator--61deb2a5-305a-490e-83b3-9839a9677368', target_ref='malware--9fe343d8-edf7-4f4a-bb6c-a221fb75142d')]"
+       "[Relationship(type='relationship', spec_version='2.1', id='relationship--ef837187-773c-41e4-ae86-c66189a832f5', created='2020-06-26T13:59:21.929336Z', modified='2020-06-26T13:59:21.929336Z', relationship_type='indicates', source_ref='indicator--9f10f6f2-b93d-488e-be35-72c3ec1087c3', target_ref='malware--315597db-2a74-4a29-8e54-38572e1ac07b')]"
       ]
      },
-     "execution_count": 13,
+     "execution_count": 17,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -658,16 +660,16 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 14,
+   "execution_count": 18,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "[Relationship(type='relationship', id='relationship--8d322508-423b-4d51-be85-a95ad083f8af', created='2018-04-05T20:43:54.134Z', modified='2018-04-05T20:43:54.134Z', relationship_type='targets', source_ref='malware--9fe343d8-edf7-4f4a-bb6c-a221fb75142d', target_ref='identity--b67cf8d4-cc1a-4bb7-9402-fffcff17c9a9')]"
+       "[Relationship(type='relationship', spec_version='2.1', id='relationship--43f5f7a7-8a99-4bbf-8d93-e6f3fd2951a3', created='2020-06-26T13:59:21.937132Z', modified='2020-06-26T13:59:21.937132Z', relationship_type='targets', source_ref='malware--315597db-2a74-4a29-8e54-38572e1ac07b', target_ref='identity--a2628104-e357-44a0-b16f-d5f36c0fd0ec')]"
       ]
      },
-     "execution_count": 14,
+     "execution_count": 18,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -685,17 +687,17 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 15,
+   "execution_count": 19,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "[Relationship(type='relationship', id='relationship--3b9cb248-5c2c-425d-85d0-680bfef6e69d', created='2018-04-05T20:43:54.134Z', modified='2018-04-05T20:43:54.134Z', relationship_type='indicates', source_ref='indicator--61deb2a5-305a-490e-83b3-9839a9677368', target_ref='malware--9fe343d8-edf7-4f4a-bb6c-a221fb75142d'),\n",
-       " Relationship(type='relationship', id='relationship--93e5afe0-d1fb-4315-8d08-10951f7a99b6', created='2018-04-05T20:43:54.134Z', modified='2018-04-05T20:43:54.134Z', relationship_type='uses', source_ref='campaign--edfd885c-bc31-4051-9bc2-08e057542d56', target_ref='malware--9fe343d8-edf7-4f4a-bb6c-a221fb75142d')]"
+       "[Relationship(type='relationship', spec_version='2.1', id='relationship--ef837187-773c-41e4-ae86-c66189a832f5', created='2020-06-26T13:59:21.929336Z', modified='2020-06-26T13:59:21.929336Z', relationship_type='indicates', source_ref='indicator--9f10f6f2-b93d-488e-be35-72c3ec1087c3', target_ref='malware--315597db-2a74-4a29-8e54-38572e1ac07b'),\n",
+       " Relationship(type='relationship', spec_version='2.1', id='relationship--596c196f-2f05-4584-b643-2186b327a94f', created='2020-06-26T13:59:21.937354Z', modified='2020-06-26T13:59:21.937354Z', relationship_type='uses', source_ref='campaign--d359f872-7e44-4090-8e08-c5bd10bc5f2d', target_ref='malware--315597db-2a74-4a29-8e54-38572e1ac07b')]"
       ]
      },
-     "execution_count": 15,
+     "execution_count": 19,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -713,16 +715,16 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 16,
+   "execution_count": 20,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "[Campaign(type='campaign', id='campaign--edfd885c-bc31-4051-9bc2-08e057542d56', created='2018-04-05T20:43:54.117Z', modified='2018-04-05T20:43:54.117Z', name='Charge', description='Attack!')]"
+       "[Campaign(type='campaign', spec_version='2.1', id='campaign--d359f872-7e44-4090-8e08-c5bd10bc5f2d', created='2020-06-26T13:59:21.923792Z', modified='2020-06-26T13:59:21.923792Z', name='Charge', description='Attack!')]"
       ]
      },
-     "execution_count": 16,
+     "execution_count": 20,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -748,7 +750,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.6.7"
+   "version": "3.9.0a6"
   }
  },
  "nbformat": 4,
diff --git a/docs/guide/environment.ipynb b/docs/guide/environment.ipynb
index 8d22e39..a3417b5 100644
--- a/docs/guide/environment.ipynb
+++ b/docs/guide/environment.ipynb
@@ -15,6 +15,7 @@
     "def hide_traceback(exc_tuple=None, filename=None, tb_offset=None,\n",
     "                   exception_only=False, running_compiled_code=False):\n",
     "    etype, value, tb = sys.exc_info()\n",
+    "    value.__cause__ = None  # suppress chained exceptions\n",
     "    return ipython._showtraceback(etype, value, ipython.InteractiveTB.get_exception_only(etype, value))\n",
     "\n",
     "ipython.showtraceback = hide_traceback"
@@ -67,7 +68,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": 3,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -113,7 +114,7 @@
     "from stix2 import Indicator\n",
     "\n",
     "indicator = Indicator(id=\"indicator--a740531e-63ff-4e49-a9e1-a0a3eed0e3e7\",\n",
-    "                      labels=[\"malicious-activity\"],\n",
+    "                      pattern_type=\"stix\",\n",
     "                      pattern=\"[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']\")\n",
     "env.add(indicator)"
    ]
@@ -203,14 +204,14 @@
        ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n",
        ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "indicator",\n",
+       "    "spec_version": "2.1",\n",
        "    "id": "indicator--a740531e-63ff-4e49-a9e1-a0a3eed0e3e7",\n",
-       "    "created": "2018-04-05T19:27:53.923Z",\n",
-       "    "modified": "2018-04-05T19:27:53.923Z",\n",
+       "    "created": "2020-06-26T14:46:08.384618Z",\n",
+       "    "modified": "2020-06-26T14:46:08.384618Z",\n",
        "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
-       "    "valid_from": "2018-04-05T19:27:53.923548Z",\n",
-       "    "labels": [\n",
-       "        "malicious-activity"\n",
-       "    ]\n",
+       "    "pattern_type": "stix",\n",
+       "    "pattern_version": "2.1",\n",
+       "    "valid_from": "2020-06-26T14:46:08.384618Z"\n",
        "}\n",
        "
\n" ], @@ -238,7 +239,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ @@ -258,7 +259,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 10, "metadata": {}, "outputs": [ { @@ -334,15 +335,15 @@ ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n", ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "indicator",\n",
-       "    "id": "indicator--c1b421c0-9c6b-4276-9b73-1b8684a5a0d2",\n",
+       "    "spec_version": "2.1",\n",
+       "    "id": "indicator--4db1493b-8822-4b1c-a471-1c1cdc53ec6d",\n",
        "    "created_by_ref": "identity--311b2d2d-f010-4473-83ec-1edf84858f4c",\n",
-       "    "created": "2018-04-05T19:28:48.776Z",\n",
-       "    "modified": "2018-04-05T19:28:48.776Z",\n",
+       "    "created": "2020-06-26T14:46:36.666866Z",\n",
+       "    "modified": "2020-06-26T14:46:36.666866Z",\n",
        "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
-       "    "valid_from": "2018-04-05T19:28:48.776442Z",\n",
-       "    "labels": [\n",
-       "        "malicious-activity"\n",
-       "    ]\n",
+       "    "pattern_type": "stix",\n",
+       "    "pattern_version": "2.1",\n",
+       "    "valid_from": "2020-06-26T14:46:36.666866Z"\n",
        "}\n",
        "
\n" ], @@ -350,14 +351,14 @@ "" ] }, - "execution_count": 14, + "execution_count": 10, "metadata": {}, "output_type": "execute_result" } ], "source": [ "ind = factory.create(Indicator,\n", - " labels=[\"malicious-activity\"],\n", + " pattern_type=\"stix\",\n", " pattern=\"[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']\")\n", "print(ind)" ] @@ -380,7 +381,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 11, "metadata": {}, "outputs": [ { @@ -456,14 +457,14 @@ ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n", ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "indicator",\n",
-       "    "id": "indicator--30a3b39c-5f57-4e7f-9eaf-e1abcb643da4",\n",
-       "    "created": "2017-09-25T18:07:46.255Z",\n",
-       "    "modified": "2017-09-25T18:07:46.255Z",\n",
+       "    "spec_version": "2.1",\n",
+       "    "id": "indicator--e7e92c87-df40-4ffb-a6da-9667b0acddb1",\n",
+       "    "created": "2017-09-25T18:07:46.255472Z",\n",
+       "    "modified": "2017-09-25T18:07:46.255472Z",\n",
        "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
-       "    "valid_from": "2018-04-05T19:28:53.268567Z",\n",
-       "    "labels": [\n",
-       "        "malicious-activity"\n",
-       "    ]\n",
+       "    "pattern_type": "stix",\n",
+       "    "pattern_version": "2.1",\n",
+       "    "valid_from": "2020-06-26T14:47:58.470047Z"\n",
        "}\n",
        "
\n" ], @@ -471,7 +472,7 @@ "" ] }, - "execution_count": 15, + "execution_count": 11, "metadata": {}, "output_type": "execute_result" } @@ -483,14 +484,14 @@ "\n", "ind2 = env2.create(Indicator,\n", " created_by_ref=None,\n", - " labels=[\"malicious-activity\"],\n", + " pattern_type=\"stix\",\n", " pattern=\"[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']\")\n", "print(ind2)" ] }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 12, "metadata": {}, "outputs": [ { @@ -566,15 +567,15 @@ ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n", ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "indicator",\n",
-       "    "id": "indicator--6c5bbaaf-6dac-44b0-a0df-86c27b3f6ecb",\n",
+       "    "spec_version": "2.1",\n",
+       "    "id": "indicator--40540b9b-47a7-4855-81a3-b6d3ff6f8592",\n",
        "    "created_by_ref": "identity--962cabe5-f7f3-438a-9169-585a8c971d12",\n",
-       "    "created": "2017-09-25T18:07:46.255Z",\n",
-       "    "modified": "2017-09-25T18:07:46.255Z",\n",
+       "    "created": "2017-09-25T18:07:46.255472Z",\n",
+       "    "modified": "2017-09-25T18:07:46.255472Z",\n",
        "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
-       "    "valid_from": "2018-04-05T19:29:56.55129Z",\n",
-       "    "labels": [\n",
-       "        "malicious-activity"\n",
-       "    ]\n",
+       "    "pattern_type": "stix",\n",
+       "    "pattern_version": "2.1",\n",
+       "    "valid_from": "2020-06-26T14:48:11.028904Z"\n",
        "}\n",
        "
\n" ], @@ -582,7 +583,7 @@ "" ] }, - "execution_count": 16, + "execution_count": 12, "metadata": {}, "output_type": "execute_result" } @@ -590,7 +591,7 @@ "source": [ "ind3 = env2.create(Indicator,\n", " created_by_ref=\"identity--962cabe5-f7f3-438a-9169-585a8c971d12\",\n", - " labels=[\"malicious-activity\"],\n", + " pattern_type=\"stix\",\n", " pattern=\"[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']\")\n", "print(ind3)" ] @@ -606,7 +607,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 13, "metadata": {}, "outputs": [ { @@ -682,15 +683,15 @@ ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n", ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "indicator",\n",
-       "    "id": "indicator--d1b8c3f6-1de1-44c1-b079-3df307224a0d",\n",
+       "    "spec_version": "2.1",\n",
+       "    "id": "indicator--3ab656d1-e549-4a6e-a2df-e84ff515fcd3",\n",
        "    "created_by_ref": "identity--311b2d2d-f010-4473-83ec-1edf84858f4c",\n",
-       "    "created": "2018-04-05T19:29:59.605Z",\n",
-       "    "modified": "2018-04-05T19:29:59.605Z",\n",
+       "    "created": "2020-06-26T14:48:20.238719Z",\n",
+       "    "modified": "2020-06-26T14:48:20.238719Z",\n",
        "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
-       "    "valid_from": "2018-04-05T19:29:59.605463Z",\n",
-       "    "labels": [\n",
-       "        "malicious-activity"\n",
-       "    ]\n",
+       "    "pattern_type": "stix",\n",
+       "    "pattern_version": "2.1",\n",
+       "    "valid_from": "2020-06-26T14:48:20.238719Z"\n",
        "}\n",
        "
\n" ], @@ -698,7 +699,7 @@ "" ] }, - "execution_count": 17, + "execution_count": 13, "metadata": {}, "output_type": "execute_result" } @@ -708,7 +709,7 @@ " MemoryStore())\n", "\n", "i = environ.create(Indicator,\n", - " labels=[\"malicious-activity\"],\n", + " pattern_type=\"stix\",\n", " pattern=\"[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']\")\n", "environ.add(i)\n", "print(environ.get(i.id))" @@ -731,7 +732,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.3" + "version": "3.9.0a6" } }, "nbformat": 4, diff --git a/docs/guide/equivalence.ipynb b/docs/guide/equivalence.ipynb index 05d0d99..c0b9331 100644 --- a/docs/guide/equivalence.ipynb +++ b/docs/guide/equivalence.ipynb @@ -15,6 +15,7 @@ "def hide_traceback(exc_tuple=None, filename=None, tb_offset=None,\n", " exception_only=False, running_compiled_code=False):\n", " etype, value, tb = sys.exc_info()\n", + " value.__cause__ = None # suppress chained exceptions\n", " return ipython._showtraceback(etype, value, ipython.InteractiveTB.get_exception_only(etype, value))\n", "\n", "ipython.showtraceback = hide_traceback" @@ -167,8 +168,7 @@ ], "source": [ "import stix2\n", - "from stix2 import Environment, MemoryStore\n", - "from stix2.v21 import AttackPattern\n", + "from stix2 import AttackPattern, Environment, MemoryStore\n", "\n", "env = Environment(store=MemoryStore())\n", "\n", @@ -291,7 +291,7 @@ } ], "source": [ - "from stix2.v21 import Campaign\n", + "from stix2 import Campaign\n", "\n", "c1 = Campaign(\n", " name=\"Someone Attacks Somebody\",)\n", @@ -399,7 +399,7 @@ } ], "source": [ - "from stix2.v21 import Identity\n", + "from stix2 import Identity\n", "\n", "id1 = Identity(\n", " name=\"John Smith\",\n", @@ -645,7 +645,7 @@ } ], "source": [ - "from stix2.v21 import Location\n", + "from stix2 import Location\n", "\n", "loc1 = Location(\n", " latitude=38.889,\n", @@ -758,7 +758,7 @@ } ], "source": [ - "from stix2.v21 import Malware\n", + "from stix2 import Malware\n", "\n", "MALWARE_ID = \"malware--9c4638ec-f1de-4ddb-abf4-1b760417654e\"\n", "\n", @@ -875,7 +875,7 @@ } ], "source": [ - "from stix2.v21 import ThreatActor\n", + "from stix2 import ThreatActor\n", "\n", "THREAT_ACTOR_ID = \"threat-actor--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f\"\n", "\n", @@ -992,7 +992,7 @@ } ], "source": [ - "from stix2.v21 import Tool\n", + "from stix2 import Tool\n", "\n", "t1 = Tool(\n", " tool_types=[\"remote-access\"],\n", @@ -1104,7 +1104,7 @@ } ], "source": [ - "from stix2.v21 import Vulnerability\n", + "from stix2 import Vulnerability\n", "\n", "vuln1 = Vulnerability(\n", " name=\"Heartbleed\",\n", @@ -1251,7 +1251,7 @@ } ], "source": [ - "from stix2.v21 import Report\n", + "from stix2 import Report\n", "\n", "r1 = Report(\n", " report_types=[\"campaign\"],\n", @@ -1422,7 +1422,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "Starting semantic equivalence process between: 'threat-actor--664624c7-394e-49ad-ae2a-12f7a48a54a3' and 'threat-actor--1d67719e-6be6-4194-9226-1685986514f5'\n", + "Starting semantic equivalence process between: 'threat-actor--01538fad-697d-498f-b387-a49cc35de773' and 'threat-actor--dabd290b-6827-4c7f-8dc8-079b6fc83a76'\n", "--\t\tpartial_string_based 'Evil Org' 'James Bond'\tresult: '11'\n", "'name' check -- weight: 60, contributing score: 6.6\n", "--\t\tpartial_list_based '['crime-syndicate']' '['spy']'\tresult: '0.0'\n", @@ -1549,7 +1549,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 17, "metadata": {}, "outputs": [ { @@ -1630,7 +1630,7 @@ "" ] }, - "execution_count": 18, + "execution_count": 17, "metadata": {}, "output_type": "execute_result" }, @@ -1712,7 +1712,7 @@ "" ] }, - "execution_count": 18, + "execution_count": 17, "metadata": {}, "output_type": "execute_result" }, @@ -1794,7 +1794,7 @@ "" ] }, - "execution_count": 18, + "execution_count": 17, "metadata": {}, "output_type": "execute_result" }, @@ -1876,7 +1876,7 @@ "" ] }, - "execution_count": 18, + "execution_count": 17, "metadata": {}, "output_type": "execute_result" }, @@ -1958,7 +1958,7 @@ "" ] }, - "execution_count": 18, + "execution_count": 17, "metadata": {}, "output_type": "execute_result" }, @@ -2040,7 +2040,7 @@ "" ] }, - "execution_count": 18, + "execution_count": 17, "metadata": {}, "output_type": "execute_result" }, @@ -2122,7 +2122,7 @@ "" ] }, - "execution_count": 18, + "execution_count": 17, "metadata": {}, "output_type": "execute_result" } @@ -2180,7 +2180,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 18, "metadata": {}, "outputs": [ { @@ -2261,7 +2261,7 @@ "" ] }, - "execution_count": 19, + "execution_count": 18, "metadata": {}, "output_type": "execute_result" }, @@ -2343,7 +2343,7 @@ "" ] }, - "execution_count": 19, + "execution_count": 18, "metadata": {}, "output_type": "execute_result" } @@ -2373,7 +2373,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 19, "metadata": {}, "outputs": [ { @@ -2382,7 +2382,7 @@ "9.95" ] }, - "execution_count": 20, + "execution_count": 19, "metadata": {}, "output_type": "execute_result" }, @@ -2464,7 +2464,7 @@ "" ] }, - "execution_count": 20, + "execution_count": 19, "metadata": {}, "output_type": "execute_result" } @@ -2492,7 +2492,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 20, "metadata": {}, "outputs": [ { @@ -2573,7 +2573,7 @@ "" ] }, - "execution_count": 21, + "execution_count": 20, "metadata": {}, "output_type": "execute_result" } @@ -2610,7 +2610,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 21, "metadata": {}, "outputs": [ { @@ -2691,7 +2691,7 @@ "" ] }, - "execution_count": 22, + "execution_count": 21, "metadata": {}, "output_type": "execute_result" }, @@ -2773,7 +2773,7 @@ "" ] }, - "execution_count": 22, + "execution_count": 21, "metadata": {}, "output_type": "execute_result" } @@ -2817,7 +2817,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 22, "metadata": {}, "outputs": [ { @@ -2898,7 +2898,7 @@ "" ] }, - "execution_count": 23, + "execution_count": 22, "metadata": {}, "output_type": "execute_result" }, @@ -2980,7 +2980,7 @@ "" ] }, - "execution_count": 23, + "execution_count": 22, "metadata": {}, "output_type": "execute_result" } @@ -3050,7 +3050,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.1" + "version": "3.9.0a6" } }, "nbformat": 4, diff --git a/docs/guide/filesystem.ipynb b/docs/guide/filesystem.ipynb index b3aca88..845ded4 100644 --- a/docs/guide/filesystem.ipynb +++ b/docs/guide/filesystem.ipynb @@ -15,6 +15,7 @@ "def hide_traceback(exc_tuple=None, filename=None, tb_offset=None,\n", " exception_only=False, running_compiled_code=False):\n", " etype, value, tb = sys.exc_info()\n", + " value.__cause__ = None # suppress chained exceptions\n", " return ipython._showtraceback(etype, value, ipython.InteractiveTB.get_exception_only(etype, value))\n", "\n", "ipython.showtraceback = hide_traceback" @@ -126,7 +127,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 7, "metadata": {}, "outputs": [ { @@ -202,25 +203,27 @@ ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n", ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "malware",\n",
-       "    "id": "malware--00c3bfcb-99bd-4767-8c03-b08f585f5c8a",\n",
+       "    "spec_version": "2.1",\n",
+       "    "id": "malware--92ec0cbd-2c30-44a2-b270-73f4ec949841",\n",
        "    "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5",\n",
-       "    "created": "2017-05-31T21:33:19.746Z",\n",
-       "    "modified": "2017-05-31T21:33:19.746Z",\n",
-       "    "name": "PowerDuke",\n",
-       "    "description": "PowerDuke is a backdoor that was used by APT29 in 2016. It has primarily been delivered through Microsoft Word or Excel attachments containing malicious macros.[[Citation: Volexity PowerDuke November 2016]]",\n",
-       "    "labels": [\n",
+       "    "created": "2017-05-31T21:33:26.565Z",\n",
+       "    "modified": "2017-05-31T21:33:26.565Z",\n",
+       "    "name": "RTM",\n",
+       "    "description": "RTM is custom malware written in Delphi. It is used by the group of the same name (RTM).[[Citation: ESET RTM Feb 2017]]",\n",
+       "    "malware_types": [\n",
        "        "malware"\n",
        "    ],\n",
+       "    "is_family": false,\n",
        "    "external_references": [\n",
        "        {\n",
        "            "source_name": "mitre-attack",\n",
-       "            "url": "https://attack.mitre.org/wiki/Software/S0139",\n",
-       "            "external_id": "S0139"\n",
+       "            "url": "https://attack.mitre.org/wiki/Software/S0148",\n",
+       "            "external_id": "S0148"\n",
        "        },\n",
        "        {\n",
-       "            "source_name": "Volexity PowerDuke November 2016",\n",
-       "            "description": "Adair, S.. (2016, November 9). PowerDuke: Widespread Post-Election Spear Phishing Campaigns Targeting Think Tanks and NGOs. Retrieved January 11, 2017.",\n",
-       "            "url": "https://www.volexity.com/blog/2016/11/09/powerduke-post-election-spear-phishing-campaigns-targeting-think-tanks-and-ngos/"\n",
+       "            "source_name": "ESET RTM Feb 2017",\n",
+       "            "description": "Faou, M. and Boutin, J.. (2017, February). Read The Manual: A Guide to the RTM Banking Trojan. Retrieved March 9, 2017.",\n",
+       "            "url": "https://www.welivesecurity.com/wp-content/uploads/2017/02/Read-The-Manual.pdf"\n",
        "        }\n",
        "    ],\n",
        "    "object_marking_refs": [\n",
@@ -233,7 +236,7 @@
        ""
       ]
      },
-     "execution_count": 4,
+     "execution_count": 7,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -245,8 +248,8 @@
     "fs = FileSystemStore(\"/tmp/stix2_store\")\n",
     "\n",
     "# retrieve STIX2 content from FileSystemStore\n",
-    "ap = fs.get(\"attack-pattern--00d0b012-8a03-410e-95de-5826bf542de6\")\n",
-    "mal = fs.get(\"malware--00c3bfcb-99bd-4767-8c03-b08f585f5c8a\")\n",
+    "ap = fs.get(\"attack-pattern--0a3ead4e-6d47-4ccb-854c-a6a4f9d96b22\")\n",
+    "mal = fs.get(\"malware--92ec0cbd-2c30-44a2-b270-73f4ec949841\")\n",
     "\n",
     "# for visual purposes\n",
     "print(mal)"
@@ -254,32 +257,29 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
-   "metadata": {
-    "collapsed": true
-   },
+   "execution_count": 8,
+   "metadata": {},
    "outputs": [],
    "source": [
     "from stix2 import ThreatActor, Indicator\n",
     "\n",
     "# create new STIX threat-actor\n",
     "ta = ThreatActor(name=\"Adjective Bear\",\n",
-    "                labels=[\"nation-state\"],\n",
-    "                sophistication=\"innovator\",\n",
-    "                resource_level=\"government\",\n",
-    "                goals=[\n",
-    "                    \"compromising media outlets\",\n",
-    "                    \"water-hole attacks geared towards political, military targets\",\n",
-    "                    \"intelligence collection\"\n",
-    "                ])\n",
+    "                 sophistication=\"innovator\",\n",
+    "                 resource_level=\"government\",\n",
+    "                 goals=[\n",
+    "                     \"compromising media outlets\",\n",
+    "                     \"water-hole attacks geared towards political, military targets\",\n",
+    "                     \"intelligence collection\"\n",
+    "                 ])\n",
     "\n",
     "# create new indicators\n",
     "ind = Indicator(description=\"Crusades C2 implant\",\n",
-    "                labels=[\"malicious-activity\"],\n",
+    "                pattern_type=\"stix\",\n",
     "                pattern=\"[file:hashes.'SHA-256' = '54b7e05e39a59428743635242e4a867c932140a999f52a1e54fa7ee6a440c73b']\")\n",
     "\n",
     "ind1 = Indicator(description=\"Crusades C2 implant 2\",\n",
-    "                 labels=[\"malicious-activity\"],\n",
+    "                 pattern_type=\"stix\",\n",
     "                 pattern=\"[file:hashes.'SHA-256' = '64c7e05e40a59511743635242e4a867c932140a999f52a1e54fa7ee6a440c73b']\")\n",
     "\n",
     "# add STIX object (threat-actor) to FileSystemStore\n",
@@ -300,7 +300,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": 9,
    "metadata": {},
    "outputs": [
     {
@@ -376,23 +376,34 @@
        ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n",
        ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "attack-pattern",\n",
-       "    "id": "attack-pattern--00d0b012-8a03-410e-95de-5826bf542de6",\n",
+       "    "spec_version": "2.1",\n",
+       "    "id": "attack-pattern--0a3ead4e-6d47-4ccb-854c-a6a4f9d96b22",\n",
        "    "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5",\n",
-       "    "created": "2017-05-31T21:30:54.176Z",\n",
-       "    "modified": "2017-05-31T21:30:54.176Z",\n",
-       "    "name": "Indicator Removal from Tools",\n",
-       "    "description": "If a malicious...command-line parameters, Process monitoring",\n",
+       "    "created": "2017-05-31T21:30:19.735Z",\n",
+       "    "modified": "2017-05-31T21:30:19.735Z",\n",
+       "    "name": "Credential Dumping",\n",
+       "    "description": "Credential dumping is the process of obtaining account login and password information from the operating system and software. Credentials can be used to perform Windows Credential Editor, Mimikatz, and gsecdump. These tools are in use by both professional security testers and adversaries.\\n\\nPlaintext passwords can be obtained using tools such as Mimikatz to extract passwords stored by the Local Security Authority (LSA). If smart cards are used to authenticate to a domain using a personal identification number (PIN), then that PIN is also cached as a result and may be dumped.Mimikatz access the LSA Subsystem Service (LSASS) process by opening the process, locating the LSA secrets key, and decrypting the sections in memory where credential details are stored. Credential dumpers may also use methods for reflective DLL Injection to reduce potential indicators of malicious activity.\\n\\nNTLM hash dumpers open the Security Accounts Manager (SAM) on the local file system (%SystemRoot%/system32/config/SAM) or create a dump of the Registry SAM key to access stored account password hashes. Some hash dumpers will open the local file system as a device and parse to the SAM table to avoid file access defenses. Others will make an in-memory copy of the SAM table before reading hashes. Detection of compromised Legitimate Credentials in-use by adversaries may help as well. \\n\\nOn Windows 8.1 and Windows Server 2012 R2, monitor Windows Logs for LSASS.exe creation to verify that LSASS started as a protected process.\\n\\nMonitor processes and command-line arguments for program execution that may be indicative of credential dumping. Remote access tools may contain built-in features or incorporate existing tools like Mimikatz. PowerShell scripts also exist that contain credential dumping functionality, such as PowerSploit's Invoke-Mimikatz module,[[Citation: Powersploit]] which may require additional logging features to be configured in the operating system to collect necessary information for analysis.\\n\\nPlatforms: Windows Server 2003, Windows Server 2008, Windows Server 2012, Windows XP, Windows 7, Windows 8, Windows Server 2003 R2, Windows Server 2008 R2, Windows Server 2012 R2, Windows Vista, Windows 8.1\\n\\nData Sources: API monitoring, Process command-line parameters, Process monitoring, PowerShell logs",\n",
        "    "kill_chain_phases": [\n",
        "        {\n",
        "            "kill_chain_name": "mitre-attack",\n",
-       "            "phase_name": "defense-evasion"\n",
+       "            "phase_name": "credential-access"\n",
        "        }\n",
        "    ],\n",
        "    "external_references": [\n",
        "        {\n",
        "            "source_name": "mitre-attack",\n",
-       "            "url": "https://attack.mitre.org/wiki/Technique/T1066",\n",
-       "            "external_id": "T1066"\n",
+       "            "url": "https://attack.mitre.org/wiki/Technique/T1003",\n",
+       "            "external_id": "T1003"\n",
+       "        },\n",
+       "        {\n",
+       "            "source_name": "Github Mimikatz Module sekurlsa",\n",
+       "            "description": "Delpy, B. (2014, September 14). Mimikatz module ~ sekurlsa. Retrieved January 10, 2016.",\n",
+       "            "url": "https://github.com/gentilkiwi/mimikatz/wiki/module-~-sekurlsa"\n",
+       "        },\n",
+       "        {\n",
+       "            "source_name": "Powersploit",\n",
+       "            "description": "PowerSploit. (n.d.).  Retrieved December 4, 2014.",\n",
+       "            "url": "https://github.com/mattifestation/PowerSploit"\n",
        "        }\n",
        "    ],\n",
        "    "object_marking_refs": [\n",
@@ -405,7 +416,7 @@
        ""
       ]
      },
-     "execution_count": 6,
+     "execution_count": 9,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -417,7 +428,7 @@
     "fs_source = FileSystemSource(\"/tmp/stix2_source\")\n",
     "\n",
     "# retrieve STIX 2 objects\n",
-    "ap = fs_source.get(\"attack-pattern--00d0b012-8a03-410e-95de-5826bf542de6\")\n",
+    "ap = fs_source.get(\"attack-pattern--0a3ead4e-6d47-4ccb-854c-a6a4f9d96b22\")\n",
     "\n",
     "# for visual purposes\n",
     "print(ap)"
@@ -425,7 +436,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 10,
    "metadata": {},
    "outputs": [
     {
@@ -499,14 +510,14 @@
        ".highlight .vg { color: #19177C } /* Name.Variable.Global */\n",
        ".highlight .vi { color: #19177C } /* Name.Variable.Instance */\n",
        ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n",
-       ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
malware--96b08451-b27a-4ff6-893f-790e26393a8e\n",
+       ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
malware--92ec0cbd-2c30-44a2-b270-73f4ec949841\n",
        "
\n" ], "text/plain": [ "" ] }, - "execution_count": 7, + "execution_count": 10, "metadata": {}, "output_type": "execute_result" }, @@ -588,7 +599,89 @@ "" ] }, - "execution_count": 7, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "text/html": [ + "
malware--96b08451-b27a-4ff6-893f-790e26393a8e\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 10, "metadata": {}, "output_type": "execute_result" }, @@ -670,7 +763,7 @@ "" ] }, - "execution_count": 7, + "execution_count": 10, "metadata": {}, "output_type": "execute_result" }, @@ -745,14 +838,178 @@ ".highlight .vg { color: #19177C } /* Name.Variable.Global */\n", ".highlight .vi { color: #19177C } /* Name.Variable.Instance */\n", ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n", - ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
malware--92ec0cbd-2c30-44a2-b270-73f4ec949841\n",
+       ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
malware--6b616fc1-1505-48e3-8b2c-0d19337bff38\n",
        "
\n" ], "text/plain": [ "" ] }, - "execution_count": 7, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "text/html": [ + "
malware--6b616fc1-1505-48e3-8b2c-0d19337bff38\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "text/html": [ + "
malware--6b616fc1-1505-48e3-8b2c-0d19337bff38\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 10, "metadata": {}, "output_type": "execute_result" } @@ -772,7 +1029,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 11, "metadata": {}, "outputs": [ { @@ -853,7 +1110,253 @@ "" ] }, - "execution_count": 8, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "text/html": [ + "
malware--6b616fc1-1505-48e3-8b2c-0d19337bff38\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "text/html": [ + "
malware--6b616fc1-1505-48e3-8b2c-0d19337bff38\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "text/html": [ + "
malware--6b616fc1-1505-48e3-8b2c-0d19337bff38\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 11, "metadata": {}, "output_type": "execute_result" } @@ -880,7 +1383,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 13, "metadata": {}, "outputs": [], "source": [ @@ -891,15 +1394,15 @@ "\n", "# create STIX objects and add to sink\n", "camp = Campaign(name=\"The Crusades\",\n", - " objective=\"Infiltrating Israeli, Iranian and Palestinian digital infrastructure and government systems.\",\n", - " aliases=[\"Desert Moon\"])\n", + " objective=\"Infiltrating Israeli, Iranian and Palestinian digital infrastructure and government systems.\",\n", + " aliases=[\"Desert Moon\"])\n", "\n", "ind = Indicator(description=\"Crusades C2 implant\",\n", - " labels=[\"malicious-activity\"],\n", + " pattern_type=\"stix\",\n", " pattern=\"[file:hashes.'SHA-256' = '54b7e05e39a59428743635242e4a867c932140a999f52a1e54fa7ee6a440c73b']\")\n", "\n", "ind1 = Indicator(description=\"Crusades C2 implant\",\n", - " labels=[\"malicious-activity\"],\n", + " pattern_type=\"stix\",\n", " pattern=\"[file:hashes.'SHA-256' = '54b7e05e39a59428743635242e4a867c932140a999f52a1e54fa7ee6a440c73b']\")\n", "\n", "# add Campaign object to FileSystemSink\n", @@ -926,7 +1429,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.3" + "version": "3.9.0a6" } }, "nbformat": 4, diff --git a/docs/guide/markings.ipynb b/docs/guide/markings.ipynb index 44e023a..1e1609c 100644 --- a/docs/guide/markings.ipynb +++ b/docs/guide/markings.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 6, + "execution_count": 1, "metadata": { "nbsphinx": "hidden" }, @@ -15,6 +15,7 @@ "def hide_traceback(exc_tuple=None, filename=None, tb_offset=None,\n", " exception_only=False, running_compiled_code=False):\n", " etype, value, tb = sys.exc_info()\n", + " value.__cause__ = None # suppress chained exceptions\n", " return ipython._showtraceback(etype, value, ipython.InteractiveTB.get_exception_only(etype, value))\n", "\n", "ipython.showtraceback = hide_traceback" @@ -22,7 +23,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 2, "metadata": { "nbsphinx": "hidden" }, @@ -70,7 +71,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 3, "metadata": {}, "outputs": [ { @@ -146,14 +147,14 @@ ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n", ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "indicator",\n",
-       "    "id": "indicator--95a71cff-fad0-4ffb-a641-8a6eaa642290",\n",
-       "    "created": "2018-04-05T19:49:47.924Z",\n",
-       "    "modified": "2018-04-05T19:49:47.924Z",\n",
+       "    "spec_version": "2.1",\n",
+       "    "id": "indicator--46498844-7689-4e7b-be25-b119d8401159",\n",
+       "    "created": "2020-06-24T20:55:56.088861Z",\n",
+       "    "modified": "2020-06-24T20:55:56.088861Z",\n",
        "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
-       "    "valid_from": "2018-04-05T19:49:47.924708Z",\n",
-       "    "labels": [\n",
-       "        "malicious-activity"\n",
-       "    ],\n",
+       "    "pattern_type": "stix",\n",
+       "    "pattern_version": "2.1",\n",
+       "    "valid_from": "2020-06-24T20:55:56.088861Z",\n",
        "    "object_marking_refs": [\n",
        "        "marking-definition--f88d31f6-486f-44da-b317-01333bde0b82"\n",
        "    ]\n",
@@ -164,7 +165,7 @@
        ""
       ]
      },
-     "execution_count": 7,
+     "execution_count": 3,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -172,7 +173,7 @@
    "source": [
     "from stix2 import Indicator, TLP_AMBER\n",
     "\n",
-    "indicator = Indicator(labels=[\"malicious-activity\"],\n",
+    "indicator = Indicator(pattern_type=\"stix\",\n",
     "                      pattern=\"[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']\",\n",
     "                      object_marking_refs=TLP_AMBER)\n",
     "print(indicator)"
@@ -185,6 +186,343 @@
     "If you’re creating your own marking (for example, a ``Statement`` marking), first create the statement marking:"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "
{\n",
+       "    "type": "marking-definition",\n",
+       "    "spec_version": "2.1",\n",
+       "    "id": "marking-definition--9a4efc30-a7ac-42d0-8776-16f390a0fd44",\n",
+       "    "created": "2020-06-24T20:56:06.779241Z",\n",
+       "    "definition_type": "statement",\n",
+       "    "definition": {\n",
+       "        "statement": "Copyright 2017, Example Corp"\n",
+       "    }\n",
+       "}\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from stix2 import MarkingDefinition, StatementMarking\n", + "\n", + "marking_definition = MarkingDefinition( \n", + " definition_type=\"statement\", \n", + " definition=StatementMarking(statement=\"Copyright 2017, Example Corp\")\n", + ")\n", + "print(marking_definition)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Then you can add it to an object as it’s being created (passing either full object or the the ID as a keyword argument, like with relationships)." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
{\n",
+       "    "type": "indicator",\n",
+       "    "spec_version": "2.1",\n",
+       "    "id": "indicator--75d66696-9960-4229-ba89-2caac50891b3",\n",
+       "    "created": "2020-06-24T20:56:29.80259Z",\n",
+       "    "modified": "2020-06-24T20:56:29.80259Z",\n",
+       "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
+       "    "pattern_type": "stix",\n",
+       "    "pattern_version": "2.1",\n",
+       "    "valid_from": "2020-06-24T20:56:29.80259Z",\n",
+       "    "object_marking_refs": [\n",
+       "        "marking-definition--9a4efc30-a7ac-42d0-8776-16f390a0fd44"\n",
+       "    ]\n",
+       "}\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "indicator2 = Indicator(pattern_type=\"stix\",\n", + " pattern=\"[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']\",\n", + " object_marking_refs=marking_definition)\n", + "print(indicator2)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
{\n",
+       "    "type": "indicator",\n",
+       "    "spec_version": "2.1",\n",
+       "    "id": "indicator--757ea853-138c-44e2-bb00-e78eebfaa378",\n",
+       "    "created": "2020-06-24T20:56:43.703563Z",\n",
+       "    "modified": "2020-06-24T20:56:43.703563Z",\n",
+       "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
+       "    "pattern_type": "stix",\n",
+       "    "pattern_version": "2.1",\n",
+       "    "valid_from": "2020-06-24T20:56:43.703563Z",\n",
+       "    "object_marking_refs": [\n",
+       "        "marking-definition--f88d31f6-486f-44da-b317-01333bde0b82"\n",
+       "    ]\n",
+       "}\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "indicator3 = Indicator(pattern_type=\"stix\",\n", + " pattern=\"[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']\",\n", + " object_marking_refs=\"marking-definition--f88d31f6-486f-44da-b317-01333bde0b82\")\n", + "print(indicator3)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Granular markings work in the same way, except you also need to provide a full granular-marking object (including the selector)." + ] + }, { "cell_type": "code", "execution_count": 8, @@ -262,13 +600,28 @@ ".highlight .vi { color: #19177C } /* Name.Variable.Instance */\n", ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n", ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
-       "    "type": "marking-definition",\n",
-       "    "id": "marking-definition--13680b12-3d19-4b42-abe6-0d31effe5368",\n",
-       "    "created": "2018-04-05T19:49:53.98008Z",\n",
-       "    "definition_type": "statement",\n",
-       "    "definition": {\n",
-       "        "statement": "Copyright 2017, Example Corp"\n",
-       "    }\n",
+       "    "type": "malware",\n",
+       "    "spec_version": "2.1",\n",
+       "    "id": "malware--1752bbec-765a-4711-a304-f0e92ca902ae",\n",
+       "    "created": "2020-06-24T21:21:07.148194Z",\n",
+       "    "modified": "2020-06-24T21:21:07.148194Z",\n",
+       "    "name": "Poison Ivy",\n",
+       "    "description": "A ransomware related to ...",\n",
+       "    "is_family": false,\n",
+       "    "granular_markings": [\n",
+       "        {\n",
+       "            "marking_ref": "marking-definition--9a4efc30-a7ac-42d0-8776-16f390a0fd44",\n",
+       "            "selectors": [\n",
+       "                "description"\n",
+       "            ]\n",
+       "        },\n",
+       "        {\n",
+       "            "marking_ref": "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9",\n",
+       "            "selectors": [\n",
+       "                "name"\n",
+       "            ]\n",
+       "        }\n",
+       "    ]\n",
        "}\n",
        "
\n" ], @@ -282,20 +635,29 @@ } ], "source": [ - "from stix2 import MarkingDefinition, StatementMarking\n", + "from stix2 import Malware, TLP_WHITE\n", "\n", - "marking_definition = MarkingDefinition( \n", - " definition_type=\"statement\", \n", - " definition=StatementMarking(statement=\"Copyright 2017, Example Corp\")\n", - ")\n", - "print(marking_definition)" + "malware = Malware(name=\"Poison Ivy\",\n", + " description=\"A ransomware related to ...\",\n", + " is_family=False,\n", + " granular_markings=[\n", + " {\n", + " \"selectors\": [\"description\"],\n", + " \"marking_ref\": marking_definition\n", + " },\n", + " {\n", + " \"selectors\": [\"name\"],\n", + " \"marking_ref\": TLP_WHITE\n", + " }\n", + " ])\n", + "print(malware)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Then you can add it to an object as it’s being created (passing either full object or the the ID as a keyword argument, like with relationships)." + "Make sure that the selector is a field that exists and is populated on the object, otherwise this will cause an error:" ] }, { @@ -304,106 +666,42 @@ "metadata": {}, "outputs": [ { - "data": { - "text/html": [ - "
{\n",
-       "    "type": "indicator",\n",
-       "    "id": "indicator--7caeab49-2472-41bb-a988-2f990aea99bd",\n",
-       "    "created": "2018-04-05T19:49:55.763Z",\n",
-       "    "modified": "2018-04-05T19:49:55.763Z",\n",
-       "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
-       "    "valid_from": "2018-04-05T19:49:55.763364Z",\n",
-       "    "labels": [\n",
-       "        "malicious-activity"\n",
-       "    ],\n",
-       "    "object_marking_refs": [\n",
-       "        "marking-definition--13680b12-3d19-4b42-abe6-0d31effe5368"\n",
-       "    ]\n",
-       "}\n",
-       "
\n" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" + "ename": "InvalidSelectorError", + "evalue": "Selector title in Malware is not valid!", + "output_type": "error", + "traceback": [ + "\u001b[0;31mInvalidSelectorError\u001b[0m\u001b[0;31m:\u001b[0m Selector title in Malware is not valid!\n" + ] } ], "source": [ - "indicator2 = Indicator(labels=[\"malicious-activity\"],\n", - " pattern=\"[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']\",\n", - " object_marking_refs=marking_definition)\n", - "print(indicator2)" + "Malware(name=\"Poison Ivy\",\n", + " description=\"A ransomware related to ...\",\n", + " is_family=False,\n", + " granular_markings=[\n", + " {\n", + " \"selectors\": [\"title\"],\n", + " \"marking_ref\": marking_definition\n", + " }\n", + " ])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Adding Data Markings To Existing Objects" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "[Several functions](../api/stix2.markings.rst) exist to support working with data markings.\n", + "\n", + "Both object markings and granular markings can be added to STIX objects which have already been created.\n", + "\n", + "**Note**: Doing so will create a new version of the object (note the updated ``modified`` time)." ] }, { @@ -484,16 +782,17 @@ ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n", ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "indicator",\n",
-       "    "id": "indicator--4eb21bbe-b8a9-4348-86cf-1ed52f9abdd7",\n",
-       "    "created": "2018-04-05T19:49:57.248Z",\n",
-       "    "modified": "2018-04-05T19:49:57.248Z",\n",
+       "    "spec_version": "2.1",\n",
+       "    "id": "indicator--46498844-7689-4e7b-be25-b119d8401159",\n",
+       "    "created": "2020-06-24T20:55:56.088861Z",\n",
+       "    "modified": "2020-06-24T21:21:39.898475Z",\n",
        "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
-       "    "valid_from": "2018-04-05T19:49:57.248658Z",\n",
-       "    "labels": [\n",
-       "        "malicious-activity"\n",
-       "    ],\n",
+       "    "pattern_type": "stix",\n",
+       "    "pattern_version": "2.1",\n",
+       "    "valid_from": "2020-06-24T20:55:56.088861Z",\n",
        "    "object_marking_refs": [\n",
-       "        "marking-definition--f88d31f6-486f-44da-b317-01333bde0b82"\n",
+       "        "marking-definition--f88d31f6-486f-44da-b317-01333bde0b82",\n",
+       "        "marking-definition--9a4efc30-a7ac-42d0-8776-16f390a0fd44"\n",
        "    ]\n",
        "}\n",
        "
\n" @@ -508,17 +807,15 @@ } ], "source": [ - "indicator3 = Indicator(labels=[\"malicious-activity\"],\n", - " pattern=\"[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']\",\n", - " object_marking_refs=\"marking-definition--f88d31f6-486f-44da-b317-01333bde0b82\")\n", - "print(indicator3)" + "indicator4 = indicator.add_markings(marking_definition)\n", + "print(indicator4)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Granular markings work in the same way, except you also need to provide a full granular-marking object (including the selector)." + "You can also remove specific markings from STIX objects. This will also create a new version of the object." ] }, { @@ -598,28 +895,17 @@ ".highlight .vi { color: #19177C } /* Name.Variable.Instance */\n", ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n", ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
-       "    "type": "malware",\n",
-       "    "id": "malware--ef1eddbb-b5a5-47e0-b607-75b9870d8d91",\n",
-       "    "created": "2018-04-05T19:49:59.103Z",\n",
-       "    "modified": "2018-04-05T19:49:59.103Z",\n",
-       "    "name": "Poison Ivy",\n",
-       "    "description": "A ransomware related to ...",\n",
-       "    "labels": [\n",
-       "        "remote-access-trojan"\n",
-       "    ],\n",
-       "    "granular_markings": [\n",
-       "        {\n",
-       "            "marking_ref": "marking-definition--13680b12-3d19-4b42-abe6-0d31effe5368",\n",
-       "            "selectors": [\n",
-       "                "description"\n",
-       "            ]\n",
-       "        },\n",
-       "        {\n",
-       "            "marking_ref": "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9",\n",
-       "            "selectors": [\n",
-       "                "name"\n",
-       "            ]\n",
-       "        }\n",
+       "    "type": "indicator",\n",
+       "    "spec_version": "2.1",\n",
+       "    "id": "indicator--46498844-7689-4e7b-be25-b119d8401159",\n",
+       "    "created": "2020-06-24T20:55:56.088861Z",\n",
+       "    "modified": "2020-06-24T21:21:43.529702Z",\n",
+       "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
+       "    "pattern_type": "stix",\n",
+       "    "pattern_version": "2.1",\n",
+       "    "valid_from": "2020-06-24T20:55:56.088861Z",\n",
+       "    "object_marking_refs": [\n",
+       "        "marking-definition--f88d31f6-486f-44da-b317-01333bde0b82"\n",
        "    ]\n",
        "}\n",
        "
\n" @@ -634,29 +920,15 @@ } ], "source": [ - "from stix2 import Malware, TLP_WHITE\n", - "\n", - "malware = Malware(name=\"Poison Ivy\",\n", - " labels=['remote-access-trojan'],\n", - " description=\"A ransomware related to ...\",\n", - " granular_markings=[\n", - " {\n", - " \"selectors\": [\"description\"],\n", - " \"marking_ref\": marking_definition\n", - " },\n", - " {\n", - " \"selectors\": [\"name\"],\n", - " \"marking_ref\": TLP_WHITE\n", - " }\n", - " ])\n", - "print(malware)" + "indicator5 = indicator4.remove_markings(marking_definition)\n", + "print(indicator5)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Make sure that the selector is a field that exists and is populated on the object, otherwise this will cause an error:" + "The markings on an object can be replaced with a different set of markings:" ] }, { @@ -665,42 +937,114 @@ "metadata": {}, "outputs": [ { - "ename": "InvalidSelectorError", - "evalue": "Selector title in Malware is not valid!", - "output_type": "error", - "traceback": [ - "\u001b[0;31mInvalidSelectorError\u001b[0m\u001b[0;31m:\u001b[0m Selector title in Malware is not valid!\n" - ] + "data": { + "text/html": [ + "
{\n",
+       "    "type": "indicator",\n",
+       "    "spec_version": "2.1",\n",
+       "    "id": "indicator--46498844-7689-4e7b-be25-b119d8401159",\n",
+       "    "created": "2020-06-24T20:55:56.088861Z",\n",
+       "    "modified": "2020-06-24T21:21:47.703212Z",\n",
+       "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
+       "    "pattern_type": "stix",\n",
+       "    "pattern_version": "2.1",\n",
+       "    "valid_from": "2020-06-24T20:55:56.088861Z",\n",
+       "    "object_marking_refs": [\n",
+       "        "marking-definition--9a4efc30-a7ac-42d0-8776-16f390a0fd44",\n",
+       "        "marking-definition--34098fce-860f-48ae-8e50-ebd3cc5e41da"\n",
+       "    ]\n",
+       "}\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ - "Malware(name=\"Poison Ivy\",\n", - " labels=['remote-access-trojan'],\n", - " description=\"A ransomware related to ...\",\n", - " granular_markings=[\n", - " {\n", - " \"selectors\": [\"title\"],\n", - " \"marking_ref\": marking_definition\n", - " }\n", - " ])" + "from stix2 import TLP_GREEN\n", + "\n", + "indicator6 = indicator5.set_markings([TLP_GREEN, marking_definition])\n", + "print(indicator6)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "### Adding Data Markings To Existing Objects" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "[Several functions](../api/stix2.markings.rst) exist to support working with data markings.\n", - "\n", - "Both object markings and granular markings can be added to STIX objects which have already been created.\n", - "\n", - "**Note**: Doing so will create a new version of the object (note the updated ``modified`` time)." + "STIX objects can also be cleared of all markings with [clear_markings()](../api/stix2.markings.rst#stix2.markings.clear_markings):" ] }, { @@ -781,18 +1125,14 @@ ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n", ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "indicator",\n",
-       "    "id": "indicator--95a71cff-fad0-4ffb-a641-8a6eaa642290",\n",
-       "    "created": "2018-04-05T19:49:47.924Z",\n",
-       "    "modified": "2018-04-05T19:50:03.387Z",\n",
+       "    "spec_version": "2.1",\n",
+       "    "id": "indicator--46498844-7689-4e7b-be25-b119d8401159",\n",
+       "    "created": "2020-06-24T20:55:56.088861Z",\n",
+       "    "modified": "2020-06-24T21:21:53.287178Z",\n",
        "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
-       "    "valid_from": "2018-04-05T19:49:47.924708Z",\n",
-       "    "labels": [\n",
-       "        "malicious-activity"\n",
-       "    ],\n",
-       "    "object_marking_refs": [\n",
-       "        "marking-definition--13680b12-3d19-4b42-abe6-0d31effe5368",\n",
-       "        "marking-definition--f88d31f6-486f-44da-b317-01333bde0b82"\n",
-       "    ]\n",
+       "    "pattern_type": "stix",\n",
+       "    "pattern_version": "2.1",\n",
+       "    "valid_from": "2020-06-24T20:55:56.088861Z"\n",
        "}\n",
        "
\n" ], @@ -805,345 +1145,6 @@ "output_type": "execute_result" } ], - "source": [ - "indicator4 = indicator.add_markings(marking_definition)\n", - "print(indicator4)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "You can also remove specific markings from STIX objects. This will also create a new version of the object." - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
{\n",
-       "    "type": "indicator",\n",
-       "    "id": "indicator--95a71cff-fad0-4ffb-a641-8a6eaa642290",\n",
-       "    "created": "2018-04-05T19:49:47.924Z",\n",
-       "    "modified": "2018-04-05T19:50:05.109Z",\n",
-       "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
-       "    "valid_from": "2018-04-05T19:49:47.924708Z",\n",
-       "    "labels": [\n",
-       "        "malicious-activity"\n",
-       "    ],\n",
-       "    "object_marking_refs": [\n",
-       "        "marking-definition--f88d31f6-486f-44da-b317-01333bde0b82"\n",
-       "    ]\n",
-       "}\n",
-       "
\n" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "indicator5 = indicator4.remove_markings(marking_definition)\n", - "print(indicator5)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The markings on an object can be replaced with a different set of markings:" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
{\n",
-       "    "type": "indicator",\n",
-       "    "id": "indicator--95a71cff-fad0-4ffb-a641-8a6eaa642290",\n",
-       "    "created": "2018-04-05T19:49:47.924Z",\n",
-       "    "modified": "2018-04-05T19:50:06.773Z",\n",
-       "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
-       "    "valid_from": "2018-04-05T19:49:47.924708Z",\n",
-       "    "labels": [\n",
-       "        "malicious-activity"\n",
-       "    ],\n",
-       "    "object_marking_refs": [\n",
-       "        "marking-definition--13680b12-3d19-4b42-abe6-0d31effe5368",\n",
-       "        "marking-definition--34098fce-860f-48ae-8e50-ebd3cc5e41da"\n",
-       "    ]\n",
-       "}\n",
-       "
\n" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from stix2 import TLP_GREEN\n", - "\n", - "indicator6 = indicator5.set_markings([TLP_GREEN, marking_definition])\n", - "print(indicator6)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "STIX objects can also be cleared of all markings with [clear_markings()](../api/stix2.markings.rst#stix2.markings.clear_markings):" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
{\n",
-       "    "type": "indicator",\n",
-       "    "id": "indicator--95a71cff-fad0-4ffb-a641-8a6eaa642290",\n",
-       "    "created": "2018-04-05T19:49:47.924Z",\n",
-       "    "modified": "2018-04-05T19:50:08.616Z",\n",
-       "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
-       "    "valid_from": "2018-04-05T19:49:47.924708Z",\n",
-       "    "labels": [\n",
-       "        "malicious-activity"\n",
-       "    ]\n",
-       "}\n",
-       "
\n" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 16, - "metadata": {}, - "output_type": "execute_result" - } - ], "source": [ "indicator7 = indicator5.clear_markings()\n", "print(indicator7)" @@ -1167,17 +1168,17 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 14, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "['marking-definition--13680b12-3d19-4b42-abe6-0d31effe5368',\n", + "['marking-definition--9a4efc30-a7ac-42d0-8776-16f390a0fd44',\n", " 'marking-definition--34098fce-860f-48ae-8e50-ebd3cc5e41da']" ] }, - "execution_count": 17, + "execution_count": 14, "metadata": {}, "output_type": "execute_result" } @@ -1195,7 +1196,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 15, "metadata": {}, "outputs": [ { @@ -1204,7 +1205,7 @@ "['marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9']" ] }, - "execution_count": 18, + "execution_count": 15, "metadata": {}, "output_type": "execute_result" } @@ -1224,7 +1225,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 16, "metadata": {}, "outputs": [ { @@ -1233,7 +1234,7 @@ "['marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9']" ] }, - "execution_count": 19, + "execution_count": 16, "metadata": {}, "output_type": "execute_result" } @@ -1251,7 +1252,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 17, "metadata": {}, "outputs": [ { @@ -1260,7 +1261,7 @@ "True" ] }, - "execution_count": 20, + "execution_count": 17, "metadata": {}, "output_type": "execute_result" } @@ -1271,7 +1272,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 18, "metadata": {}, "outputs": [ { @@ -1280,7 +1281,7 @@ "True" ] }, - "execution_count": 21, + "execution_count": 18, "metadata": {}, "output_type": "execute_result" } @@ -1291,7 +1292,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 19, "metadata": { "scrolled": true }, @@ -1302,7 +1303,7 @@ "False" ] }, - "execution_count": 22, + "execution_count": 19, "metadata": {}, "output_type": "execute_result" } @@ -1322,54 +1323,375 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 20, "metadata": {}, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "{\n", - " \"type\": \"indicator\",\n", - " \"spec_version\": \"2.1\",\n", - " \"id\": \"indicator--634ef462-d6b5-48bc-9d9f-b46a6919227c\",\n", - " \"created\": \"2019-05-03T18:36:44.354Z\",\n", - " \"modified\": \"2019-05-03T18:36:44.354Z\",\n", - " \"description\": \"Una descripcion sobre este indicador\",\n", - " \"indicator_types\": [\n", - " \"malware\"\n", - " ],\n", - " \"pattern\": \"[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']\",\n", - " \"valid_from\": \"2019-05-03T18:36:44.354443Z\",\n", - " \"object_marking_refs\": [\n", - " \"marking-definition--f88d31f6-486f-44da-b317-01333bde0b82\"\n", - " ],\n", - " \"granular_markings\": [\n", - " {\n", - " \"lang\": \"es\",\n", - " \"selectors\": [\n", - " \"description\"\n", - " ]\n", - " },\n", - " {\n", - " \"marking_ref\": \"marking-definition--34098fce-860f-48ae-8e50-ebd3cc5e41da\",\n", - " \"selectors\": [\n", - " \"description\"\n", - " ]\n", - " }\n", - " ]\n", - "}\n", - "['es', 'marking-definition--34098fce-860f-48ae-8e50-ebd3cc5e41da']\n", - "['marking-definition--34098fce-860f-48ae-8e50-ebd3cc5e41da']\n", - "['es']\n" - ] + "data": { + "text/html": [ + "
{\n",
+       "    "type": "indicator",\n",
+       "    "spec_version": "2.1",\n",
+       "    "id": "indicator--f4004de9-a6d9-4c7b-823e-3d8199173c09",\n",
+       "    "created": "2020-06-24T21:35:08.630228Z",\n",
+       "    "modified": "2020-06-24T21:35:08.630228Z",\n",
+       "    "description": "Una descripcion sobre este indicador",\n",
+       "    "indicator_types": [\n",
+       "        "malware"\n",
+       "    ],\n",
+       "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
+       "    "pattern_type": "stix",\n",
+       "    "pattern_version": "2.1",\n",
+       "    "valid_from": "2020-06-24T21:35:08.630228Z",\n",
+       "    "object_marking_refs": [\n",
+       "        "marking-definition--f88d31f6-486f-44da-b317-01333bde0b82"\n",
+       "    ],\n",
+       "    "granular_markings": [\n",
+       "        {\n",
+       "            "lang": "es",\n",
+       "            "selectors": [\n",
+       "                "description"\n",
+       "            ]\n",
+       "        },\n",
+       "        {\n",
+       "            "marking_ref": "marking-definition--34098fce-860f-48ae-8e50-ebd3cc5e41da",\n",
+       "            "selectors": [\n",
+       "                "description"\n",
+       "            ]\n",
+       "        }\n",
+       "    ]\n",
+       "}\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "text/html": [ + "
['marking-definition--34098fce-860f-48ae-8e50-ebd3cc5e41da', 'es']\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "text/html": [ + "
['marking-definition--34098fce-860f-48ae-8e50-ebd3cc5e41da']\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "text/html": [ + "
['es']\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ - "from stix2 import v21\n", + "from stix2 import Indicator\n", "\n", - "v21_indicator = v21.Indicator(\n", + "v21_indicator = Indicator(\n", " description=\"Una descripcion sobre este indicador\",\n", + " pattern_type=\"stix\",\n", " pattern=\"[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']\",\n", " object_marking_refs=['marking-definition--f88d31f6-486f-44da-b317-01333bde0b82'],\n", " indicator_types=['malware'],\n", @@ -1405,30 +1727,107 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 21, "metadata": {}, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "{\n", - " \"type\": \"indicator\",\n", - " \"spec_version\": \"2.1\",\n", - " \"id\": \"indicator--a612665a-2df4-4fd2-851c-7fbb8c92339a\",\n", - " \"created\": \"2019-05-03T19:13:59.010Z\",\n", - " \"modified\": \"2019-05-03T19:15:41.173Z\",\n", - " \"description\": \"Una descripcion sobre este indicador\",\n", - " \"indicator_types\": [\n", - " \"malware\"\n", - " ],\n", - " \"pattern\": \"[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']\",\n", - " \"valid_from\": \"2019-05-03T19:13:59.010624Z\",\n", - " \"object_marking_refs\": [\n", - " \"marking-definition--f88d31f6-486f-44da-b317-01333bde0b82\"\n", - " ]\n", - "}\n" - ] + "data": { + "text/html": [ + "
{\n",
+       "    "type": "indicator",\n",
+       "    "spec_version": "2.1",\n",
+       "    "id": "indicator--f4004de9-a6d9-4c7b-823e-3d8199173c09",\n",
+       "    "created": "2020-06-24T21:35:08.630228Z",\n",
+       "    "modified": "2020-06-24T21:35:14.54482Z",\n",
+       "    "description": "Una descripcion sobre este indicador",\n",
+       "    "indicator_types": [\n",
+       "        "malware"\n",
+       "    ],\n",
+       "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
+       "    "pattern_type": "stix",\n",
+       "    "pattern_version": "2.1",\n",
+       "    "valid_from": "2020-06-24T21:35:08.630228Z",\n",
+       "    "object_marking_refs": [\n",
+       "        "marking-definition--f88d31f6-486f-44da-b317-01333bde0b82"\n",
+       "    ]\n",
+       "}\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ @@ -1437,38 +1836,115 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 22, "metadata": {}, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "{\n", - " \"type\": \"indicator\",\n", - " \"spec_version\": \"2.1\",\n", - " \"id\": \"indicator--982aeb4d-4dd3-4b04-aa50-a1d00c31986c\",\n", - " \"created\": \"2019-05-03T19:19:26.542Z\",\n", - " \"modified\": \"2019-05-03T19:20:51.818Z\",\n", - " \"description\": \"Una descripcion sobre este indicador\",\n", - " \"indicator_types\": [\n", - " \"malware\"\n", - " ],\n", - " \"pattern\": \"[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']\",\n", - " \"valid_from\": \"2019-05-03T19:19:26.542267Z\",\n", - " \"object_marking_refs\": [\n", - " \"marking-definition--f88d31f6-486f-44da-b317-01333bde0b82\"\n", - " ],\n", - " \"granular_markings\": [\n", - " {\n", - " \"lang\": \"es\",\n", - " \"selectors\": [\n", - " \"description\"\n", - " ]\n", - " }\n", - " ]\n", - "}\n" - ] + "data": { + "text/html": [ + "
{\n",
+       "    "type": "indicator",\n",
+       "    "spec_version": "2.1",\n",
+       "    "id": "indicator--f4004de9-a6d9-4c7b-823e-3d8199173c09",\n",
+       "    "created": "2020-06-24T21:35:08.630228Z",\n",
+       "    "modified": "2020-06-24T21:35:39.298138Z",\n",
+       "    "description": "Una descripcion sobre este indicador",\n",
+       "    "indicator_types": [\n",
+       "        "malware"\n",
+       "    ],\n",
+       "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
+       "    "pattern_type": "stix",\n",
+       "    "pattern_version": "2.1",\n",
+       "    "valid_from": "2020-06-24T21:35:08.630228Z",\n",
+       "    "object_marking_refs": [\n",
+       "        "marking-definition--f88d31f6-486f-44da-b317-01333bde0b82"\n",
+       "    ],\n",
+       "    "granular_markings": [\n",
+       "        {\n",
+       "            "lang": "es",\n",
+       "            "selectors": [\n",
+       "                "description"\n",
+       "            ]\n",
+       "        }\n",
+       "    ]\n",
+       "}\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ @@ -1478,38 +1954,115 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 23, "metadata": {}, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "{\n", - " \"type\": \"indicator\",\n", - " \"spec_version\": \"2.1\",\n", - " \"id\": \"indicator--de0316d6-38e1-43c2-af4f-649305251864\",\n", - " \"created\": \"2019-05-03T19:40:21.459Z\",\n", - " \"modified\": \"2019-05-03T19:40:26.431Z\",\n", - " \"description\": \"Una descripcion sobre este indicador\",\n", - " \"indicator_types\": [\n", - " \"malware\"\n", - " ],\n", - " \"pattern\": \"[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']\",\n", - " \"valid_from\": \"2019-05-03T19:40:21.459582Z\",\n", - " \"object_marking_refs\": [\n", - " \"marking-definition--f88d31f6-486f-44da-b317-01333bde0b82\"\n", - " ],\n", - " \"granular_markings\": [\n", - " {\n", - " \"marking_ref\": \"marking-definition--34098fce-860f-48ae-8e50-ebd3cc5e41da\",\n", - " \"selectors\": [\n", - " \"description\"\n", - " ]\n", - " }\n", - " ]\n", - "}\n" - ] + "data": { + "text/html": [ + "
{\n",
+       "    "type": "indicator",\n",
+       "    "spec_version": "2.1",\n",
+       "    "id": "indicator--f4004de9-a6d9-4c7b-823e-3d8199173c09",\n",
+       "    "created": "2020-06-24T21:35:08.630228Z",\n",
+       "    "modified": "2020-06-24T21:35:42.684794Z",\n",
+       "    "description": "Una descripcion sobre este indicador",\n",
+       "    "indicator_types": [\n",
+       "        "malware"\n",
+       "    ],\n",
+       "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
+       "    "pattern_type": "stix",\n",
+       "    "pattern_version": "2.1",\n",
+       "    "valid_from": "2020-06-24T21:35:08.630228Z",\n",
+       "    "object_marking_refs": [\n",
+       "        "marking-definition--f88d31f6-486f-44da-b317-01333bde0b82"\n",
+       "    ],\n",
+       "    "granular_markings": [\n",
+       "        {\n",
+       "            "marking_ref": "marking-definition--34098fce-860f-48ae-8e50-ebd3cc5e41da",\n",
+       "            "selectors": [\n",
+       "                "description"\n",
+       "            ]\n",
+       "        }\n",
+       "    ]\n",
+       "}\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ @@ -1534,7 +2087,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.3" + "version": "3.9.0a6" } }, "nbformat": 4, diff --git a/docs/guide/memory.ipynb b/docs/guide/memory.ipynb index 491187b..d8a7000 100644 --- a/docs/guide/memory.ipynb +++ b/docs/guide/memory.ipynb @@ -15,6 +15,7 @@ "def hide_traceback(exc_tuple=None, filename=None, tb_offset=None,\n", " exception_only=False, running_compiled_code=False):\n", " etype, value, tb = sys.exc_info()\n", + " value.__cause__ = None # suppress chained exceptions\n", " return ipython._showtraceback(etype, value, ipython.InteractiveTB.get_exception_only(etype, value))\n", "\n", "ipython.showtraceback = hide_traceback" @@ -151,15 +152,15 @@ ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n", ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "indicator",\n",
-       "    "id": "indicator--41a960c7-a6d4-406d-9156-0069cb3bd40d",\n",
-       "    "created": "2018-04-05T19:50:41.222Z",\n",
-       "    "modified": "2018-04-05T19:50:41.222Z",\n",
+       "    "spec_version": "2.1",\n",
+       "    "id": "indicator--96120abd-f767-4292-b8b0-b739749e03b6",\n",
+       "    "created": "2020-06-26T18:28:55.582226Z",\n",
+       "    "modified": "2020-06-26T18:28:55.582226Z",\n",
        "    "description": "Crusades C2 implant",\n",
        "    "pattern": "[file:hashes.'SHA-256' = '54b7e05e39a59428743635242e4a867c932140a999f52a1e54fa7ee6a440c73b']",\n",
-       "    "valid_from": "2018-04-05T19:50:41.222522Z",\n",
-       "    "labels": [\n",
-       "        "malicious-activity"\n",
-       "    ]\n",
+       "    "pattern_type": "stix",\n",
+       "    "pattern_version": "2.1",\n",
+       "    "valid_from": "2020-06-26T18:28:55.582226Z"\n",
        "}\n",
        "
\n" ], @@ -180,7 +181,7 @@ "\n", "# insert newly created indicator into memory\n", "ind = Indicator(description=\"Crusades C2 implant\",\n", - " labels=[\"malicious-activity\"],\n", + " pattern_type=\"stix\",\n", " pattern=\"[file:hashes.'SHA-256' = '54b7e05e39a59428743635242e4a867c932140a999f52a1e54fa7ee6a440c73b']\")\n", "\n", "mem.add(ind)\n", @@ -267,15 +268,15 @@ ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n", ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "indicator",\n",
-       "    "id": "indicator--ba2a7acb-a3ac-420b-9288-09988aa99408",\n",
-       "    "created": "2018-04-05T19:50:43.343Z",\n",
-       "    "modified": "2018-04-05T19:50:43.343Z",\n",
+       "    "spec_version": "2.1",\n",
+       "    "id": "indicator--dd5a0203-356d-415c-a118-fb6b0eae9de0",\n",
+       "    "created": "2020-06-26T18:28:58.047811Z",\n",
+       "    "modified": "2020-06-26T18:28:58.047811Z",\n",
        "    "description": "Crusades stage 2 implant variant",\n",
        "    "pattern": "[file:hashes.'SHA-256' = '31a45e777e4d58b97f4c43e38006f8cd6580ddabc4037905b2fad734712b582c']",\n",
-       "    "valid_from": "2018-04-05T19:50:43.343298Z",\n",
-       "    "labels": [\n",
-       "        "malicious-activity"\n",
-       "    ]\n",
+       "    "pattern_type": "stix",\n",
+       "    "pattern_version": "2.1",\n",
+       "    "valid_from": "2020-06-26T18:28:58.047811Z"\n",
        "}\n",
        "
\n" ], @@ -293,12 +294,12 @@ "\n", "# add multiple STIX objects into memory\n", "ind2 = Indicator(description=\"Crusades stage 2 implant\",\n", - " labels=[\"malicious-activity\"],\n", + " pattern_type=\"stix\",\n", " pattern=\"[file:hashes.'SHA-256' = '70fa62fb218dd9d936ee570dbe531dfa4e7c128ff37e6af7a6a6b2485487e50a']\")\n", "ind3 = Indicator(description=\"Crusades stage 2 implant variant\",\n", - " labels=[\"malicious-activity\"],\n", + " pattern_type=\"stix\",\n", " pattern=\"[file:hashes.'SHA-256' = '31a45e777e4d58b97f4c43e38006f8cd6580ddabc4037905b2fad734712b582c']\")\n", - "mal = Malware(labels=[\"rootkit\"], name= \"Alexios\")\n", + "mal = Malware(malware_types=[\"rootkit\"], name=\"Alexios\", is_family=False)\n", "\n", "mem.add([ind2,ind3, mal])\n", "\n", @@ -386,13 +387,15 @@ ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n", ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "malware",\n",
-       "    "id": "malware--9e9b87ce-2b2b-455a-8d5b-26384ccc8d52",\n",
-       "    "created": "2018-04-05T19:50:43.346Z",\n",
-       "    "modified": "2018-04-05T19:50:43.346Z",\n",
+       "    "spec_version": "2.1",\n",
+       "    "id": "malware--6cee28b8-4d42-4e72-bd77-ea47897672c0",\n",
+       "    "created": "2020-06-26T18:28:58.049244Z",\n",
+       "    "modified": "2020-06-26T18:28:58.049244Z",\n",
        "    "name": "Alexios",\n",
-       "    "labels": [\n",
+       "    "malware_types": [\n",
        "        "rootkit"\n",
-       "    ]\n",
+       "    ],\n",
+       "    "is_family": false\n",
        "}\n",
        "
\n" ], @@ -408,7 +411,7 @@ "source": [ "from stix2 import Filter\n", "\n", - "mal = mem.query([Filter(\"labels\",\"=\", \"rootkit\")])[0]\n", + "mal = mem.query([Filter(\"malware_types\",\"=\", \"rootkit\")])[0]\n", "print(mal)" ] }, @@ -421,7 +424,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 7, "metadata": {}, "outputs": [ { @@ -497,13 +500,15 @@ ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n", ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "malware",\n",
-       "    "id": "malware--9e9b87ce-2b2b-455a-8d5b-26384ccc8d52",\n",
-       "    "created": "2018-04-05T19:50:43.346Z",\n",
-       "    "modified": "2018-04-05T19:50:43.346Z",\n",
+       "    "spec_version": "2.1",\n",
+       "    "id": "malware--6cee28b8-4d42-4e72-bd77-ea47897672c0",\n",
+       "    "created": "2020-06-26T18:28:58.049244Z",\n",
+       "    "modified": "2020-06-26T18:28:58.049244Z",\n",
        "    "name": "Alexios",\n",
-       "    "labels": [\n",
+       "    "malware_types": [\n",
        "        "rootkit"\n",
-       "    ]\n",
+       "    ],\n",
+       "    "is_family": false\n",
        "}\n",
        "
\n" ], @@ -511,7 +516,7 @@ "" ] }, - "execution_count": 8, + "execution_count": 7, "metadata": {}, "output_type": "execute_result" } @@ -525,7 +530,7 @@ "# load(add) STIX content from json file into MemoryStore\n", "mem_2.load_from_file(\"path_to_target_file.json\")\n", "\n", - "report = mem_2.get(\"malware--9e9b87ce-2b2b-455a-8d5b-26384ccc8d52\")\n", + "report = mem_2.get(\"malware--6cee28b8-4d42-4e72-bd77-ea47897672c0\")\n", "\n", "# for visual purposes\n", "print(report)" @@ -548,7 +553,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.3" + "version": "3.9.0a6" } }, "nbformat": 4, diff --git a/docs/guide/parsing.ipynb b/docs/guide/parsing.ipynb index 5843981..0762c6c 100644 --- a/docs/guide/parsing.ipynb +++ b/docs/guide/parsing.ipynb @@ -15,6 +15,7 @@ "def hide_traceback(exc_tuple=None, filename=None, tb_offset=None,\n", " exception_only=False, running_compiled_code=False):\n", " etype, value, tb = sys.exc_info()\n", + " value.__cause__ = None # suppress chained exceptions\n", " return ipython._showtraceback(etype, value, ipython.InteractiveTB.get_exception_only(etype, value))\n", "\n", "ipython.showtraceback = hide_traceback" @@ -63,7 +64,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Parsing STIX content is as easy as calling the [parse()](../api/stix2.core.rst#stix2.core.parse) function on a JSON string, dictionary, or file-like object. It will automatically determine the type of the object. The STIX objects within `bundle` objects, and the cyber observables contained within `observed-data` objects will be parsed as well.\n", + "Parsing STIX content is as easy as calling the [parse()](../api/stix2.core.rst#stix2.core.parse) function on a JSON string, dictionary, or file-like object. It will automatically determine the type of the object. The STIX objects within `bundle` objects, and any cyber observables contained within `observed-data` objects will be parsed as well.\n", "\n", "**Parsing a string**" ] @@ -144,7 +145,7 @@ ".highlight .vg { color: #19177C } /* Name.Variable.Global */\n", ".highlight .vi { color: #19177C } /* Name.Variable.Instance */\n", ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n", - ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
<class 'stix2.v20.sdo.ObservedData'>\n",
+       ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
<class 'stix2.v21.sdo.ObservedData'>\n",
        "
\n" ], "text/plain": [ @@ -228,6 +229,7 @@ ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n", ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "observed-data",\n",
+       "    "spec_version": "2.1",\n",
        "    "id": "observed-data--b67d30ff-02ac-498a-92f9-32f845f448cf",\n",
        "    "created": "2016-04-06T19:58:16.000Z",\n",
        "    "modified": "2016-04-06T19:58:16.000Z",\n",
@@ -237,6 +239,8 @@
        "    "objects": {\n",
        "        "0": {\n",
        "            "type": "file",\n",
+       "            "id": "file--5d0833b7-065e-571f-8bf2-657cb9569570",\n",
+       "            "spec_version": "2.1",\n",
        "            "hashes": {\n",
        "                "SHA-256": "0969de02ecf8a5f003e3f6d063d848c8a193aada092623f8ce408c15bcb5f038"\n",
        "            }\n",
@@ -260,6 +264,7 @@
     "input_string = \"\"\"{\n",
     "    \"type\": \"observed-data\",\n",
     "    \"id\": \"observed-data--b67d30ff-02ac-498a-92f9-32f845f448cf\",\n",
+    "    \"spec_version\": \"2.1\",\n",
     "    \"created\": \"2016-04-06T19:58:16.000Z\",\n",
     "    \"modified\": \"2016-04-06T19:58:16.000Z\",\n",
     "    \"first_observed\": \"2015-12-21T19:00:00Z\",\n",
@@ -363,7 +368,7 @@
        ".highlight .vg { color: #19177C } /* Name.Variable.Global */\n",
        ".highlight .vi { color: #19177C } /* Name.Variable.Instance */\n",
        ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n",
-       ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
<class 'stix2.v20.sdo.Identity'>\n",
+       ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
<class 'stix2.v21.sdo.Identity'>\n",
        "
\n" ], "text/plain": [ @@ -447,6 +452,7 @@ ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n", ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "identity",\n",
+       "    "spec_version": "2.1",\n",
        "    "id": "identity--311b2d2d-f010-4473-83ec-1edf84858f4c",\n",
        "    "created": "2015-12-21T19:59:11.000Z",\n",
        "    "modified": "2015-12-21T19:59:11.000Z",\n",
@@ -468,6 +474,7 @@
     "input_dict = {\n",
     "    \"type\": \"identity\",\n",
     "    \"id\": \"identity--311b2d2d-f010-4473-83ec-1edf84858f4c\",\n",
+    "    \"spec_version\": \"2.1\",\n",
     "    \"created\": \"2015-12-21T19:59:11Z\",\n",
     "    \"modified\": \"2015-12-21T19:59:11Z\",\n",
     "    \"name\": \"Cole Powers\",\n",
@@ -488,7 +495,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": 7,
    "metadata": {},
    "outputs": [
     {
@@ -562,14 +569,14 @@
        ".highlight .vg { color: #19177C } /* Name.Variable.Global */\n",
        ".highlight .vi { color: #19177C } /* Name.Variable.Instance */\n",
        ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n",
-       ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
<class 'stix2.v20.sdo.CourseOfAction'>\n",
+       ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
<class 'stix2.v21.sdo.CourseOfAction'>\n",
        "
\n" ], "text/plain": [ "" ] }, - "execution_count": 5, + "execution_count": 7, "metadata": {}, "output_type": "execute_result" }, @@ -646,6 +653,7 @@ ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n", ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "course-of-action",\n",
+       "    "spec_version": "2.1",\n",
        "    "id": "course-of-action--d9727aee-48b8-4fdb-89e2-4c49746ba4dd",\n",
        "    "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5",\n",
        "    "created": "2017-05-31T21:30:41.022Z",\n",
@@ -659,13 +667,13 @@
        ""
       ]
      },
-     "execution_count": 5,
+     "execution_count": 7,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
-    "file_handle = open(\"/tmp/stix2_store/course-of-action/course-of-action--d9727aee-48b8-4fdb-89e2-4c49746ba4dd.json\")\n",
+    "file_handle = open(\"/tmp/stix2_store/course-of-action/course-of-action--d9727aee-48b8-4fdb-89e2-4c49746ba4dd/20170531213041022744.json\")\n",
     "\n",
     "obj = parse(file_handle)\n",
     "print(type(obj))\n",
@@ -731,7 +739,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.6.3"
+   "version": "3.9.0a6"
   }
  },
  "nbformat": 4,
diff --git a/docs/guide/patterns.ipynb b/docs/guide/patterns.ipynb
index ee06675..4dc0175 100644
--- a/docs/guide/patterns.ipynb
+++ b/docs/guide/patterns.ipynb
@@ -1,5 +1,58 @@
 {
  "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {
+    "nbsphinx": "hidden"
+   },
+   "outputs": [],
+   "source": [
+    "# Delete this cell to re-enable tracebacks\n",
+    "import sys\n",
+    "ipython = get_ipython()\n",
+    "\n",
+    "def hide_traceback(exc_tuple=None, filename=None, tb_offset=None,\n",
+    "                   exception_only=False, running_compiled_code=False):\n",
+    "    etype, value, tb = sys.exc_info()\n",
+    "    value.__cause__ = None  # suppress chained exceptions\n",
+    "    return ipython._showtraceback(etype, value, ipython.InteractiveTB.get_exception_only(etype, value))\n",
+    "\n",
+    "ipython.showtraceback = hide_traceback"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {
+    "nbsphinx": "hidden"
+   },
+   "outputs": [],
+   "source": [
+    "# JSON output syntax highlighting\n",
+    "from __future__ import print_function\n",
+    "from pygments import highlight\n",
+    "from pygments.lexers import JsonLexer, TextLexer\n",
+    "from pygments.formatters import HtmlFormatter\n",
+    "from IPython.display import display, HTML\n",
+    "from IPython.core.interactiveshell import InteractiveShell\n",
+    "\n",
+    "InteractiveShell.ast_node_interactivity = \"all\"\n",
+    "\n",
+    "def json_print(inpt):\n",
+    "    string = str(inpt)\n",
+    "    formatter = HtmlFormatter()\n",
+    "    if string[0] == '{':\n",
+    "        lexer = JsonLexer()\n",
+    "    else:\n",
+    "        lexer = TextLexer()\n",
+    "    return HTML('{}'.format(\n",
+    "                formatter.get_style_defs('.highlight'),\n",
+    "                highlight(string, lexer, formatter)))\n",
+    "\n",
+    "globals()['print'] = json_print"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -80,18 +133,172 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 4,
    "metadata": {},
    "outputs": [
     {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "\t[domain-name:value = 'site.of.interest.zaz']\n",
-      "\n",
-      "\t[file:parent_directory_ref.path = 'C:\\\\Windows\\\\System32']\n",
-      "\n"
-     ]
+     "data": {
+      "text/html": [
+       "
\t[domain-name:value = 'site.of.interest.zaz']\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "text/html": [ + "
\t[file:parent_directory_ref.path = 'C:\\\\Windows\\\\System32']\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ @@ -117,12 +324,86 @@ "metadata": {}, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "\t[file:extensions.windows-pebinary-ext.sections[*].entropy > 7.0]\n", - "\n" - ] + "data": { + "text/html": [ + "
\t[file:extensions.'windows-pebinary-ext'.sections[*].entropy > 7.0]\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ @@ -144,12 +425,86 @@ "metadata": {}, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "\t[network-traffic:dst_ref.value ISSUBSET '2001:0db8:dead:beef:0000:0000:0000:0000/64']\n", - "\n" - ] + "data": { + "text/html": [ + "
\t[network-traffic:dst_ref.value ISSUBSET '2001:0db8:dead:beef:0000:0000:0000:0000/64']\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ @@ -167,7 +522,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 7, "metadata": {}, "outputs": [], "source": [ @@ -187,17 +542,91 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 8, "metadata": {}, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "(AND)\n", - "[email-message:sender_ref.value = 'stark@example.com' AND email-message:subject = 'Conference Info']\n", - "\n" - ] + "data": { + "text/html": [ + "
(AND)\n",
+       "[email-message:sender_ref.value = 'stark@example.com' AND email-message:subject = 'Conference Info']\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ @@ -216,17 +645,91 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 9, "metadata": {}, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "(OR)\n", - "[url:value = 'http://example.com/foo' OR url:value = 'http://example.com/bar']\n", - "\n" - ] + "data": { + "text/html": [ + "
(OR)\n",
+       "[url:value = 'http://example.com/foo' OR url:value = 'http://example.com/bar']\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ @@ -245,17 +748,91 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 10, "metadata": {}, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "(OR,AND)\n", - "[(file:name = 'pdf.exe' OR file:size = 371712) AND file:created = 2014-01-13 07:03:17+00:00]\n", - "\n" - ] + "data": { + "text/html": [ + "
(OR,AND)\n",
+       "[(file:name = 'pdf.exe' OR file:size = 371712) AND file:created = 2014-01-13 07:03:17+00:00]\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ @@ -277,17 +854,91 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 11, "metadata": {}, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "(AND,OR,OR)\n", - "([file:name = 'foo.dll'] AND [win-registry-key:key = 'HKEY_LOCAL_MACHINE\\\\foo\\\\bar']) OR [process:name = 'fooproc' OR process:name = 'procfoo']\n", - "\n" - ] + "data": { + "text/html": [ + "
(AND,OR,OR)\n",
+       "([file:name = 'foo.dll'] AND [win-registry-key:key = 'HKEY_LOCAL_MACHINE\\\\foo\\\\bar']) OR [process:name = 'fooproc' OR process:name = 'procfoo']\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ @@ -312,17 +963,91 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 12, "metadata": {}, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "(FollowedBy)\n", - "[file:hashes.MD5 = '79054025255fb1a26e4bc422aef54eb4'] FOLLOWEDBY [win-registry-key:key = 'HKEY_LOCAL_MACHINE\\\\foo\\\\bar']\n", - "\n" - ] + "data": { + "text/html": [ + "
(FollowedBy)\n",
+       "[file:hashes.MD5 = '79054025255fb1a26e4bc422aef54eb4'] FOLLOWEDBY [win-registry-key:key = 'HKEY_LOCAL_MACHINE\\\\foo\\\\bar']\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ @@ -341,7 +1066,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 13, "metadata": {}, "outputs": [], "source": [ @@ -360,17 +1085,91 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 14, "metadata": {}, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "(WITHIN)\n", - "([file:hashes.MD5 = '79054025255fb1a26e4bc422aef54eb4'] FOLLOWEDBY [win-registry-key:key = 'HKEY_LOCAL_MACHINE\\\\foo\\\\bar']) WITHIN 300 SECONDS\n", - "\n" - ] + "data": { + "text/html": [ + "
(WITHIN)\n",
+       "([file:hashes.MD5 = '79054025255fb1a26e4bc422aef54eb4'] FOLLOWEDBY [win-registry-key:key = 'HKEY_LOCAL_MACHINE\\\\foo\\\\bar']) WITHIN 300 SECONDS\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ @@ -391,17 +1190,91 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 15, "metadata": {}, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "(REPEAT, WITHIN)\n", - "[network-traffic:dst_ref.type = 'domain-name' AND network-traffic:dst_ref.value = 'example.com'] REPEATS 5 TIMES WITHIN 180 SECONDS\n", - "\n" - ] + "data": { + "text/html": [ + "
(REPEAT, WITHIN)\n",
+       "[network-traffic:dst_ref.type = 'domain-name' AND network-traffic:dst_ref.value = 'example.com'] REPEATS 5 TIMES WITHIN 180 SECONDS\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ @@ -421,17 +1294,91 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 16, "metadata": {}, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "(START-STOP)\n", - "[file:name = 'foo.dll'] START t'2016-06-01T00:00:00Z' STOP t'2016-07-01T00:00:00Z'\n", - "\n" - ] + "data": { + "text/html": [ + "
(START-STOP)\n",
+       "[file:name = 'foo.dll'] START t'2016-06-01T00:00:00Z' STOP t'2016-07-01T00:00:00Z'\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ @@ -453,34 +1400,108 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 17, "metadata": {}, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--219bc5fc-fdbf-4b54-a2fc-921be7ab3acb\",\n", - " \"created\": \"2018-08-29T23:58:00.548Z\",\n", - " \"modified\": \"2018-08-29T23:58:00.548Z\",\n", - " \"name\": \"Cryptotorch\",\n", - " \"pattern\": \"[file:name = '$$t00rzch$$.elf']\",\n", - " \"valid_from\": \"2018-08-29T23:58:00.548391Z\",\n", - " \"labels\": [\n", - " \"malware\",\n", - " \"ransomware\"\n", - " ]\n", - "}\n" - ] + "data": { + "text/html": [ + "
{\n",
+       "    "type": "indicator",\n",
+       "    "spec_version": "2.1",\n",
+       "    "id": "indicator--30dbe788-c10c-4905-b2f0-8f738c847f4b",\n",
+       "    "created": "2020-06-26T18:45:26.0832Z",\n",
+       "    "modified": "2020-06-26T18:45:26.0832Z",\n",
+       "    "name": "Cryptotorch",\n",
+       "    "pattern": "[file:name = '$$t00rzch$$.elf']",\n",
+       "    "pattern_type": "stix",\n",
+       "    "pattern_version": "2.1",\n",
+       "    "valid_from": "2020-06-26T18:45:26.0832Z"\n",
+       "}\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ "from stix2 import Indicator, EqualityComparisonExpression, ObservationExpression\n", "\n", "ece14 = ObservationExpression(EqualityComparisonExpression(ObjectPath(\"file\", [\"name\"]), \"$$t00rzch$$.elf\"))\n", - "ind = Indicator(name=\"Cryptotorch\", labels=[\"malware\", \"ransomware\"], pattern=ece14)\n", + "ind = Indicator(name=\"Cryptotorch\", pattern_type=\"stix\", pattern=ece14)\n", "print(ind)" ] } @@ -501,7 +1522,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.3" + "version": "3.9.0a6" } }, "nbformat": 4, diff --git a/docs/guide/serializing.ipynb b/docs/guide/serializing.ipynb index e58302e..9e7e268 100644 --- a/docs/guide/serializing.ipynb +++ b/docs/guide/serializing.ipynb @@ -15,6 +15,7 @@ "def hide_traceback(exc_tuple=None, filename=None, tb_offset=None,\n", " exception_only=False, running_compiled_code=False):\n", " etype, value, tb = sys.exc_info()\n", + " value.__cause__ = None # suppress chained exceptions\n", " return ipython._showtraceback(etype, value, ipython.InteractiveTB.get_exception_only(etype, value))\n", "\n", "ipython.showtraceback = hide_traceback" @@ -144,15 +145,15 @@ ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n", ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "indicator",\n",
-       "    "id": "indicator--4336ace8-d985-413a-8e32-f749ba268dc3",\n",
-       "    "created": "2018-04-05T20:01:20.012Z",\n",
-       "    "modified": "2018-04-05T20:01:20.012Z",\n",
+       "    "spec_version": "2.1",\n",
+       "    "id": "indicator--5e515461-93ad-41a8-a540-4f9d1a098939",\n",
+       "    "created": "2020-06-26T18:47:20.215931Z",\n",
+       "    "modified": "2020-06-26T18:47:20.215931Z",\n",
        "    "name": "File hash for malware variant",\n",
        "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
-       "    "valid_from": "2018-04-05T20:01:20.012209Z",\n",
-       "    "labels": [\n",
-       "        "malicious-activity"\n",
-       "    ]\n",
+       "    "pattern_type": "stix",\n",
+       "    "pattern_version": "2.1",\n",
+       "    "valid_from": "2020-06-26T18:47:20.215931Z"\n",
        "}\n",
        "
\n" ], @@ -169,7 +170,7 @@ "from stix2 import Indicator\n", "\n", "indicator = Indicator(name=\"File hash for malware variant\",\n", - " labels=[\"malicious-activity\"],\n", + " pattern_type=\"stix\",\n", " pattern=\"[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']\")\n", "\n", "print(str(indicator))" @@ -258,7 +259,7 @@ ".highlight .vg { color: #19177C } /* Name.Variable.Global */\n", ".highlight .vi { color: #19177C } /* Name.Variable.Instance */\n", ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n", - ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{"name": "File hash for malware variant", "labels": ["malicious-activity"], "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']", "type": "indicator", "id": "indicator--4336ace8-d985-413a-8e32-f749ba268dc3", "created": "2018-04-05T20:01:20.012Z", "modified": "2018-04-05T20:01:20.012Z", "valid_from": "2018-04-05T20:01:20.012209Z"}\n",
+       ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{"name": "File hash for malware variant", "pattern_type": "stix", "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']", "pattern_version": "2.1", "type": "indicator", "spec_version": "2.1", "id": "indicator--5e515461-93ad-41a8-a540-4f9d1a098939", "created": "2020-06-26T18:47:20.215931Z", "modified": "2020-06-26T18:47:20.215931Z", "valid_from": "2020-06-26T18:47:20.215931Z"}\n",
        "
\n" ], "text/plain": [ @@ -359,15 +360,15 @@ ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n", ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "name": "File hash for malware variant",\n",
-       "    "labels": [\n",
-       "        "malicious-activity"\n",
-       "    ],\n",
+       "    "pattern_type": "stix",\n",
        "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
+       "    "pattern_version": "2.1",\n",
        "    "type": "indicator",\n",
-       "    "id": "indicator--4336ace8-d985-413a-8e32-f749ba268dc3",\n",
-       "    "created": "2018-04-05T20:01:20.012Z",\n",
-       "    "modified": "2018-04-05T20:01:20.012Z",\n",
-       "    "valid_from": "2018-04-05T20:01:20.012209Z"\n",
+       "    "spec_version": "2.1",\n",
+       "    "id": "indicator--5e515461-93ad-41a8-a540-4f9d1a098939",\n",
+       "    "created": "2020-06-26T18:47:20.215931Z",\n",
+       "    "modified": "2020-06-26T18:47:20.215931Z",\n",
+       "    "valid_from": "2020-06-26T18:47:20.215931Z"\n",
        "}\n",
        "
\n" ], @@ -408,7 +409,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.3" + "version": "3.9.0a6" } }, "nbformat": 4, diff --git a/docs/guide/taxii.ipynb b/docs/guide/taxii.ipynb index 8ec8307..2ec5c24 100644 --- a/docs/guide/taxii.ipynb +++ b/docs/guide/taxii.ipynb @@ -4,7 +4,6 @@ "cell_type": "code", "execution_count": 1, "metadata": { - "collapsed": true, "nbsphinx": "hidden" }, "outputs": [], @@ -16,16 +15,16 @@ "def hide_traceback(exc_tuple=None, filename=None, tb_offset=None,\n", " exception_only=False, running_compiled_code=False):\n", " etype, value, tb = sys.exc_info()\n", + " value.__cause__ = None # suppress chained exceptions\n", " return ipython._showtraceback(etype, value, ipython.InteractiveTB.get_exception_only(etype, value))\n", "\n", - "ipython.showtraceback = hide_traceback" + "#ipython.showtraceback = hide_traceback" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { - "collapsed": true, "nbsphinx": "hidden" }, "outputs": [], @@ -73,38 +72,475 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 3, "metadata": {}, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "{\n", - " \"type\": \"malware\",\n", - " \"id\": \"malware--fdd60b30-b67c-41e3-b0b9-f01faf20d111\",\n", - " \"created\": \"2017-01-27T13:49:53.997Z\",\n", - " \"modified\": \"2017-01-27T13:49:53.997Z\",\n", - " \"name\": \"Poison Ivy\",\n", - " \"description\": \"Poison Ivy\",\n", - " \"labels\": [\n", - " \"remote-access-trojan\"\n", - " ]\n", - "}\n", - "-------\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--a932fcc6-e032-476c-826f-cb970a5a1ade\",\n", - " \"created\": \"2014-05-08T09:00:00.000Z\",\n", - " \"modified\": \"2014-05-08T09:00:00.000Z\",\n", - " \"name\": \"File hash for Poison Ivy variant\",\n", - " \"pattern\": \"[file:hashes.'SHA-256' = 'ef537f25c895bfa782526529a9b63d97aa631564d5d789c2b765448c8635fb6c']\",\n", - " \"valid_from\": \"2014-05-08T09:00:00Z\",\n", - " \"labels\": [\n", - " \"file-hash-watchlist\"\n", - " ]\n", - "}\n" - ] + "data": { + "text/html": [ + "
{\n",
+       "    "type": "malware",\n",
+       "    "spec_version": "2.1",\n",
+       "    "id": "malware--c0931cc6-c75e-47e5-9036-78fabc95d4ec",\n",
+       "    "created": "2017-01-27T13:49:53.997Z",\n",
+       "    "modified": "2017-01-27T13:49:53.997Z",\n",
+       "    "name": "Poison Ivy",\n",
+       "    "description": "Poison Ivy",\n",
+       "    "malware_types": [\n",
+       "        "remote-access-trojan"\n",
+       "    ],\n",
+       "    "is_family": true\n",
+       "}\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "text/html": [ + "
-------\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "text/html": [ + "
{\n",
+       "    "type": "indicator",\n",
+       "    "spec_version": "2.1",\n",
+       "    "id": "indicator--6770298f-0fd8-471a-ab8c-1c658a46574e",\n",
+       "    "created": "2016-11-03T12:30:59.000Z",\n",
+       "    "modified": "2016-11-03T12:30:59.000Z",\n",
+       "    "name": "Malicious site hosting downloader",\n",
+       "    "description": "Accessing this url will infect your machine with malware.",\n",
+       "    "indicator_types": [\n",
+       "        "url-watchlist"\n",
+       "    ],\n",
+       "    "pattern": "[url:value = 'http://z4z10farb.cn/4712']",\n",
+       "    "pattern_type": "stix",\n",
+       "    "pattern_version": "2.1",\n",
+       "    "valid_from": "2017-01-27T13:49:53.935382Z"\n",
+       "}\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "text/html": [ + "
{\n",
+       "    "type": "indicator",\n",
+       "    "spec_version": "2.1",\n",
+       "    "id": "indicator--6770298f-0fd8-471a-ab8c-1c658a46574e",\n",
+       "    "created": "2016-11-03T12:30:59.000Z",\n",
+       "    "modified": "2016-12-25T12:30:59.444Z",\n",
+       "    "name": "Malicious site hosting downloader",\n",
+       "    "description": "Accessing this url will infect your machine with malware. Updated indicator",\n",
+       "    "indicator_types": [\n",
+       "        "url-watchlist"\n",
+       "    ],\n",
+       "    "pattern": "[url:value = 'http://x4z9arb.cn/4712']",\n",
+       "    "pattern_type": "stix",\n",
+       "    "pattern_version": "2.1",\n",
+       "    "valid_from": "2017-01-27T13:49:53.935382Z"\n",
+       "}\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "text/html": [ + "
{\n",
+       "    "type": "indicator",\n",
+       "    "spec_version": "2.1",\n",
+       "    "id": "indicator--6770298f-0fd8-471a-ab8c-1c658a46574e",\n",
+       "    "created": "2016-11-03T12:30:59.000Z",\n",
+       "    "modified": "2017-01-27T13:49:53.935Z",\n",
+       "    "name": "Malicious site hosting downloader",\n",
+       "    "description": "Accessing this url will infect your machine with malware. This is the last updated indicator",\n",
+       "    "indicator_types": [\n",
+       "        "url-watchlist"\n",
+       "    ],\n",
+       "    "pattern": "[url:value = 'http://x4z9arb.cn/4712']",\n",
+       "    "pattern_type": "stix",\n",
+       "    "pattern_version": "2.1",\n",
+       "    "valid_from": "2016-11-03T12:30:59Z"\n",
+       "}\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ @@ -117,8 +553,8 @@ "tc_source = TAXIICollectionSource(collection)\n", "\n", "#retrieve STIX objects by id\n", - "stix_obj = tc_source.get(\"malware--fdd60b30-b67c-41e3-b0b9-f01faf20d111\")\n", - "stix_obj_versions = tc_source.all_versions(\"indicator--a932fcc6-e032-476c-826f-cb970a5a1ade\")\n", + "stix_obj = tc_source.get(\"malware--c0931cc6-c75e-47e5-9036-78fabc95d4ec\")\n", + "stix_obj_versions = tc_source.all_versions(\"indicator--6770298f-0fd8-471a-ab8c-1c658a46574e\")\n", "\n", "#for visual purposes\n", "print(stix_obj)\n", @@ -129,26 +565,387 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 4, "metadata": {}, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--a932fcc6-e032-476c-826f-cb970a5a1ade\",\n", - " \"created\": \"2014-05-08T09:00:00.000Z\",\n", - " \"modified\": \"2014-05-08T09:00:00.000Z\",\n", - " \"name\": \"File hash for Poison Ivy variant\",\n", - " \"pattern\": \"[file:hashes.'SHA-256' = 'ef537f25c895bfa782526529a9b63d97aa631564d5d789c2b765448c8635fb6c']\",\n", - " \"valid_from\": \"2014-05-08T09:00:00Z\",\n", - " \"labels\": [\n", - " \"file-hash-watchlist\"\n", - " ]\n", - "}\n" - ] + "data": { + "text/html": [ + "
{\n",
+       "    "type": "indicator",\n",
+       "    "spec_version": "2.1",\n",
+       "    "id": "indicator--cd981c25-8042-4166-8945-51178443bdac",\n",
+       "    "created": "2014-05-08T09:00:00.000Z",\n",
+       "    "modified": "2014-05-08T09:00:00.000Z",\n",
+       "    "name": "File hash for Poison Ivy variant",\n",
+       "    "indicator_types": [\n",
+       "        "file-hash-watchlist"\n",
+       "    ],\n",
+       "    "pattern": "[file:hashes.'SHA-256' = 'ef537f25c895bfa782526529a9b63d97aa631564d5d789c2b765448c8635fb6c']",\n",
+       "    "pattern_type": "stix",\n",
+       "    "pattern_version": "2.1",\n",
+       "    "valid_from": "2014-05-08T09:00:00Z"\n",
+       "}\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "text/html": [ + "
{\n",
+       "    "type": "indicator",\n",
+       "    "spec_version": "2.1",\n",
+       "    "id": "indicator--6770298f-0fd8-471a-ab8c-1c658a46574e",\n",
+       "    "created": "2016-11-03T12:30:59.000Z",\n",
+       "    "modified": "2017-01-27T13:49:53.935Z",\n",
+       "    "name": "Malicious site hosting downloader",\n",
+       "    "description": "Accessing this url will infect your machine with malware. This is the last updated indicator",\n",
+       "    "indicator_types": [\n",
+       "        "url-watchlist"\n",
+       "    ],\n",
+       "    "pattern": "[url:value = 'http://x4z9arb.cn/4712']",\n",
+       "    "pattern_type": "stix",\n",
+       "    "pattern_version": "2.1",\n",
+       "    "valid_from": "2016-11-03T12:30:59Z"\n",
+       "}\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "text/html": [ + "
{\n",
+       "    "type": "indicator",\n",
+       "    "spec_version": "2.1",\n",
+       "    "id": "indicator--d8f573d9-5796-4d3f-98fd-d3b953738520",\n",
+       "    "created": "2020-06-26T19:04:05.608201Z",\n",
+       "    "modified": "2020-06-26T19:04:05.608201Z",\n",
+       "    "description": "Smokey Bear implant",\n",
+       "    "pattern": "[file:hashes.'SHA-256' = '09c7e05a39a59428743635242e4a867c932140a909f12a1e54fa7ee6a440c73b']",\n",
+       "    "pattern_type": "stix",\n",
+       "    "pattern_version": "2.1",\n",
+       "    "valid_from": "2020-06-26T19:04:05.608201Z"\n",
+       "}\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "text/html": [ + "
{\n",
+       "    "type": "indicator",\n",
+       "    "spec_version": "2.1",\n",
+       "    "id": "indicator--acd03fef-b8df-4c45-bd89-1ede90c39959",\n",
+       "    "created": "2020-06-26T19:04:09.476525Z",\n",
+       "    "modified": "2020-06-26T19:04:09.476525Z",\n",
+       "    "description": "Smokey Bear implant",\n",
+       "    "pattern": "[file:hashes.'SHA-256' = '09c7e05a39a59428743635242e4a867c932140a909f12a1e54fa7ee6a440c73b']",\n",
+       "    "pattern_type": "stix",\n",
+       "    "pattern_version": "2.1",\n",
+       "    "valid_from": "2020-06-26T19:04:09.476525Z"\n",
+       "}\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ @@ -176,10 +973,8 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": true - }, + "execution_count": 5, + "metadata": {}, "outputs": [], "source": [ "from stix2 import TAXIICollectionSink, ThreatActor\n", @@ -189,7 +984,7 @@ "\n", "# create new STIX threat-actor\n", "ta = ThreatActor(name=\"Teddy Bear\",\n", - " labels=[\"nation-state\"],\n", + " threat_actor_types=[\"nation-state\"],\n", " sophistication=\"innovator\",\n", " resource_level=\"government\",\n", " goals=[\n", @@ -197,9 +992,7 @@ " \"water-hole attacks geared towards energy sector\",\n", " ])\n", "\n", - "tc_sink.add(ta)\n", - "\n", - "\n" + "tc_sink.add(ta)" ] }, { @@ -211,25 +1004,102 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 6, "metadata": {}, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "{\n", - " \"type\": \"malware\",\n", - " \"id\": \"malware--fdd60b30-b67c-41e3-b0b9-f01faf20d111\",\n", - " \"created\": \"2017-01-27T13:49:53.997Z\",\n", - " \"modified\": \"2017-01-27T13:49:53.997Z\",\n", - " \"name\": \"Poison Ivy\",\n", - " \"description\": \"Poison Ivy\",\n", - " \"labels\": [\n", - " \"remote-access-trojan\"\n", - " ]\n", - "}\n" - ] + "data": { + "text/html": [ + "
{\n",
+       "    "type": "malware",\n",
+       "    "spec_version": "2.1",\n",
+       "    "id": "malware--c0931cc6-c75e-47e5-9036-78fabc95d4ec",\n",
+       "    "created": "2017-01-27T13:49:53.997Z",\n",
+       "    "modified": "2017-01-27T13:49:53.997Z",\n",
+       "    "name": "Poison Ivy",\n",
+       "    "description": "Poison Ivy",\n",
+       "    "malware_types": [\n",
+       "        "remote-access-trojan"\n",
+       "    ],\n",
+       "    "is_family": true\n",
+       "}\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ @@ -241,27 +1111,25 @@ "\n", "# retrieve STIX object by id from TAXII Collection through\n", "# TAXIICollectionStore\n", - "stix_obj2 = tc_source.get(\"malware--fdd60b30-b67c-41e3-b0b9-f01faf20d111\")\n", + "stix_obj2 = tc_source.get(\"malware--c0931cc6-c75e-47e5-9036-78fabc95d4ec\")\n", "\n", "print(stix_obj2)" ] }, { "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": true - }, + "execution_count": 7, + "metadata": {}, "outputs": [], "source": [ - "from stix2 import indicator\n", + "from stix2 import Indicator\n", "\n", "# add STIX object to TAXIICollectionStore\n", "ind = Indicator(description=\"Smokey Bear implant\",\n", - " labels=[\"malicious-activity\"],\n", + " pattern_type=\"stix\",\n", " pattern=\"[file:hashes.'SHA-256' = '09c7e05a39a59428743635242e4a867c932140a909f12a1e54fa7ee6a440c73b']\")\n", "\n", - "tc_store.add(ind)\n" + "tc_store.add(ind)" ] }, { @@ -321,7 +1189,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.3" + "version": "3.9.0a6" } }, "nbformat": 4, diff --git a/docs/guide/ts_support.ipynb b/docs/guide/ts_support.ipynb index 2d36f5a..7aa3fea 100644 --- a/docs/guide/ts_support.ipynb +++ b/docs/guide/ts_support.ipynb @@ -4,7 +4,6 @@ "cell_type": "code", "execution_count": 1, "metadata": { - "collapsed": true, "nbsphinx": "hidden" }, "outputs": [], @@ -16,6 +15,7 @@ "def hide_traceback(exc_tuple=None, filename=None, tb_offset=None,\n", " exception_only=False, running_compiled_code=False):\n", " etype, value, tb = sys.exc_info()\n", + " value.__cause__ = None # suppress chained exceptions\n", " return ipython._showtraceback(etype, value, ipython.InteractiveTB.get_exception_only(etype, value))\n", "\n", "ipython.showtraceback = hide_traceback" @@ -23,7 +23,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 2, "metadata": { "nbsphinx": "hidden" }, @@ -237,7 +237,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 3, "metadata": {}, "outputs": [ { @@ -329,7 +329,7 @@ "" ] }, - "execution_count": 2, + "execution_count": 3, "metadata": {}, "output_type": "execute_result" } @@ -365,31 +365,29 @@ "source": [ "### How custom content works\n", "\n", - "[CustomObject](../api/v20/stix2.v20.sdo.rst#stix2.v20.sdo.CustomObject), [CustomObservable](../api/v20/stix2.v20.observables.rst#stix2.v20.observables.CustomObservable), [CustomMarking](../api/v20/stix2.v20.common.rst#stix2.v20.common.CustomMarking) and [CustomExtension](../api/v20/stix2.v20.observables.rst#stix2.v20.observables.CustomExtension) must be registered explicitly by STIX version. This is a design decision since properties or requirements may change as the STIX Technical Specification advances.\n", + "[CustomObject](../api/v21/stix2.v21.sdo.rst#stix2.v21.sdo.CustomObject), [CustomObservable](../api/v21/stix2.v21.observables.rst#stix2.v21.observables.CustomObservable), [CustomMarking](../api/v21/stix2.v21.common.rst#stix2.v21.common.CustomMarking) and [CustomExtension](../api/v21/stix2.v21.observables.rst#stix2.v21.observables.CustomExtension) must be registered explicitly by STIX version. This is a design decision since properties or requirements may change as the STIX Technical Specification advances.\n", "\n", "You can perform this by:" ] }, { "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": true - }, + "execution_count": 4, + "metadata": {}, "outputs": [], "source": [ "import stix2\n", "\n", "# Make my custom observable available in STIX 2.0\n", "@stix2.v20.CustomObservable('x-new-object-type',\n", - " ((\"prop\", stix2.properties.BooleanProperty())))\n", + " [(\"prop\", stix2.properties.BooleanProperty())])\n", "class NewObject2(object):\n", " pass\n", "\n", "\n", "# Make my custom observable available in STIX 2.1\n", "@stix2.v21.CustomObservable('x-new-object-type',\n", - " ((\"prop\", stix2.properties.BooleanProperty())))\n", + " [(\"prop\", stix2.properties.BooleanProperty())])\n", "class NewObject2(object):\n", " pass" ] @@ -411,7 +409,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.3" + "version": "3.9.0a6" } }, "nbformat": 4, diff --git a/docs/guide/versioning.ipynb b/docs/guide/versioning.ipynb index eb0708a..45686bf 100644 --- a/docs/guide/versioning.ipynb +++ b/docs/guide/versioning.ipynb @@ -15,6 +15,7 @@ "def hide_traceback(exc_tuple=None, filename=None, tb_offset=None,\n", " exception_only=False, running_compiled_code=False):\n", " etype, value, tb = sys.exc_info()\n", + " value.__cause__ = None # suppress chained exceptions\n", " return ipython._showtraceback(etype, value, ipython.InteractiveTB.get_exception_only(etype, value))\n", "\n", "ipython.showtraceback = hide_traceback" @@ -63,12 +64,12 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "To create a new version of an existing object, specify the property(ies) you want to change and their new values. For example, here we change the label from \"anomalous-activity\" to \"malicious-activity\":" + "To create a new version of an existing object, specify the property(ies) you want to change and their new values. For example, here we change the indicator type from \"anomalous-activity\" to \"malicious-activity\":" ] }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 4, "metadata": {}, "outputs": [ { @@ -144,13 +145,19 @@ ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n", ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "indicator",\n",
-       "    "id": "indicator--8ad18fc7-457c-475d-b292-1ec44febe0fd",\n",
+       "    "spec_version": "2.1",\n",
+       "    "id": "indicator--6a7f1c8a-3c9a-471f-8ef0-e95e51457c3f",\n",
        "    "created": "2016-01-01T08:00:00.000Z",\n",
-       "    "modified": "2019-07-25T17:59:34.815Z",\n",
+       "    "modified": "2020-06-26T19:27:20.792845Z",\n",
        "    "name": "File hash for Foobar malware",\n",
        "    "description": "A file indicator",\n",
+       "    "indicator_types": [\n",
+       "        "anomalous-activity"\n",
+       "    ],\n",
        "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
-       "    "valid_from": "2019-07-25T17:59:34.779826Z",\n",
+       "    "pattern_type": "stix",\n",
+       "    "pattern_version": "2.1",\n",
+       "    "valid_from": "2020-06-26T19:27:20.759788Z",\n",
        "    "labels": [\n",
        "        "malicious-activity"\n",
        "    ]\n",
@@ -161,7 +168,7 @@
        ""
       ]
      },
-     "execution_count": 3,
+     "execution_count": 4,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -172,7 +179,8 @@
     "indicator = Indicator(created=\"2016-01-01T08:00:00.000Z\",\n",
     "                      name=\"File hash for suspicious file\",\n",
     "                      description=\"A file indicator\",\n",
-    "                      labels=[\"anomalous-activity\"],\n",
+    "                      indicator_types=[\"anomalous-activity\"],\n",
+    "                      pattern_type=\"stix\",\n",
     "                      pattern=\"[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']\")\n",
     "\n",
     "indicator2 = indicator.new_version(name=\"File hash for Foobar malware\",\n",
@@ -189,7 +197,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 5,
    "metadata": {
     "scrolled": true
    },
@@ -214,119 +222,6 @@
     "You can remove optional or custom properties by setting them to `None` when you call `new_version()`."
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": 5,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/html": [
-       "
{\n",
-       "    "type": "indicator",\n",
-       "    "id": "indicator--8ad18fc7-457c-475d-b292-1ec44febe0fd",\n",
-       "    "created": "2016-01-01T08:00:00.000Z",\n",
-       "    "modified": "2019-07-25T17:59:42.648Z",\n",
-       "    "name": "File hash for suspicious file",\n",
-       "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
-       "    "valid_from": "2019-07-25T17:59:34.779826Z",\n",
-       "    "labels": [\n",
-       "        "anomalous-activity"\n",
-       "    ]\n",
-       "}\n",
-       "
\n" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "indicator3 = indicator.new_version(description=None)\n", - "print(indicator3)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "collapsed": true - }, - "source": [ - "To revoke an object:" - ] - }, { "cell_type": "code", "execution_count": 6, @@ -405,16 +300,18 @@ ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n", ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "indicator",\n",
-       "    "id": "indicator--8ad18fc7-457c-475d-b292-1ec44febe0fd",\n",
+       "    "spec_version": "2.1",\n",
+       "    "id": "indicator--6a7f1c8a-3c9a-471f-8ef0-e95e51457c3f",\n",
        "    "created": "2016-01-01T08:00:00.000Z",\n",
-       "    "modified": "2019-07-25T17:59:52.198Z",\n",
+       "    "modified": "2020-06-26T19:29:37.055139Z",\n",
        "    "name": "File hash for suspicious file",\n",
-       "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
-       "    "valid_from": "2019-07-25T17:59:34.779826Z",\n",
-       "    "revoked": true,\n",
-       "    "labels": [\n",
+       "    "indicator_types": [\n",
        "        "anomalous-activity"\n",
-       "    ]\n",
+       "    ],\n",
+       "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
+       "    "pattern_type": "stix",\n",
+       "    "pattern_version": "2.1",\n",
+       "    "valid_from": "2020-06-26T19:27:20.759788Z"\n",
        "}\n",
        "
\n" ], @@ -427,6 +324,123 @@ "output_type": "execute_result" } ], + "source": [ + "indicator3 = indicator.new_version(description=None)\n", + "print(indicator3)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "collapsed": true + }, + "source": [ + "To revoke an object:" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
{\n",
+       "    "type": "indicator",\n",
+       "    "spec_version": "2.1",\n",
+       "    "id": "indicator--6a7f1c8a-3c9a-471f-8ef0-e95e51457c3f",\n",
+       "    "created": "2016-01-01T08:00:00.000Z",\n",
+       "    "modified": "2020-06-26T19:29:38.943037Z",\n",
+       "    "name": "File hash for suspicious file",\n",
+       "    "indicator_types": [\n",
+       "        "anomalous-activity"\n",
+       "    ],\n",
+       "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
+       "    "pattern_type": "stix",\n",
+       "    "pattern_version": "2.1",\n",
+       "    "valid_from": "2020-06-26T19:27:20.759788Z",\n",
+       "    "revoked": true\n",
+       "}\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "indicator4 = indicator3.revoke()\n", "print(indicator4)" @@ -449,7 +463,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.3" + "version": "3.9.0a6" } }, "nbformat": 4, diff --git a/docs/guide/workbench.ipynb b/docs/guide/workbench.ipynb index de243cc..3cce593 100644 --- a/docs/guide/workbench.ipynb +++ b/docs/guide/workbench.ipynb @@ -15,6 +15,7 @@ "def hide_traceback(exc_tuple=None, filename=None, tb_offset=None,\n", " exception_only=False, running_compiled_code=False):\n", " etype, value, tb = sys.exc_info()\n", + " value.__cause__ = None # suppress chained exceptions\n", " return ipython._showtraceback(etype, value, ipython.InteractiveTB.get_exception_only(etype, value))\n", "\n", "ipython.showtraceback = hide_traceback" @@ -218,7 +219,7 @@ ".highlight .vg { color: #19177C } /* Name.Variable.Global */\n", ".highlight .vi { color: #19177C } /* Name.Variable.Instance */\n", ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n", - ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
indicator--a932fcc6-e032-476c-826f-cb970a5a1ade\n",
+       ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
indicator--cd981c25-8042-4166-8945-51178443bdac\n",
        "
\n" ], "text/plain": [ @@ -382,7 +383,7 @@ ".highlight .vg { color: #19177C } /* Name.Variable.Global */\n", ".highlight .vi { color: #19177C } /* Name.Variable.Instance */\n", ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n", - ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
malware--fdd60b30-b67c-41e3-b0b9-f01faf20d111\n",
+       ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
malware--c0931cc6-c75e-47e5-9036-78fabc95d4ec\n",
        "
\n" ], "text/plain": [ @@ -480,14 +481,16 @@ ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n", ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "malware",\n",
-       "    "id": "malware--fdd60b30-b67c-41e3-b0b9-f01faf20d111",\n",
+       "    "spec_version": "2.1",\n",
+       "    "id": "malware--c0931cc6-c75e-47e5-9036-78fabc95d4ec",\n",
        "    "created": "2017-01-27T13:49:53.997Z",\n",
        "    "modified": "2017-01-27T13:49:53.997Z",\n",
        "    "name": "Poison Ivy",\n",
        "    "description": "Poison Ivy",\n",
-       "    "labels": [\n",
+       "    "malware_types": [\n",
        "        "remote-access-trojan"\n",
-       "    ]\n",
+       "    ],\n",
+       "    "is_family": true\n",
        "}\n",
        "
\n" ], @@ -515,7 +518,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 10, "metadata": {}, "outputs": [ { @@ -591,15 +594,18 @@ ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n", ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "indicator",\n",
-       "    "id": "indicator--a932fcc6-e032-476c-826f-cb970a5a1ade",\n",
+       "    "spec_version": "2.1",\n",
+       "    "id": "indicator--cd981c25-8042-4166-8945-51178443bdac",\n",
        "    "created": "2014-05-08T09:00:00.000Z",\n",
        "    "modified": "2014-05-08T09:00:00.000Z",\n",
        "    "name": "File hash for Poison Ivy variant",\n",
-       "    "pattern": "[file:hashes.'SHA-256' = 'ef537f25c895bfa782526529a9b63d97aa631564d5d789c2b765448c8635fb6c']",\n",
-       "    "valid_from": "2014-05-08T09:00:00Z",\n",
-       "    "labels": [\n",
+       "    "indicator_types": [\n",
        "        "file-hash-watchlist"\n",
-       "    ]\n",
+       "    ],\n",
+       "    "pattern": "[file:hashes.'SHA-256' = 'ef537f25c895bfa782526529a9b63d97aa631564d5d789c2b765448c8635fb6c']",\n",
+       "    "pattern_type": "stix",\n",
+       "    "pattern_version": "2.1",\n",
+       "    "valid_from": "2014-05-08T09:00:00Z"\n",
        "}\n",
        "
\n" ], @@ -607,13 +613,13 @@ "" ] }, - "execution_count": 9, + "execution_count": 10, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "malware = get('malware--fdd60b30-b67c-41e3-b0b9-f01faf20d111')\n", + "malware = get('malware--c0931cc6-c75e-47e5-9036-78fabc95d4ec')\n", "indicator = malware.related(filters=Filter('type', '=', 'indicator'))\n", "print(indicator[0])" ] @@ -629,7 +635,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 11, "metadata": {}, "outputs": [], "source": [ @@ -646,7 +652,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 12, "metadata": {}, "outputs": [], "source": [ @@ -662,7 +668,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 14, "metadata": {}, "outputs": [ { @@ -743,13 +749,13 @@ "" ] }, - "execution_count": 12, + "execution_count": 14, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "indicator = Indicator(labels=[\"malicious-activity\"], pattern=\"[file:hashes.MD5 = 'd41d8cd98f00b204e9800998ecf8427e']\")\n", + "indicator = Indicator(pattern_type=\"stix\", pattern=\"[file:hashes.MD5 = 'd41d8cd98f00b204e9800998ecf8427e']\")\n", "save(indicator)\n", "\n", "indicator_creator = get(indicator.created_by_ref)\n", @@ -793,7 +799,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.3" + "version": "3.9.0a6" } }, "nbformat": 4, diff --git a/stix2/workbench.py b/stix2/workbench.py index 7b3f46a..3724bdb 100644 --- a/stix2/workbench.py +++ b/stix2/workbench.py @@ -62,7 +62,6 @@ from .datastore.filters import FilterSet # Enable some adaptation to the current default supported STIX version. _STIX_VID = "v" + stix2.DEFAULT_VERSION.replace(".", "") -print(_STIX_VID) # Use an implicit MemoryStore From e25734d2b64d96dce248887e9637ec9ec3040cfa Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Fri, 26 Jun 2020 16:41:53 -0400 Subject: [PATCH 09/30] Fix trailing commas --- stix2/properties.py | 6 ++++-- stix2/test/v21/constants.py | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/stix2/properties.py b/stix2/properties.py index 9da5c29..1ca2dbe 100644 --- a/stix2/properties.py +++ b/stix2/properties.py @@ -170,8 +170,10 @@ class Property(object): self.required = required if required and default: - raise STIXError("Cant't use 'required' and 'default' together. 'required'" - "really means 'the user must provide this.'") + raise STIXError( + "Cant't use 'required' and 'default' together. 'required'" + "really means 'the user must provide this.'", + ) if fixed: self._fixed_value = fixed diff --git a/stix2/test/v21/constants.py b/stix2/test/v21/constants.py index d5bef24..78ee076 100644 --- a/stix2/test/v21/constants.py +++ b/stix2/test/v21/constants.py @@ -131,7 +131,7 @@ MALWARE_ANALYSIS_KWARGS = dict( NOTE_KWARGS = dict( content="Heartbleed", - object_refs=[CAMPAIGN_ID] + object_refs=[CAMPAIGN_ID], ) OBSERVED_DATA_KWARGS = dict( @@ -148,7 +148,7 @@ OBSERVED_DATA_KWARGS = dict( OPINION_KWARGS = dict( opinion="agree", - object_refs=[CAMPAIGN_ID] + object_refs=[CAMPAIGN_ID], ) REPORT_KWARGS = dict( From 9a0cb02ab32dcb3008861fd2275d149f87c9ede0 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Tue, 30 Jun 2020 13:58:19 -0400 Subject: [PATCH 10/30] nit-pick on v21 Indicator constructor --- stix2/v21/sdo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stix2/v21/sdo.py b/stix2/v21/sdo.py index 9418c40..decbf07 100644 --- a/stix2/v21/sdo.py +++ b/stix2/v21/sdo.py @@ -204,7 +204,7 @@ class Indicator(_DomainObject): if kwargs.get('pattern') and kwargs.get('pattern_type') == 'stix' and not kwargs.get('pattern_version'): kwargs['pattern_version'] = '2.1' - super(_DomainObject, self).__init__(*args, **kwargs) + super(Indicator, self).__init__(*args, **kwargs) def _check_object_constraints(self): super(Indicator, self)._check_object_constraints() From e957e8012667d083972b204a10c9ab1e837c6d7b Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Tue, 30 Jun 2020 22:04:40 -0400 Subject: [PATCH 11/30] remove stix2.core.rst --- docs/api/stix2.core.rst | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 docs/api/stix2.core.rst diff --git a/docs/api/stix2.core.rst b/docs/api/stix2.core.rst deleted file mode 100644 index dbd5256..0000000 --- a/docs/api/stix2.core.rst +++ /dev/null @@ -1,5 +0,0 @@ -core -========== - -.. automodule:: stix2.core - :members: From de4ea5b805b1c17ff77da03f129a5de4307da805 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Tue, 30 Jun 2020 22:11:29 -0400 Subject: [PATCH 12/30] edits to parsing.ipynb and workbench.ipynb Create the warning admonitions correctly --- docs/guide/parsing.ipynb | 9 ++++++++- docs/guide/workbench.ipynb | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/docs/guide/parsing.ipynb b/docs/guide/parsing.ipynb index 0762c6c..5346d16 100644 --- a/docs/guide/parsing.ipynb +++ b/docs/guide/parsing.ipynb @@ -691,7 +691,14 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Parsing custom STIX objects and/or STIX objects with custom properties is also completed easily with [parse()](../api/stix2.core.rst#stix2.core.parse). Just supply the keyword argument ``allow_custom=True``. When ``allow_custom`` is specified, [parse()](../api/stix2.core.rst#stix2.core.parse) will attempt to convert the supplied STIX content to known STIX 2 domain objects and/or previously defined [custom STIX 2 objects](custom.ipynb). If the conversion cannot be completed (and ``allow_custom`` is specified), [parse()](../api/stix2.core.rst#stix2.core.parse) will treat the supplied STIX 2 content as valid STIX 2 objects and return them. **Warning: Specifying allow_custom may lead to critical errors if further processing (searching, filtering, modifying etc...) of the custom content occurs where the custom content supplied is not valid STIX 2**. This is an axiomatic possibility as the ``stix2`` library cannot guarantee proper processing of unknown custom STIX 2 objects that were explicitly flagged to be allowed, and thus may not be valid.\n", + "Parsing custom STIX objects and/or STIX objects with custom properties is also completed easily with [parse()](../api/stix2.core.rst#stix2.core.parse). Just supply the keyword argument ``allow_custom=True``. When ``allow_custom`` is specified, [parse()](../api/stix2.core.rst#stix2.core.parse) will attempt to convert the supplied STIX content to known STIX 2 domain objects and/or previously defined [custom STIX 2 objects](custom.ipynb). If the conversion cannot be completed (and ``allow_custom`` is specified), [parse()](../api/stix2.core.rst#stix2.core.parse) will treat the supplied STIX 2 content as valid STIX 2 objects and return them. This is an axiomatic possibility as the ``stix2`` library cannot guarantee proper processing of unknown custom STIX 2 objects that were explicitly flagged to be allowed, and thus may not be valid.\n", + "
\n", + "\n", + "**Warning**\n", + "\n", + "Specifying allow_custom may lead to critical errors if further processing (searching, filtering, modifying etc...) of the custom content occurs where the custom content supplied is not valid STIX 2\n", + "\n", + "
\n", "\n", "For examples of parsing STIX 2 objects with custom STIX properties, see [Custom STIX Content: Custom Properties](custom.ipynb#Custom-Properties)\n", "\n", diff --git a/docs/guide/workbench.ipynb b/docs/guide/workbench.ipynb index 3cce593..c0c4ee0 100644 --- a/docs/guide/workbench.ipynb +++ b/docs/guide/workbench.ipynb @@ -775,7 +775,7 @@ "source": [ "
\n", "\n", - "**Warning:**\n", + "**Warning**\n", "\n", "The workbench layer replaces STIX Object classes with special versions of them that use \"wrappers\" to provide extra functionality. Because of this, we recommend that you **either use the workbench layer or the rest of the library, but not both**. In other words, don't import from both ``stix2.workbench`` and any other submodules of ``stix2``.\n", "\n", From 21e109d8812926eb2f278099cfdfb63abc2c885e Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Tue, 30 Jun 2020 22:16:15 -0400 Subject: [PATCH 13/30] remove `core` and added `parsing`, `versioning` to autosummary --- stix2/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/stix2/__init__.py b/stix2/__init__.py index 10384b1..2f94806 100644 --- a/stix2/__init__.py +++ b/stix2/__init__.py @@ -4,16 +4,17 @@ :toctree: api confidence - core datastore environment exceptions markings + parsing patterns properties utils v20 v21 + versioning workbench """ From 7ef3a02971b562a7fc927df4fb165e00ae12970c Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Tue, 30 Jun 2020 22:32:04 -0400 Subject: [PATCH 14/30] update docstrings environment.py and parsing.py --- stix2/environment.py | 2 ++ stix2/parsing.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/stix2/environment.py b/stix2/environment.py index 6e3c76a..4ba912c 100644 --- a/stix2/environment.py +++ b/stix2/environment.py @@ -200,6 +200,8 @@ class Environment(DataStoreMixin): Args: obj1: A stix2 object instance obj2: A stix2 object instance + prop_scores: A dictionary that can hold individual property scores, + weights, contributing score, matching score and sum of weights. weight_dict: A dictionary that can be used to override settings in the semantic equivalence process diff --git a/stix2/parsing.py b/stix2/parsing.py index baa3b7b..c0c7bf8 100644 --- a/stix2/parsing.py +++ b/stix2/parsing.py @@ -1,4 +1,4 @@ -"""STIX2 Core Objects and Methods.""" +"""STIX2 Core parsing methods.""" import copy import importlib From a9af34b1e3b418a46939142fd557f36b4c0bfcaf Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Tue, 30 Jun 2020 22:34:38 -0400 Subject: [PATCH 15/30] add file stix2.versioning.rst --- docs/api/stix2.versioning.rst | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 docs/api/stix2.versioning.rst diff --git a/docs/api/stix2.versioning.rst b/docs/api/stix2.versioning.rst new file mode 100644 index 0000000..e69de29 From 2f80b82139f1c931cbd7430dd2bdfb7424a1c891 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Tue, 30 Jun 2020 22:35:03 -0400 Subject: [PATCH 16/30] update stix2.versioning.rst --- docs/api/stix2.versioning.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/api/stix2.versioning.rst b/docs/api/stix2.versioning.rst index e69de29..adc2eab 100644 --- a/docs/api/stix2.versioning.rst +++ b/docs/api/stix2.versioning.rst @@ -0,0 +1,5 @@ +versioning +=============== + +.. automodule:: stix2.versioning + :members: From 1518a6d55e3d49d61f2dcbe4e171662364f88c94 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Tue, 30 Jun 2020 22:35:40 -0400 Subject: [PATCH 17/30] add file stix2.parsing.rst --- docs/api/stix2.parsing.rst | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 docs/api/stix2.parsing.rst diff --git a/docs/api/stix2.parsing.rst b/docs/api/stix2.parsing.rst new file mode 100644 index 0000000..e69de29 From 03ca082a50158a2efc1075af3f0eff719c792244 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Tue, 30 Jun 2020 22:39:58 -0400 Subject: [PATCH 18/30] update stix2.parsing.rst --- docs/api/stix2.parsing.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/api/stix2.parsing.rst b/docs/api/stix2.parsing.rst index e69de29..bee3e4e 100644 --- a/docs/api/stix2.parsing.rst +++ b/docs/api/stix2.parsing.rst @@ -0,0 +1,5 @@ +parsing +=============== + +.. automodule:: stix2.parsing + :members: From 056bc4de1983f929e6d12c4abd668f699ff7c268 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Tue, 30 Jun 2020 22:54:40 -0400 Subject: [PATCH 19/30] update custom.ipynb update JSON to STIX 2.1 --- docs/guide/custom.ipynb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/guide/custom.ipynb b/docs/guide/custom.ipynb index 3ad105d..78e7b9c 100644 --- a/docs/guide/custom.ipynb +++ b/docs/guide/custom.ipynb @@ -416,6 +416,7 @@ "\n", "input_string = \"\"\"{\n", " \"type\": \"identity\",\n", + " \"spec_version\": \"2.1\",\n", " \"id\": \"identity--311b2d2d-f010-4473-83ec-1edf84858f4c\",\n", " \"created\": \"2015-12-21T19:59:11Z\",\n", " \"modified\": \"2015-12-21T19:59:11Z\",\n", @@ -512,6 +513,7 @@ ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n", ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "identity",\n",
+       "    "spec_version": "2.1",\n",
        "    "id": "identity--311b2d2d-f010-4473-83ec-1edf84858f4c",\n",
        "    "created": "2015-12-21T19:59:11.000Z",\n",
        "    "modified": "2020-06-24T18:29:24.099Z",\n",

From bc2a6322a9a50e3910d960e920c7cf3d5b49cc44 Mon Sep 17 00:00:00 2001
From: Emmanuelle Vargas-Gonzalez 
Date: Tue, 30 Jun 2020 23:11:59 -0400
Subject: [PATCH 20/30] update ts_support.ipynb

---
 docs/guide/ts_support.ipynb | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/docs/guide/ts_support.ipynb b/docs/guide/ts_support.ipynb
index 7aa3fea..0421898 100644
--- a/docs/guide/ts_support.ipynb
+++ b/docs/guide/ts_support.ipynb
@@ -63,7 +63,14 @@
     "\n",
     "Imports can be used in different ways depending on the use case and support levels.\n",
     "\n",
-    "People who want to support the latest version of STIX 2.X without having to make changes, can implicitly use the latest version:"
+    "People who want to support the latest version of STIX 2 without having to make changes, can implicitly use the latest version:",
+    "
\n", + "\n", + "**Warning**\n", + "\n", + "The implicit import method can cause the code to break between major releases to support a newer approved committee specification. Therefore, not recommended for large scale applications relying on specific object support.\n", + "\n", + "
" ] }, { @@ -143,7 +150,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "or even," + "or even, (less preferred)" ] }, { @@ -230,7 +237,7 @@ "metadata": {}, "source": [ "### How parsing works\n", - "If the ``version`` positional argument is not provided. The library will make the best attempt using the \"spec_version\" property found on a Bundle, SDOs, and SROs.\n", + "If the ``version`` positional argument is not provided the library will make the best attempt using the \"spec_version\" property found on a Bundle, SDOs, SCOs, or SROs.\n", "\n", "You can lock your [parse()](../api/stix2.core.rst#stix2.core.parse) method to a specific STIX version by:" ] @@ -356,7 +363,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Keep in mind that if a 2.1 or higher object is parsed, the operation will fail." + "In the example above if a 2.1 or higher object is parsed, the operation will fail." ] }, { From 339780536786bb31e7cafaba173b275f6c880bd9 Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Wed, 1 Jul 2020 09:41:10 -0400 Subject: [PATCH 21/30] Consolidate version-independent property tests --- stix2/test/test_properties.py | 341 ++++++++++++++++++++++++++++++ stix2/test/v20/test_properties.py | 334 +---------------------------- stix2/test/v21/test_properties.py | 238 +-------------------- 3 files changed, 347 insertions(+), 566 deletions(-) create mode 100644 stix2/test/test_properties.py diff --git a/stix2/test/test_properties.py b/stix2/test/test_properties.py new file mode 100644 index 0000000..a129b12 --- /dev/null +++ b/stix2/test/test_properties.py @@ -0,0 +1,341 @@ +import datetime as dt + +import pytest +import pytz + +import stix2 +from stix2.exceptions import ExtraPropertiesError +from stix2.properties import ( + BinaryProperty, BooleanProperty, EmbeddedObjectProperty, EnumProperty, + FloatProperty, HexProperty, IntegerProperty, ListProperty, Property, + StringProperty, TimestampProperty, TypeProperty, +) + + +def test_property(): + p = Property() + + assert p.required is False + assert p.clean('foo') == 'foo' + assert p.clean(3) == 3 + + +def test_basic_clean(): + class Prop(Property): + + def clean(self, value): + if value == 42: + return value + else: + raise ValueError("Must be 42") + + p = Prop() + + assert p.clean(42) == 42 + with pytest.raises(ValueError): + p.clean(41) + + +def test_property_default(): + class Prop(Property): + + def default(self): + return 77 + + p = Prop() + + assert p.default() == 77 + + +def test_fixed_property(): + p = Property(fixed="2.0") + + assert p.clean("2.0") + with pytest.raises(ValueError): + assert p.clean("x") is False + with pytest.raises(ValueError): + assert p.clean(2.0) is False + + assert p.default() == "2.0" + assert p.clean(p.default()) + + +def test_list_property(): + p = ListProperty(StringProperty) + + assert p.clean(['abc', 'xyz']) + with pytest.raises(ValueError): + p.clean([]) + + +def test_list_property_property_type_custom(): + class TestObj(stix2.base._STIXBase): + _type = "test" + _properties = { + "foo": StringProperty(), + } + p = ListProperty(EmbeddedObjectProperty(type=TestObj)) + + objs_custom = [ + TestObj(foo="abc", bar=123, allow_custom=True), + TestObj(foo="xyz"), + ] + + assert p.clean(objs_custom) + + dicts_custom = [ + {"foo": "abc", "bar": 123}, + {"foo": "xyz"}, + ] + + # no opportunity to set allow_custom=True when using dicts + with pytest.raises(ExtraPropertiesError): + p.clean(dicts_custom) + + +def test_list_property_object_type(): + class TestObj(stix2.base._STIXBase): + _type = "test" + _properties = { + "foo": StringProperty(), + } + p = ListProperty(TestObj) + + objs = [TestObj(foo="abc"), TestObj(foo="xyz")] + assert p.clean(objs) + + dicts = [{"foo": "abc"}, {"foo": "xyz"}] + assert p.clean(dicts) + + +def test_list_property_object_type_custom(): + class TestObj(stix2.base._STIXBase): + _type = "test" + _properties = { + "foo": StringProperty(), + } + p = ListProperty(TestObj) + + objs_custom = [ + TestObj(foo="abc", bar=123, allow_custom=True), + TestObj(foo="xyz"), + ] + + assert p.clean(objs_custom) + + dicts_custom = [ + {"foo": "abc", "bar": 123}, + {"foo": "xyz"}, + ] + + # no opportunity to set allow_custom=True when using dicts + with pytest.raises(ExtraPropertiesError): + p.clean(dicts_custom) + + +def test_list_property_bad_element_type(): + with pytest.raises(TypeError): + ListProperty(1) + + +def test_list_property_bad_value_type(): + class TestObj(stix2.base._STIXBase): + _type = "test" + _properties = { + "foo": StringProperty(), + } + + list_prop = ListProperty(TestObj) + with pytest.raises(ValueError): + list_prop.clean([1]) + + +def test_string_property(): + prop = StringProperty() + + assert prop.clean('foobar') + assert prop.clean(1) + assert prop.clean([1, 2, 3]) + + +def test_type_property(): + prop = TypeProperty('my-type') + + assert prop.clean('my-type') + with pytest.raises(ValueError): + prop.clean('not-my-type') + assert prop.clean(prop.default()) + + +@pytest.mark.parametrize( + "value", [ + 2, + -1, + 3.14, + False, + ], +) +def test_integer_property_valid(value): + int_prop = IntegerProperty() + assert int_prop.clean(value) is not None + + +@pytest.mark.parametrize( + "value", [ + -1, + -100, + -50 * 6, + ], +) +def test_integer_property_invalid_min_with_constraints(value): + int_prop = IntegerProperty(min=0, max=180) + with pytest.raises(ValueError) as excinfo: + int_prop.clean(value) + assert "minimum value is" in str(excinfo.value) + + +@pytest.mark.parametrize( + "value", [ + 181, + 200, + 50 * 6, + ], +) +def test_integer_property_invalid_max_with_constraints(value): + int_prop = IntegerProperty(min=0, max=180) + with pytest.raises(ValueError) as excinfo: + int_prop.clean(value) + assert "maximum value is" in str(excinfo.value) + + +@pytest.mark.parametrize( + "value", [ + "something", + StringProperty(), + ], +) +def test_integer_property_invalid(value): + int_prop = IntegerProperty() + with pytest.raises(ValueError): + int_prop.clean(value) + + +@pytest.mark.parametrize( + "value", [ + 2, + -1, + 3.14, + False, + ], +) +def test_float_property_valid(value): + int_prop = FloatProperty() + assert int_prop.clean(value) is not None + + +@pytest.mark.parametrize( + "value", [ + "something", + StringProperty(), + ], +) +def test_float_property_invalid(value): + int_prop = FloatProperty() + with pytest.raises(ValueError): + int_prop.clean(value) + + +@pytest.mark.parametrize( + "value", [ + True, + False, + 'True', + 'False', + 'true', + 'false', + 'TRUE', + 'FALSE', + 'T', + 'F', + 't', + 'f', + 1, + 0, + ], +) +def test_boolean_property_valid(value): + bool_prop = BooleanProperty() + + assert bool_prop.clean(value) is not None + + +@pytest.mark.parametrize( + "value", [ + 'abc', + ['false'], + {'true': 'true'}, + 2, + -1, + ], +) +def test_boolean_property_invalid(value): + bool_prop = BooleanProperty() + with pytest.raises(ValueError): + bool_prop.clean(value) + + +@pytest.mark.parametrize( + "value", [ + '2017-01-01T12:34:56Z', + ], +) +def test_timestamp_property_valid(value): + ts_prop = TimestampProperty() + assert ts_prop.clean(value) == dt.datetime(2017, 1, 1, 12, 34, 56, tzinfo=pytz.utc) + + +def test_timestamp_property_invalid(): + ts_prop = TimestampProperty() + with pytest.raises(TypeError): + ts_prop.clean(1) + with pytest.raises(ValueError): + ts_prop.clean("someday sometime") + + +def test_binary_property(): + bin_prop = BinaryProperty() + + assert bin_prop.clean("TG9yZW0gSXBzdW0=") + with pytest.raises(ValueError): + bin_prop.clean("foobar") + + +def test_hex_property(): + hex_prop = HexProperty() + + assert hex_prop.clean("4c6f72656d20497073756d") + with pytest.raises(ValueError): + hex_prop.clean("foobar") + + +@pytest.mark.parametrize( + "value", [ + ['a', 'b', 'c'], + ('a', 'b', 'c'), + 'b', + ], +) +def test_enum_property_valid(value): + enum_prop = EnumProperty(value) + assert enum_prop.clean('b') + + +def test_enum_property_clean(): + enum_prop = EnumProperty(['1']) + assert enum_prop.clean(1) == '1' + + +def test_enum_property_invalid(): + enum_prop = EnumProperty(['a', 'b', 'c']) + with pytest.raises(ValueError): + enum_prop.clean('z') diff --git a/stix2/test/v20/test_properties.py b/stix2/test/v20/test_properties.py index 6810966..b03879c 100644 --- a/stix2/test/v20/test_properties.py +++ b/stix2/test/v20/test_properties.py @@ -6,175 +6,16 @@ import stix2 import stix2.base from stix2.exceptions import ( AtLeastOnePropertyError, CustomContentError, DictionaryKeyError, - ExtraPropertiesError, ) from stix2.properties import ( - BinaryProperty, BooleanProperty, DictionaryProperty, - EmbeddedObjectProperty, EnumProperty, ExtensionsProperty, FloatProperty, - HashesProperty, HexProperty, IDProperty, IntegerProperty, ListProperty, - Property, ReferenceProperty, STIXObjectProperty, StringProperty, - TimestampProperty, TypeProperty, + DictionaryProperty, EmbeddedObjectProperty, ExtensionsProperty, + HashesProperty, IDProperty, ListProperty, ReferenceProperty, + STIXObjectProperty, ) from stix2.v20.common import MarkingProperty from . import constants - -def test_property(): - p = Property() - - assert p.required is False - assert p.clean('foo') == 'foo' - assert p.clean(3) == 3 - - -def test_basic_clean(): - class Prop(Property): - - def clean(self, value): - if value == 42: - return value - else: - raise ValueError("Must be 42") - - p = Prop() - - assert p.clean(42) == 42 - with pytest.raises(ValueError): - p.clean(41) - - -def test_property_default(): - class Prop(Property): - - def default(self): - return 77 - - p = Prop() - - assert p.default() == 77 - - -def test_fixed_property(): - p = Property(fixed="2.0") - - assert p.clean("2.0") - with pytest.raises(ValueError): - assert p.clean("x") is False - with pytest.raises(ValueError): - assert p.clean(2.0) is False - - assert p.default() == "2.0" - assert p.clean(p.default()) - - -def test_list_property_property_type(): - p = ListProperty(StringProperty) - - assert p.clean(['abc', 'xyz']) - with pytest.raises(ValueError): - p.clean([]) - - -def test_list_property_property_type_custom(): - class TestObj(stix2.base._STIXBase): - _type = "test" - _properties = { - "foo": StringProperty(), - } - p = ListProperty(EmbeddedObjectProperty(type=TestObj)) - - objs_custom = [ - TestObj(foo="abc", bar=123, allow_custom=True), - TestObj(foo="xyz"), - ] - - assert p.clean(objs_custom) - - dicts_custom = [ - {"foo": "abc", "bar": 123}, - {"foo": "xyz"}, - ] - - # no opportunity to set allow_custom=True when using dicts - with pytest.raises(ExtraPropertiesError): - p.clean(dicts_custom) - - -def test_list_property_object_type(): - class TestObj(stix2.base._STIXBase): - _type = "test" - _properties = { - "foo": StringProperty(), - } - p = ListProperty(TestObj) - - objs = [TestObj(foo="abc"), TestObj(foo="xyz")] - assert p.clean(objs) - - dicts = [{"foo": "abc"}, {"foo": "xyz"}] - assert p.clean(dicts) - - -def test_list_property_object_type_custom(): - class TestObj(stix2.base._STIXBase): - _type = "test" - _properties = { - "foo": StringProperty(), - } - p = ListProperty(TestObj) - - objs_custom = [ - TestObj(foo="abc", bar=123, allow_custom=True), - TestObj(foo="xyz"), - ] - - assert p.clean(objs_custom) - - dicts_custom = [ - {"foo": "abc", "bar": 123}, - {"foo": "xyz"}, - ] - - # no opportunity to set allow_custom=True when using dicts - with pytest.raises(ExtraPropertiesError): - p.clean(dicts_custom) - - -def test_list_property_bad_element_type(): - with pytest.raises(TypeError): - ListProperty(1) - - -def test_list_property_bad_value_type(): - class TestObj(stix2.base._STIXBase): - _type = "test" - _properties = { - "foo": StringProperty(), - } - - list_prop = ListProperty(TestObj) - with pytest.raises(ValueError): - list_prop.clean([1]) - - -def test_string_property(): - prop = StringProperty() - - assert prop.clean('foobar') - assert prop.clean(1) - assert prop.clean([1, 2, 3]) - - -def test_type_property(): - prop = TypeProperty('my-type') - - assert prop.clean('my-type') - with pytest.raises(ValueError): - prop.clean('not-my-type') - assert prop.clean(prop.default()) - - ID_PROP = IDProperty('my-type', spec_version="2.0") MY_ID = 'my-type--232c9d3f-49fc-4440-bb01-607f638778e7' @@ -242,123 +83,6 @@ def test_id_property_default(): assert ID_PROP.clean(default) == default -@pytest.mark.parametrize( - "value", [ - 2, - -1, - 3.14, - False, - ], -) -def test_integer_property_valid(value): - int_prop = IntegerProperty() - assert int_prop.clean(value) is not None - - -@pytest.mark.parametrize( - "value", [ - -1, - -100, - -5 * 6, - ], -) -def test_integer_property_invalid_min_with_constraints(value): - int_prop = IntegerProperty(min=0, max=180) - with pytest.raises(ValueError) as excinfo: - int_prop.clean(value) - assert "minimum value is" in str(excinfo.value) - - -@pytest.mark.parametrize( - "value", [ - 181, - 200, - 50 * 6, - ], -) -def test_integer_property_invalid_max_with_constraints(value): - int_prop = IntegerProperty(min=0, max=180) - with pytest.raises(ValueError) as excinfo: - int_prop.clean(value) - assert "maximum value is" in str(excinfo.value) - - -@pytest.mark.parametrize( - "value", [ - "something", - StringProperty(), - ], -) -def test_integer_property_invalid(value): - int_prop = IntegerProperty() - with pytest.raises(ValueError): - int_prop.clean(value) - - -@pytest.mark.parametrize( - "value", [ - 2, - -1, - 3.14, - False, - ], -) -def test_float_property_valid(value): - int_prop = FloatProperty() - assert int_prop.clean(value) is not None - - -@pytest.mark.parametrize( - "value", [ - "something", - StringProperty(), - ], -) -def test_float_property_invalid(value): - int_prop = FloatProperty() - with pytest.raises(ValueError): - int_prop.clean(value) - - -@pytest.mark.parametrize( - "value", [ - True, - False, - 'True', - 'False', - 'true', - 'false', - 'TRUE', - 'FALSE', - 'T', - 'F', - 't', - 'f', - 1, - 0, - ], -) -def test_boolean_property_valid(value): - bool_prop = BooleanProperty() - - assert bool_prop.clean(value) is not None - - -@pytest.mark.parametrize( - "value", [ - 'abc', - ['false'], - {'true': 'true'}, - 2, - -1, - ], -) -def test_boolean_property_invalid(value): - bool_prop = BooleanProperty() - with pytest.raises(ValueError): - bool_prop.clean(value) - - def test_reference_property(): ref_prop = ReferenceProperty(valid_types="my-type", spec_version="2.0") @@ -381,40 +105,6 @@ def test_reference_property_specific_type(): "my-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf" -@pytest.mark.parametrize( - "value", [ - '2017-01-01T12:34:56Z', - ], -) -def test_timestamp_property_valid(value): - ts_prop = TimestampProperty() - assert ts_prop.clean(value) == constants.FAKE_TIME - - -def test_timestamp_property_invalid(): - ts_prop = TimestampProperty() - with pytest.raises(TypeError): - ts_prop.clean(1) - with pytest.raises(ValueError): - ts_prop.clean("someday sometime") - - -def test_binary_property(): - bin_prop = BinaryProperty() - - assert bin_prop.clean("TG9yZW0gSXBzdW0=") - with pytest.raises(ValueError): - bin_prop.clean("foobar") - - -def test_hex_property(): - hex_prop = HexProperty() - - assert hex_prop.clean("4c6f72656d20497073756d") - with pytest.raises(ValueError): - hex_prop.clean("foobar") - - @pytest.mark.parametrize( "d", [ {'description': 'something'}, @@ -522,24 +212,6 @@ def test_embedded_property(): emb_prop.clean("string") -@pytest.mark.parametrize( - "value", [ - ['a', 'b', 'c'], - ('a', 'b', 'c'), - 'b', - ], -) -def test_enum_property_valid(value): - enum_prop = EnumProperty(value) - assert enum_prop.clean('b') - - -def test_enum_property_invalid(): - enum_prop = EnumProperty(['a', 'b', 'c']) - with pytest.raises(ValueError): - enum_prop.clean('z') - - def test_extension_property_valid(): ext_prop = ExtensionsProperty(spec_version="2.0", enclosing_type='file') assert ext_prop({ diff --git a/stix2/test/v21/test_properties.py b/stix2/test/v21/test_properties.py index 84e87c4..36ff858 100644 --- a/stix2/test/v21/test_properties.py +++ b/stix2/test/v21/test_properties.py @@ -5,73 +5,15 @@ from stix2.exceptions import ( AtLeastOnePropertyError, CustomContentError, DictionaryKeyError, ) from stix2.properties import ( - BinaryProperty, BooleanProperty, DictionaryProperty, - EmbeddedObjectProperty, EnumProperty, ExtensionsProperty, FloatProperty, - HashesProperty, HexProperty, IDProperty, IntegerProperty, ListProperty, - Property, ReferenceProperty, StringProperty, TimestampProperty, - TypeProperty, + DictionaryProperty, EmbeddedObjectProperty, ExtensionsProperty, + HashesProperty, IDProperty, ListProperty, ReferenceProperty, + StringProperty, TypeProperty, ) from stix2.v21.common import MarkingProperty from . import constants -def test_property(): - p = Property() - - assert p.required is False - assert p.clean('foo') == 'foo' - assert p.clean(3) == 3 - - -def test_basic_clean(): - class Prop(Property): - - def clean(self, value): - if value == 42: - return value - else: - raise ValueError("Must be 42") - - p = Prop() - - assert p.clean(42) == 42 - with pytest.raises(ValueError): - p.clean(41) - - -def test_property_default(): - class Prop(Property): - - def default(self): - return 77 - - p = Prop() - - assert p.default() == 77 - - -def test_fixed_property(): - p = Property(fixed="2.0") - - assert p.clean("2.0") - with pytest.raises(ValueError): - assert p.clean("x") is False - with pytest.raises(ValueError): - assert p.clean(2.0) is False - - assert p.default() == "2.0" - assert p.clean(p.default()) - - -def test_list_property(): - p = ListProperty(StringProperty) - - assert p.clean(['abc', 'xyz']) - with pytest.raises(ValueError): - p.clean([]) - - def test_dictionary_property(): p = DictionaryProperty(StringProperty) @@ -161,123 +103,6 @@ def test_id_property_default(): assert ID_PROP.clean(default) == default -@pytest.mark.parametrize( - "value", [ - 2, - -1, - 3.14, - False, - ], -) -def test_integer_property_valid(value): - int_prop = IntegerProperty() - assert int_prop.clean(value) is not None - - -@pytest.mark.parametrize( - "value", [ - -1, - -100, - -50 * 6, - ], -) -def test_integer_property_invalid_min_with_constraints(value): - int_prop = IntegerProperty(min=0, max=180) - with pytest.raises(ValueError) as excinfo: - int_prop.clean(value) - assert "minimum value is" in str(excinfo.value) - - -@pytest.mark.parametrize( - "value", [ - 181, - 200, - 50 * 6, - ], -) -def test_integer_property_invalid_max_with_constraints(value): - int_prop = IntegerProperty(min=0, max=180) - with pytest.raises(ValueError) as excinfo: - int_prop.clean(value) - assert "maximum value is" in str(excinfo.value) - - -@pytest.mark.parametrize( - "value", [ - "something", - StringProperty(), - ], -) -def test_integer_property_invalid(value): - int_prop = IntegerProperty() - with pytest.raises(ValueError): - int_prop.clean(value) - - -@pytest.mark.parametrize( - "value", [ - 2, - -1, - 3.14, - False, - ], -) -def test_float_property_valid(value): - int_prop = FloatProperty() - assert int_prop.clean(value) is not None - - -@pytest.mark.parametrize( - "value", [ - "something", - StringProperty(), - ], -) -def test_float_property_invalid(value): - int_prop = FloatProperty() - with pytest.raises(ValueError): - int_prop.clean(value) - - -@pytest.mark.parametrize( - "value", [ - True, - False, - 'True', - 'False', - 'true', - 'false', - 'TRUE', - 'FALSE', - 'T', - 'F', - 't', - 'f', - 1, - 0, - ], -) -def test_boolean_property_valid(value): - bool_prop = BooleanProperty() - - assert bool_prop.clean(value) is not None - - -@pytest.mark.parametrize( - "value", [ - 'abc', - ['false'], - {'true': 'true'}, - 2, - -1, - ], -) -def test_boolean_property_invalid(value): - bool_prop = BooleanProperty() - with pytest.raises(ValueError): - bool_prop.clean(value) - - def test_reference_property(): ref_prop = ReferenceProperty(valid_types="my-type", spec_version="2.1") @@ -300,40 +125,6 @@ def test_reference_property_specific_type(): "my-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf" -@pytest.mark.parametrize( - "value", [ - '2017-01-01T12:34:56Z', - ], -) -def test_timestamp_property_valid(value): - ts_prop = TimestampProperty() - assert ts_prop.clean(value) == constants.FAKE_TIME - - -def test_timestamp_property_invalid(): - ts_prop = TimestampProperty() - with pytest.raises(TypeError): - ts_prop.clean(1) - with pytest.raises(ValueError): - ts_prop.clean("someday sometime") - - -def test_binary_property(): - bin_prop = BinaryProperty() - - assert bin_prop.clean("TG9yZW0gSXBzdW0=") - with pytest.raises(ValueError): - bin_prop.clean("foobar") - - -def test_hex_property(): - hex_prop = HexProperty() - - assert hex_prop.clean("4c6f72656d20497073756d") - with pytest.raises(ValueError): - hex_prop.clean("foobar") - - @pytest.mark.parametrize( "d", [ {'description': 'something'}, @@ -452,29 +243,6 @@ def test_embedded_property(): emb_prop.clean("string") -@pytest.mark.parametrize( - "value", [ - ['a', 'b', 'c'], - ('a', 'b', 'c'), - 'b', - ], -) -def test_enum_property_valid(value): - enum_prop = EnumProperty(value) - assert enum_prop.clean('b') - - -def test_enum_property_clean(): - enum_prop = EnumProperty(['1']) - assert enum_prop.clean(1) == '1' - - -def test_enum_property_invalid(): - enum_prop = EnumProperty(['a', 'b', 'c']) - with pytest.raises(ValueError): - enum_prop.clean('z') - - def test_extension_property_valid(): ext_prop = ExtensionsProperty(spec_version='2.1', enclosing_type='file') assert ext_prop({ From 53dbc5835e682ce561667cf4c7e2f7753338e9fd Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Wed, 1 Jul 2020 09:54:50 -0400 Subject: [PATCH 22/30] Add test for coverage Tests creating a property with both required and default. --- stix2/test/test_properties.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/stix2/test/test_properties.py b/stix2/test/test_properties.py index a129b12..dab713e 100644 --- a/stix2/test/test_properties.py +++ b/stix2/test/test_properties.py @@ -4,7 +4,7 @@ import pytest import pytz import stix2 -from stix2.exceptions import ExtraPropertiesError +from stix2.exceptions import ExtraPropertiesError, STIXError from stix2.properties import ( BinaryProperty, BooleanProperty, EmbeddedObjectProperty, EnumProperty, FloatProperty, HexProperty, IntegerProperty, ListProperty, Property, @@ -47,7 +47,7 @@ def test_property_default(): assert p.default() == 77 -def test_fixed_property(): +def test_property_fixed(): p = Property(fixed="2.0") assert p.clean("2.0") @@ -60,6 +60,11 @@ def test_fixed_property(): assert p.clean(p.default()) +def test_property_fixed_and_required(): + with pytest.raises(STIXError): + Property(default=lambda: 3, required=True) + + def test_list_property(): p = ListProperty(StringProperty) From 2ac3a4b524e593df69fa1eab79abdc7a244e8be5 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Wed, 1 Jul 2020 09:36:01 -0400 Subject: [PATCH 23/30] add module docstring for versioning.py --- stix2/patterns.py | 2 +- stix2/versioning.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/stix2/patterns.py b/stix2/patterns.py index 5d7c0a2..edcf017 100644 --- a/stix2/patterns.py +++ b/stix2/patterns.py @@ -1,4 +1,4 @@ -"""Classes to aid in working with the STIX 2 patterning language.""" +"""Classes to aid in working with the STIX2 patterning language.""" import base64 import binascii diff --git a/stix2/versioning.py b/stix2/versioning.py index 8db8f7a..90d1ac9 100644 --- a/stix2/versioning.py +++ b/stix2/versioning.py @@ -1,3 +1,5 @@ +"""STIX2 core versioning methods.""" + import copy import datetime as dt import itertools From fb0207fda8a479fed7605299307745d70ca78bdf Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Wed, 1 Jul 2020 10:17:00 -0400 Subject: [PATCH 24/30] add stix2.pattern_visitor.rst --- docs/api/stix2.pattern_visitor.rst | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 docs/api/stix2.pattern_visitor.rst diff --git a/docs/api/stix2.pattern_visitor.rst b/docs/api/stix2.pattern_visitor.rst new file mode 100644 index 0000000..e69de29 From c3e5da4634fa3911606eec532399bbdaf2d1fa67 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Wed, 1 Jul 2020 10:19:08 -0400 Subject: [PATCH 25/30] add `pattern_visitor` to api reference --- stix2/__init__.py | 1 + stix2/pattern_visitor.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/stix2/__init__.py b/stix2/__init__.py index 2f94806..d0051ee 100644 --- a/stix2/__init__.py +++ b/stix2/__init__.py @@ -9,6 +9,7 @@ exceptions markings parsing + pattern_visitor patterns properties utils diff --git a/stix2/pattern_visitor.py b/stix2/pattern_visitor.py index 4ec2b20..5b8300f 100644 --- a/stix2/pattern_visitor.py +++ b/stix2/pattern_visitor.py @@ -1,3 +1,5 @@ +"""STIX2 classes and methods to generate AST from patterns""" + import importlib import inspect From 3150e1d6eaec47557d22f507fe3d8e99db7b3d7b Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Wed, 1 Jul 2020 10:23:30 -0400 Subject: [PATCH 26/30] update section on `STIX 2 Technical Specification Support` README.rst --- README.rst | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/README.rst b/README.rst index 0a32af2..3b924c3 100644 --- a/README.rst +++ b/README.rst @@ -63,18 +63,20 @@ To parse a STIX JSON string into a Python STIX object, use ``parse()``: For more in-depth documentation, please see `https://stix2.readthedocs.io/ `__. -STIX 2.X Technical Specification Support ----------------------------------------- +STIX 2 Technical Specification Support +-------------------------------------- -This version of python-stix2 brings initial support to STIX 2.1 currently at the -CSD level. The intention is to help debug components of the library and also -check for problems that should be fixed in the specification. +This version of cti-python-stix2 brings support to `STIX Version 2.1 `__ +published on 20 March 2020 currently at the Committee Specification (CS) level. -The `stix2` Python library is built to support multiple versions of the STIX -Technical Specification. With every major release of stix2 the ``import stix2`` -statement will automatically load the SDO/SROs equivalent to the most recent -supported 2.X Committee Specification. Please see the library documentation for -more details. +The `stix2` Python library is built to support multiple versions of the STIX2 +Technical Specification. With newer Committee Specification Draft (CSD) the +library will bring initial support with the intention is to help debug components +of the library and also check for problems that should be fixed in the specification, +but these modules must be imported manually until the CSD reaches CS level. +With continued releases to stix2 the ``import stix2`` implicit import statement +will be updated to automatically load the STIX Objects equivalent to the most +recently supported CS. Please see the library documentation for details. Governance ---------- From 1477effb2f6bce02a32b61512698d46755604978 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Wed, 1 Jul 2020 10:32:05 -0400 Subject: [PATCH 27/30] add content for stix2.pattern_visitor.rst --- docs/api/stix2.pattern_visitor.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/api/stix2.pattern_visitor.rst b/docs/api/stix2.pattern_visitor.rst index e69de29..d75ecb1 100644 --- a/docs/api/stix2.pattern_visitor.rst +++ b/docs/api/stix2.pattern_visitor.rst @@ -0,0 +1,5 @@ +pattern_visitor +============== + +.. automodule:: stix2.pattern_visitor + :members: From f2e044115b0b23bed00838ccd6ce37429800ff08 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Wed, 1 Jul 2020 10:32:51 -0400 Subject: [PATCH 28/30] add newlines for stix2.v20.rst, stix2.v21.rst, stix2.confidence.rst --- docs/api/stix2.confidence.rst | 2 +- docs/api/stix2.v20.rst | 2 +- docs/api/stix2.v21.rst | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/api/stix2.confidence.rst b/docs/api/stix2.confidence.rst index 23ac076..5c2658b 100644 --- a/docs/api/stix2.confidence.rst +++ b/docs/api/stix2.confidence.rst @@ -2,4 +2,4 @@ confidence ================ .. automodule:: stix2.confidence - :members: \ No newline at end of file + :members: diff --git a/docs/api/stix2.v20.rst b/docs/api/stix2.v20.rst index 2cbb2e3..85f4a22 100644 --- a/docs/api/stix2.v20.rst +++ b/docs/api/stix2.v20.rst @@ -2,4 +2,4 @@ v20 ========= .. automodule:: stix2.v20 - :members: \ No newline at end of file + :members: diff --git a/docs/api/stix2.v21.rst b/docs/api/stix2.v21.rst index fc02330..7157e3d 100644 --- a/docs/api/stix2.v21.rst +++ b/docs/api/stix2.v21.rst @@ -2,4 +2,4 @@ v21 ========= .. automodule:: stix2.v21 - :members: \ No newline at end of file + :members: From 6daa9bd8a50449ecde3308617246b185cb019da2 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Wed, 1 Jul 2020 11:21:53 -0400 Subject: [PATCH 29/30] fix internal links to stix2.parsing.rst#stix2.parsing.parse --- docs/api/stix2.pattern_visitor.rst | 2 +- docs/guide/custom.ipynb | 2 +- docs/guide/parsing.ipynb | 4 ++-- docs/guide/ts_support.ipynb | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/api/stix2.pattern_visitor.rst b/docs/api/stix2.pattern_visitor.rst index d75ecb1..fe8e34f 100644 --- a/docs/api/stix2.pattern_visitor.rst +++ b/docs/api/stix2.pattern_visitor.rst @@ -1,5 +1,5 @@ pattern_visitor -============== +================ .. automodule:: stix2.pattern_visitor :members: diff --git a/docs/guide/custom.ipynb b/docs/guide/custom.ipynb index 78e7b9c..8a4ab10 100644 --- a/docs/guide/custom.ipynb +++ b/docs/guide/custom.ipynb @@ -320,7 +320,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Likewise, when parsing STIX content with custom properties, pass ``allow_custom=True`` to [parse()](../api/stix2.core.rst#stix2.core.parse):" + "Likewise, when parsing STIX content with custom properties, pass ``allow_custom=True`` to [parse()](../api/stix2.parsing.rst#stix2.parsing.parse):" ] }, { diff --git a/docs/guide/parsing.ipynb b/docs/guide/parsing.ipynb index 5346d16..5fd9499 100644 --- a/docs/guide/parsing.ipynb +++ b/docs/guide/parsing.ipynb @@ -64,7 +64,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Parsing STIX content is as easy as calling the [parse()](../api/stix2.core.rst#stix2.core.parse) function on a JSON string, dictionary, or file-like object. It will automatically determine the type of the object. The STIX objects within `bundle` objects, and any cyber observables contained within `observed-data` objects will be parsed as well.\n", + "Parsing STIX content is as easy as calling the [parse()](../api/stix2.parsing.rst#stix2.parsing.parse) function on a JSON string, dictionary, or file-like object. It will automatically determine the type of the object. The STIX objects within `bundle` objects, and any cyber observables contained within `observed-data` objects will be parsed as well.\n", "\n", "**Parsing a string**" ] @@ -691,7 +691,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Parsing custom STIX objects and/or STIX objects with custom properties is also completed easily with [parse()](../api/stix2.core.rst#stix2.core.parse). Just supply the keyword argument ``allow_custom=True``. When ``allow_custom`` is specified, [parse()](../api/stix2.core.rst#stix2.core.parse) will attempt to convert the supplied STIX content to known STIX 2 domain objects and/or previously defined [custom STIX 2 objects](custom.ipynb). If the conversion cannot be completed (and ``allow_custom`` is specified), [parse()](../api/stix2.core.rst#stix2.core.parse) will treat the supplied STIX 2 content as valid STIX 2 objects and return them. This is an axiomatic possibility as the ``stix2`` library cannot guarantee proper processing of unknown custom STIX 2 objects that were explicitly flagged to be allowed, and thus may not be valid.\n", + "Parsing custom STIX objects and/or STIX objects with custom properties is also completed easily with [parse()](../api/stix2.parsing.rst#stix2.parsing.parse). Just supply the keyword argument ``allow_custom=True``. When ``allow_custom`` is specified, [parse()](../api/stix2.parsing.rst#stix2.parsing.parse) will attempt to convert the supplied STIX content to known STIX 2 domain objects and/or previously defined [custom STIX 2 objects](custom.ipynb). If the conversion cannot be completed (and ``allow_custom`` is specified), [parse()](../api/stix2.parsing.rst#stix2.parsing.parse) will treat the supplied STIX 2 content as valid STIX 2 objects and return them. This is an axiomatic possibility as the ``stix2`` library cannot guarantee proper processing of unknown custom STIX 2 objects that were explicitly flagged to be allowed, and thus may not be valid.\n", "
\n", "\n", "**Warning**\n", diff --git a/docs/guide/ts_support.ipynb b/docs/guide/ts_support.ipynb index 0421898..99ef893 100644 --- a/docs/guide/ts_support.ipynb +++ b/docs/guide/ts_support.ipynb @@ -239,7 +239,7 @@ "### How parsing works\n", "If the ``version`` positional argument is not provided the library will make the best attempt using the \"spec_version\" property found on a Bundle, SDOs, SCOs, or SROs.\n", "\n", - "You can lock your [parse()](../api/stix2.core.rst#stix2.core.parse) method to a specific STIX version by:" + "You can lock your [parse()](../api/stix2.parsing.rst#stix2.parsing.parse) method to a specific STIX version by:" ] }, { From 4b02d220683d3e31e293b8c0cc517af7b776fd83 Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Wed, 1 Jul 2020 15:05:42 -0400 Subject: [PATCH 30/30] Simplify technical spec support explanation --- README.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/README.rst b/README.rst index 3b924c3..999a1a7 100644 --- a/README.rst +++ b/README.rst @@ -69,14 +69,14 @@ STIX 2 Technical Specification Support This version of cti-python-stix2 brings support to `STIX Version 2.1 `__ published on 20 March 2020 currently at the Committee Specification (CS) level. -The `stix2` Python library is built to support multiple versions of the STIX2 -Technical Specification. With newer Committee Specification Draft (CSD) the -library will bring initial support with the intention is to help debug components -of the library and also check for problems that should be fixed in the specification, -but these modules must be imported manually until the CSD reaches CS level. -With continued releases to stix2 the ``import stix2`` implicit import statement +The stix2 Python library supports multiple versions of the STIX 2 Technical +Specification. The library will be updated to support new Committee +Specification Drafts (CSDs) as they are released, but modules for these +versions must be imported manually until the CSD reaches CS level. In new +major releases of stix2 the ``import stix2`` implicit import statement will be updated to automatically load the STIX Objects equivalent to the most -recently supported CS. Please see the library documentation for details. +recently supported CS. Please see the `library documentation `__ +for details. Governance ----------