MISP
/
cti-python-stix2
mirror of https://github.com/MISP/cti-python-stix2
2
0
Fork 0
Code Issues Releases Wiki Activity

Merge branch 'master' of github.com:oasis-open/cti-python-stix2

master
chrisr3d 2020-05-22 22:15:40 +02:00
parent e4f08557ec 7e418252d5
commit ca61b06aa2
No known key found for this signature in database
GPG Key ID: 6BBED1B63A6D639F
15 changed files with 116 additions and 84 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
setup.py
View File

@ -52,7 +52,6 @@ setup(
packages=find_packages(exclude=['*.test', '*.test.*']),
install_requires=[
'enum34 ; python_version<"3.4"',
'python-dateutil',
'pytz',
'requests',
'simplejson',

14
stix2/base.py
View File

@ -418,25 +418,25 @@ class _Observable(_STIXBase):
properties_to_use = self._id_contributing_properties
if properties_to_use:
streamlined_obj_vals = []
streamlined_object = {}
if "hashes" in kwargs and "hashes" in properties_to_use:
possible_hash = _choose_one_hash(kwargs["hashes"])
if possible_hash:
streamlined_obj_vals.append(possible_hash)
streamlined_object["hashes"] = possible_hash
for key in properties_to_use:
if key != "hashes" and key in kwargs:
if isinstance(kwargs[key], dict) or isinstance(kwargs[key], _STIXBase):
temp_deep_copy = copy.deepcopy(dict(kwargs[key]))
_recursive_stix_to_dict(temp_deep_copy)
streamlined_obj_vals.append(temp_deep_copy)
streamlined_object[key] = temp_deep_copy
elif isinstance(kwargs[key], list):
temp_deep_copy = copy.deepcopy(kwargs[key])
_recursive_stix_list_to_dict(temp_deep_copy)
streamlined_obj_vals.append(temp_deep_copy)
streamlined_object[key] = temp_deep_copy
else:
streamlined_obj_vals.append(kwargs[key])
if streamlined_obj_vals:
data = canonicalize(streamlined_obj_vals, utf8=False)
streamlined_object[key] = kwargs[key]
if streamlined_object:
data = canonicalize(streamlined_object, utf8=False)
# The situation is complicated w.r.t. python 2/3 behavior, so
# I'd rather not rely on particular exceptions being raised to

6
stix2/pattern_visitor.py
View File

@ -310,7 +310,11 @@ class STIXPatternVisitorForSTIX2():
elif node.symbol.type == self.parser_class.BoolLiteral:
return BooleanConstant(node.getText())
elif node.symbol.type == self.parser_class.TimestampLiteral:
return TimestampConstant(node.getText())
value = node.getText()
# STIX 2.1 uses a special timestamp literal syntax
if value.startswith("t"):
value = value[2:-1]
return TimestampConstant(value)
else:
return node

32
stix2/patterns.py
View File

@ -121,21 +121,21 @@ class BooleanConstant(_Constant):
_HASH_REGEX = {
"MD5": ("^[a-fA-F0-9]{32}$", "MD5"),
"MD6": ("^[a-fA-F0-9]{32}|[a-fA-F0-9]{40}|[a-fA-F0-9]{56}|[a-fA-F0-9]{64}|[a-fA-F0-9]{96}|[a-fA-F0-9]{128}$", "MD6"),
"RIPEMD160": ("^[a-fA-F0-9]{40}$", "RIPEMD-160"),
"SHA1": ("^[a-fA-F0-9]{40}$", "SHA-1"),
"SHA224": ("^[a-fA-F0-9]{56}$", "SHA-224"),
"SHA256": ("^[a-fA-F0-9]{64}$", "SHA-256"),
"SHA384": ("^[a-fA-F0-9]{96}$", "SHA-384"),
"SHA512": ("^[a-fA-F0-9]{128}$", "SHA-512"),
"SHA3224": ("^[a-fA-F0-9]{56}$", "SHA3-224"),
"SHA3256": ("^[a-fA-F0-9]{64}$", "SHA3-256"),
"SHA3384": ("^[a-fA-F0-9]{96}$", "SHA3-384"),
"SHA3512": ("^[a-fA-F0-9]{128}$", "SHA3-512"),
"SSDEEP": ("^[a-zA-Z0-9/+:.]{1,128}$", "ssdeep"),
"WHIRLPOOL": ("^[a-fA-F0-9]{128}$", "WHIRLPOOL"),
"TLSH": ("^[a-fA-F0-9]{70}$", "TLSH"),
"MD5": (r"^[a-fA-F0-9]{32}$", "MD5"),
"MD6": (r"^[a-fA-F0-9]{32}|[a-fA-F0-9]{40}|[a-fA-F0-9]{56}|[a-fA-F0-9]{64}|[a-fA-F0-9]{96}|[a-fA-F0-9]{128}$", "MD6"),
"RIPEMD160": (r"^[a-fA-F0-9]{40}$", "RIPEMD-160"),
"SHA1": (r"^[a-fA-F0-9]{40}$", "SHA-1"),
"SHA224": (r"^[a-fA-F0-9]{56}$", "SHA-224"),
"SHA256": (r"^[a-fA-F0-9]{64}$", "SHA-256"),
"SHA384": (r"^[a-fA-F0-9]{96}$", "SHA-384"),
"SHA512": (r"^[a-fA-F0-9]{128}$", "SHA-512"),
"SHA3224": (r"^[a-fA-F0-9]{56}$", "SHA3-224"),
"SHA3256": (r"^[a-fA-F0-9]{64}$", "SHA3-256"),
"SHA3384": (r"^[a-fA-F0-9]{96}$", "SHA3-384"),
"SHA3512": (r"^[a-fA-F0-9]{128}$", "SHA3-512"),
"SSDEEP": (r"^[a-zA-Z0-9/+:.]{1,128}$", "SSDEEP"),
"WHIRLPOOL": (r"^[a-fA-F0-9]{128}$", "WHIRLPOOL"),
"TLSH": (r"^[a-fA-F0-9]{70}$", "TLSH"),
}
@ -228,7 +228,7 @@ def make_constant(value):
try:
return parse_into_datetime(value)
except ValueError:
except (ValueError, TypeError):
pass
if isinstance(value, str):

6
stix2/properties.py
View File

@ -427,7 +427,7 @@ HASHES_REGEX = {
"SHA3256": (r"^[a-fA-F0-9]{64}$", "SHA3-256"),
"SHA3384": (r"^[a-fA-F0-9]{96}$", "SHA3-384"),
"SHA3512": (r"^[a-fA-F0-9]{128}$", "SHA3-512"),
"SSDEEP": (r"^[a-zA-Z0-9/+:.]{1,128}$", "ssdeep"),
"SSDEEP": (r"^[a-zA-Z0-9/+:.]{1,128}$", "SSDEEP"),
"WHIRLPOOL": (r"^[a-fA-F0-9]{128}$", "WHIRLPOOL"),
"TLSH": (r"^[a-fA-F0-9]{70}$", "TLSH"),
}
@ -441,6 +441,8 @@ class HashesProperty(DictionaryProperty):
key = k.upper().replace('-', '')
if key in HASHES_REGEX:
vocab_key = HASHES_REGEX[key][1]
if vocab_key == "SSDEEP" and self.spec_version == "2.0":
vocab_key = vocab_key.lower()
if not re.match(HASHES_REGEX[key][0], v):
raise ValueError("'{0}' is not a valid {1} hash".format(v, vocab_key))
if k != vocab_key:
@ -513,7 +515,7 @@ class ReferenceProperty(Property):
if possible_prefix not in ref_invalid_types:
required_prefix = possible_prefix + '--'
else:
raise ValueError("An invalid type-specifying prefix '%s' was specified for this property" % (possible_prefix, value))
raise ValueError("An invalid type-specifying prefix '%s' was specified for this property" % (possible_prefix))
interoperability = self.interoperability if hasattr(self, 'interoperability') and self.interoperability else False
_validate_id(value, self.spec_version, required_prefix, interoperability)

26
stix2/test/v20/test_attack_pattern.py
View File

@ -4,6 +4,7 @@ import pytest
import pytz
import stix2
import stix2.exceptions
from .constants import ATTACK_PATTERN_ID
@ -83,19 +84,18 @@ def test_attack_pattern_invalid_labels():
def test_overly_precise_timestamps():
ap = stix2.v20.AttackPattern(
id=ATTACK_PATTERN_ID,
created="2016-05-12T08:17:27.0000342Z",
modified="2016-05-12T08:17:27.000287Z",
name="Spear Phishing",
external_references=[{
"source_name": "capec",
"external_id": "CAPEC-163",
}],
description="...",
)
assert str(ap) == EXPECTED
with pytest.raises(stix2.exceptions.InvalidValueError):
stix2.v20.AttackPattern(
id=ATTACK_PATTERN_ID,
created="2016-05-12T08:17:27.0000342Z",
modified="2016-05-12T08:17:27.000287Z",
name="Spear Phishing",
external_references=[{
"source_name": "capec",
"external_id": "CAPEC-163",
}],
description="...",
)
def test_less_precise_timestamps():

16
stix2/test/v20/test_observed_data.py
View File

@ -714,6 +714,22 @@ def test_file_example():
assert f.decryption_key == "fred" # does the key have a format we can test for?
def test_file_ssdeep_example():
f = stix2.v20.File(
name="example.dll",
hashes={
"SHA-256": "ceafbfd424be2ca4a5f0402cae090dda2fb0526cf521b60b60077c0f622b285a",
"ssdeep": "96:gS/mFkCpXTWLr/PbKQHbr/S/mFkCpXTWLr/PbKQHbrB:Tu6SXTWGQHbeu6SXTWGQHbV",
},
size=1024,
)
assert f.name == "example.dll"
assert f.size == 1024
assert f.hashes["SHA-256"] == "ceafbfd424be2ca4a5f0402cae090dda2fb0526cf521b60b60077c0f622b285a"
assert f.hashes["ssdeep"] == "96:gS/mFkCpXTWLr/PbKQHbr/S/mFkCpXTWLr/PbKQHbrB:Tu6SXTWGQHbeu6SXTWGQHbV"
def test_file_example_with_NTFSExt():
f = stix2.v20.File(
name="abc.txt",

4
stix2/test/v20/test_properties.py
View File

@ -300,8 +300,6 @@ def test_reference_property_specific_type():
@pytest.mark.parametrize(
"value", [
'2017-01-01T12:34:56Z',
'2017-01-01 12:34:56',
'Jan 1 2017 12:34:56',
],
)
def test_timestamp_property_valid(value):
@ -311,7 +309,7 @@ def test_timestamp_property_valid(value):
def test_timestamp_property_invalid():
ts_prop = TimestampProperty()
with pytest.raises(ValueError):
with pytest.raises(TypeError):
ts_prop.clean(1)
with pytest.raises(ValueError):
ts_prop.clean("someday sometime")

12
stix2/test/v20/test_utils.py
View File

@ -35,8 +35,6 @@ def test_timestamp_formatting(dttm, timestamp):
(dt.datetime(2017, 1, 1, 0, tzinfo=pytz.utc), dt.datetime(2017, 1, 1, 0, 0, 0, tzinfo=pytz.utc)),
(dt.date(2017, 1, 1), dt.datetime(2017, 1, 1, 0, 0, 0, tzinfo=pytz.utc)),
('2017-01-01T00:00:00Z', dt.datetime(2017, 1, 1, 0, 0, 0, tzinfo=pytz.utc)),
('2017-01-01T02:00:00+2:00', dt.datetime(2017, 1, 1, 0, 0, 0, tzinfo=pytz.utc)),
('2017-01-01T00:00:00', dt.datetime(2017, 1, 1, 0, 0, 0, tzinfo=pytz.utc)),
],
)
def test_parse_datetime(timestamp, dttm):
@ -45,11 +43,11 @@ def test_parse_datetime(timestamp, dttm):
@pytest.mark.parametrize(
'timestamp, dttm, precision', [
('2017-01-01T01:02:03.000001', dt.datetime(2017, 1, 1, 1, 2, 3, 0, tzinfo=pytz.utc), 'millisecond'),
('2017-01-01T01:02:03.001', dt.datetime(2017, 1, 1, 1, 2, 3, 1000, tzinfo=pytz.utc), 'millisecond'),
('2017-01-01T01:02:03.1', dt.datetime(2017, 1, 1, 1, 2, 3, 100000, tzinfo=pytz.utc), 'millisecond'),
('2017-01-01T01:02:03.45', dt.datetime(2017, 1, 1, 1, 2, 3, 450000, tzinfo=pytz.utc), 'millisecond'),
('2017-01-01T01:02:03.45', dt.datetime(2017, 1, 1, 1, 2, 3, tzinfo=pytz.utc), 'second'),
('2017-01-01T01:02:03.000001Z', dt.datetime(2017, 1, 1, 1, 2, 3, 0, tzinfo=pytz.utc), 'millisecond'),
('2017-01-01T01:02:03.001Z', dt.datetime(2017, 1, 1, 1, 2, 3, 1000, tzinfo=pytz.utc), 'millisecond'),
('2017-01-01T01:02:03.1Z', dt.datetime(2017, 1, 1, 1, 2, 3, 100000, tzinfo=pytz.utc), 'millisecond'),
('2017-01-01T01:02:03.45Z', dt.datetime(2017, 1, 1, 1, 2, 3, 450000, tzinfo=pytz.utc), 'millisecond'),
('2017-01-01T01:02:03.45Z', dt.datetime(2017, 1, 1, 1, 2, 3, tzinfo=pytz.utc), 'second'),
],
)
def test_parse_datetime_precision(timestamp, dttm, precision):

26
stix2/test/v21/test_attack_pattern.py
View File

@ -4,6 +4,7 @@ import pytest
import pytz
import stix2
import stix2.exceptions
from .constants import ATTACK_PATTERN_ID
@ -86,19 +87,18 @@ def test_attack_pattern_invalid_labels():
def test_overly_precise_timestamps():
ap = stix2.v21.AttackPattern(
id=ATTACK_PATTERN_ID,
created="2016-05-12T08:17:27.000000342Z",
modified="2016-05-12T08:17:27.000000287Z",
name="Spear Phishing",
external_references=[{
"source_name": "capec",
"external_id": "CAPEC-163",
}],
description="...",
)
assert str(ap) == EXPECTED
with pytest.raises(stix2.exceptions.InvalidValueError):
stix2.v21.AttackPattern(
id=ATTACK_PATTERN_ID,
created="2016-05-12T08:17:27.000000342Z",
modified="2016-05-12T08:17:27.000000287Z",
name="Spear Phishing",
external_references=[{
"source_name": "capec",
"external_id": "CAPEC-163",
}],
description="...",
)
def test_less_precise_timestamps():

32
stix2/test/v21/test_observed_data.py
View File

@ -46,7 +46,7 @@ def test_observed_data_example():
objects={
"0": {
"type": "file",
"id": "file--5956efbb-a7b0-566d-a7f9-a202eb05c70f",
"id": "file--7af1312c-4402-5d2f-b169-b118d73b85c4",
"name": "foo.exe",
},
},
@ -102,12 +102,12 @@ def test_observed_data_example_with_refs():
objects={
"0": {
"type": "file",
"id": "file--5956efbb-a7b0-566d-a7f9-a202eb05c70f",
"id": "file--7af1312c-4402-5d2f-b169-b118d73b85c4",
"name": "foo.exe",
},
"1": {
"type": "directory",
"id": "directory--536a61a4-0934-516b-9aad-fcbb75e0583a",
"id": "directory--ee97f78e-7e2b-5b3d-bcbd-5692968cacea",
"path": "/usr/home",
"contains_refs": ["file--5956efbb-a7b0-566d-a7f9-a202eb05c70f"],
},
@ -719,7 +719,7 @@ def test_directory_example():
assert dir1.ctime == dt.datetime(2015, 12, 21, 19, 0, 0, tzinfo=pytz.utc)
assert dir1.mtime == dt.datetime(2015, 12, 24, 19, 0, 0, tzinfo=pytz.utc)
assert dir1.atime == dt.datetime(2015, 12, 21, 20, 0, 0, tzinfo=pytz.utc)
assert dir1.contains_refs == ["file--9d050a3b-72cd-5b57-bf18-024e74e1e5eb"]
assert dir1.contains_refs == ["file--c6ae2cf8-92d3-56d0-a25f-713efad643a7"]
def test_directory_example_ref_error():
@ -747,7 +747,7 @@ def test_domain_name_example():
)
assert dn2.value == "example.com"
assert dn2.resolves_to_refs == ["domain-name--02af94ea-7e38-5718-87c3-5cc023e3d49d"]
assert dn2.resolves_to_refs == ["domain-name--5b5803bf-a7eb-5076-b799-96aa574c44eb"]
def test_domain_name_example_invalid_ref_type():
@ -785,6 +785,22 @@ def test_file_example():
assert f.atime == dt.datetime(2016, 12, 21, 20, 0, 0, tzinfo=pytz.utc)
def test_file_ssdeep_example():
f = stix2.v21.File(
name="example.dll",
hashes={
"SHA-256": "ceafbfd424be2ca4a5f0402cae090dda2fb0526cf521b60b60077c0f622b285a",
"SSDEEP": "96:gS/mFkCpXTWLr/PbKQHbr/S/mFkCpXTWLr/PbKQHbrB:Tu6SXTWGQHbeu6SXTWGQHbV",
},
size=1024,
)
assert f.name == "example.dll"
assert f.size == 1024
assert f.hashes["SHA-256"] == "ceafbfd424be2ca4a5f0402cae090dda2fb0526cf521b60b60077c0f622b285a"
assert f.hashes["SSDEEP"] == "96:gS/mFkCpXTWLr/PbKQHbr/S/mFkCpXTWLr/PbKQHbrB:Tu6SXTWGQHbeu6SXTWGQHbV"
def test_file_example_with_NTFSExt():
f = stix2.v21.File(
name="abc.txt",
@ -1032,7 +1048,7 @@ def test_ipv4_address_valid_refs():
)
assert ip4.value == "177.60.40.7"
assert ip4.resolves_to_refs == ["mac-addr--a85820f7-d9b7-567a-a3a6-dedc34139342", "mac-addr--9a59b496-fdeb-510f-97b5-7137210bc699"]
assert ip4.resolves_to_refs == ["mac-addr--f72d7d00-86bd-5cd2-8c86-52f7a83bef62", "mac-addr--875ad625-177b-5c2a-9101-d44b0ad55938"]
def test_ipv4_address_example_cidr():
@ -1565,7 +1581,7 @@ def test_id_gen_recursive_dict_conversion_1():
},
)
assert file_observable.id == "file--5219d93d-13c1-5f1f-896b-039f10ec67ea"
assert file_observable.id == "file--ced31cd4-bdcb-537d-aefa-92d291bfc11d"
def test_id_gen_recursive_dict_conversion_2():
@ -1582,4 +1598,4 @@ def test_id_gen_recursive_dict_conversion_2():
],
)
assert wrko.id == "windows-registry-key--c087d9fe-a03e-5922-a1cd-da116e5b8a7b"
assert wrko.id == "windows-registry-key--36594eba-bcc7-5014-9835-0e154264e588"

2
stix2/test/v21/test_pattern_expressions.py
View File

@ -518,7 +518,7 @@ def test_invalid_boolean_constant():
@pytest.mark.parametrize(
"hashtype, data", [
('MD5', 'zzz'),
('ssdeep', 'zzz=='),
('SSDEEP', 'zzz=='),
],
)
def test_invalid_hash_constant(hashtype, data):

4
stix2/test/v21/test_properties.py
View File

@ -303,8 +303,6 @@ def test_reference_property_specific_type():
@pytest.mark.parametrize(
"value", [
'2017-01-01T12:34:56Z',
'2017-01-01 12:34:56',
'Jan 1 2017 12:34:56',
],
)
def test_timestamp_property_valid(value):
@ -314,7 +312,7 @@ def test_timestamp_property_valid(value):
def test_timestamp_property_invalid():
ts_prop = TimestampProperty()
with pytest.raises(ValueError):
with pytest.raises(TypeError):
ts_prop.clean(1)
with pytest.raises(ValueError):
ts_prop.clean("someday sometime")

12
stix2/test/v21/test_utils.py
View File

@ -35,8 +35,6 @@ def test_timestamp_formatting(dttm, timestamp):
(dt.datetime(2017, 1, 1, 0, tzinfo=pytz.utc), dt.datetime(2017, 1, 1, 0, 0, 0, tzinfo=pytz.utc)),
(dt.date(2017, 1, 1), dt.datetime(2017, 1, 1, 0, 0, 0, tzinfo=pytz.utc)),
('2017-01-01T00:00:00Z', dt.datetime(2017, 1, 1, 0, 0, 0, tzinfo=pytz.utc)),
('2017-01-01T02:00:00+2:00', dt.datetime(2017, 1, 1, 0, 0, 0, tzinfo=pytz.utc)),
('2017-01-01T00:00:00', dt.datetime(2017, 1, 1, 0, 0, 0, tzinfo=pytz.utc)),
],
)
def test_parse_datetime(timestamp, dttm):
@ -45,11 +43,11 @@ def test_parse_datetime(timestamp, dttm):
@pytest.mark.parametrize(
'timestamp, dttm, precision', [
('2017-01-01T01:02:03.000001', dt.datetime(2017, 1, 1, 1, 2, 3, 0, tzinfo=pytz.utc), 'millisecond'),
('2017-01-01T01:02:03.001', dt.datetime(2017, 1, 1, 1, 2, 3, 1000, tzinfo=pytz.utc), 'millisecond'),
('2017-01-01T01:02:03.1', dt.datetime(2017, 1, 1, 1, 2, 3, 100000, tzinfo=pytz.utc), 'millisecond'),
('2017-01-01T01:02:03.45', dt.datetime(2017, 1, 1, 1, 2, 3, 450000, tzinfo=pytz.utc), 'millisecond'),
('2017-01-01T01:02:03.45', dt.datetime(2017, 1, 1, 1, 2, 3, tzinfo=pytz.utc), 'second'),
('2017-01-01T01:02:03.000001Z', dt.datetime(2017, 1, 1, 1, 2, 3, 0, tzinfo=pytz.utc), 'millisecond'),
('2017-01-01T01:02:03.001Z', dt.datetime(2017, 1, 1, 1, 2, 3, 1000, tzinfo=pytz.utc), 'millisecond'),
('2017-01-01T01:02:03.1Z', dt.datetime(2017, 1, 1, 1, 2, 3, 100000, tzinfo=pytz.utc), 'millisecond'),
('2017-01-01T01:02:03.45Z', dt.datetime(2017, 1, 1, 1, 2, 3, 450000, tzinfo=pytz.utc), 'millisecond'),
('2017-01-01T01:02:03.45Z', dt.datetime(2017, 1, 1, 1, 2, 3, tzinfo=pytz.utc), 'second'),
],
)
def test_parse_datetime_precision(timestamp, dttm, precision):

7
stix2/utils.py
View File

@ -10,7 +10,6 @@ import enum
import json
import re
from dateutil import parser
import pytz
import six
@ -32,6 +31,9 @@ TYPE_REGEX = re.compile(r'^\-?[a-z0-9]+(-[a-z0-9]+)*\-?$')
TYPE_21_REGEX = re.compile(r'^([a-z][a-z0-9]*)+(-[a-z0-9]+)*\-?$')
PREFIX_21_REGEX = re.compile(r'^[a-z].*')
_TIMESTAMP_FORMAT = "%Y-%m-%dT%H:%M:%SZ"
_TIMESTAMP_FORMAT_FRAC = "%Y-%m-%dT%H:%M:%S.%fZ"
class Precision(enum.Enum):
"""
@ -252,8 +254,9 @@ def parse_into_datetime(
ts = dt.datetime.combine(value, dt.time(0, 0, tzinfo=pytz.utc))
else:
# value isn't a date or datetime object so assume it's a string
fmt = _TIMESTAMP_FORMAT_FRAC if "." in value else _TIMESTAMP_FORMAT
try:
parsed = parser.parse(value)
parsed = dt.datetime.strptime(value, fmt)
except (TypeError, ValueError):
# Unknown format
raise ValueError(