Merge branch 'main' of github.com:MISP/PyMISP into main

pull/803/head
Alexandre Dulaunoy 2021-10-26 11:43:36 +02:00
commit 772bb52760
No known key found for this signature in database
GPG Key ID: 09E2CD4944E6CBCD
12 changed files with 733 additions and 280 deletions

View File

@ -2,6 +2,74 @@ Changelog
=========
v2.4.148.1 (2021-09-30)
-----------------------
New
~~~
- Add few keys to email object creator. [Raphaël Vinot]
Fix #787
- Test cases for edit objects and upload stix. [Raphaël Vinot]
Changes
~~~~~~~
- Bump changelog. [Raphaël Vinot]
- Bump misp-objects. [Raphaël Vinot]
- Bump version. [Raphaël Vinot]
- Bump deps. [Raphaël Vinot]
- [doc] Minor fixes, note and typo. [Steve Clement]
- Bump deps. [Raphaël Vinot]
- [misp-objects] updated to the latest version. [Alexandre Dulaunoy]
- [misp-objects] updated to the latest version. [Alexandre Dulaunoy]
- Update tutorial for custom objects. [Raphaël Vinot]
- Bump deps. [Raphaël Vinot]
- Bump live tests. [Raphaël Vinot]
- [misp-objects] updated to the latest version. [Alexandre Dulaunoy]
- [types] updated types/categories mapping. [Christophe Vandeplas]
- Remove test files. [Raphaël Vinot]
- Automatically pull the malwares repo when running
tests/testlive_comprehensive.py. [Raphaël Vinot]
- Remove submodules with malware. [Raphaël Vinot]
- Add test for updating a objects from a custom template. [Raphaël
Vinot]
- Re-bump changelog. [Raphaël Vinot]
Fix
~~~
- Message_from_bytes really dislikes newline at the beginning of a mail.
[Raphaël Vinot]
- Skip IPs in Received header. [Raphaël Vinot]
- Name is passed to super. [Raphaël Vinot]
- Do not create empty manifest, json load dislikes it. [Raphaël Vinot]
- Initial round of cleanup on redis feed generator. [Raphaël Vinot]
- Upload of STIX document with non-ascii characters. [Raphaël Vinot]
Due to: https://github.com/psf/requests/issues/5560
TL;DR: a variable of type str passed to data in a POST request will be
silently re-encoded to ISO-8859-1, making MISP barf on the other side.
- Remove outdated deps from setup.py. [Raphaël Vinot]
Fix https://github.com/MISP/MISP/issues/7729
Other
~~~~~
- Remove unicode to ascii parts. [Sami Tainio]
- Fix #787 and add Unicode to ASCII function. [Sami Tainio]
Fix #787
- Uses regex to pick up the hostnames/domains from the "Received: from" headers.
Unicode to ASCII function
- Spam messages more often than not contain junk text as unicode characters in the headers. The "from" and "subject" headers being the most common ones. Before this change the script would error on such emails or sometimes replace the unicode characters with questionmarks "?".
- Function takes argument as an input and then encodes it in ascii while ignoring any malformed data. It then returns an ASCII string without the unicode characters.
- Currently implemented for "from" and "subject" handling.
- Update README.md. [Raphaël Vinot]
Not using travis anymore.
v2.4.148 (2021-08-05)
---------------------

View File

@ -1,4 +1,4 @@
**IMPORTANT NOTE**: This library will require **at least** python 3.6 starting the 1st of January 2020. If you have legacy versions of python, please use PyMISP v2.4.119.1, and consider updating your system(s). Anything released within the last 2 years will do, starting with Ubuntu 18.04.
**IMPORTANT NOTE**: This library will require **at least** python 3.8 starting the 1st of January 2022. If you have legacy versions of python, please use the latest PyMISP version that will be released in December 2021, and consider updating your system(s). Anything released within the last 2 years will do, starting with Ubuntu 20.04.
# PyMISP - Python Library to access MISP

View File

@ -158,7 +158,7 @@ class FeedGenerator:
self.create_daily_event()
def flush_event(self, new_event=None):
print('Writting event on disk' + ' ' * 50)
print('Writing event on disk' + ' ' * 50)
if new_event is not None:
event_uuid = new_event['uuid']
event = new_event

741
poetry.lock generated

File diff suppressed because it is too large Load Diff

View File

@ -33,7 +33,7 @@ try:
MISPNoticelist, MISPObjectTemplate, MISPSharingGroup, MISPRole, MISPServer, MISPFeed,
MISPEventDelegation, MISPUserSetting, MISPInbox, MISPEventBlocklist, MISPOrganisationBlocklist,
MISPEventReport, MISPGalaxyCluster, MISPGalaxyClusterElement, MISPGalaxyClusterRelation,
MISPCorrelationExclusion)
MISPCorrelationExclusion, MISPGalaxy)
from .tools import AbstractMISPObjectGenerator # noqa
from .tools import Neo4j # noqa
from .tools import stix # noqa

View File

@ -1921,6 +1921,21 @@ class PyMISP:
to_return.append(s)
return to_return
def get_sharing_group(self, sharing_group: Union[MISPSharingGroup, int, str, UUID], pythonify: bool = False) -> Union[Dict, MISPSharingGroup]:
"""Get a sharing group
:param sharing_group: sharing group to find
:param pythonify: Returns a PyMISP Object instead of the plain json output
"""
sharing_group_id = get_uuid_or_id_from_abstract_misp(sharing_group)
r = self._prepare_request('GET', f'sharing_groups/view/{sharing_group_id}')
sharing_group_resp = self._check_json_response(r)
if not (self.global_pythonify or pythonify) or 'errors' in sharing_group_resp:
return sharing_group_resp
s = MISPSharingGroup()
s.from_dict(**sharing_group_resp)
return s
def add_sharing_group(self, sharing_group: MISPSharingGroup, pythonify: bool = False) -> Union[Dict, MISPSharingGroup]:
"""Add a new sharing group
@ -2030,13 +2045,18 @@ class PyMISP:
# ## BEGIN Organisation ###
def organisations(self, scope="local", pythonify: bool = False) -> Union[Dict, List[MISPOrganisation]]:
def organisations(self, scope="local", search: str = None, pythonify: bool = False) -> Union[Dict, List[MISPOrganisation]]:
"""Get all the organisations
:param scope: scope of organizations to get
:param search: The search to make against the list of organisations
:param pythonify: Returns a list of PyMISP Objects instead of the plain json output. Warning: it might use a lot of RAM
"""
r = self._prepare_request('GET', f'organisations/index/scope:{scope}')
url_path = f'organisations/index/scope:{scope}'
if search:
url_path += f"/searchall:{search}"
r = self._prepare_request('GET', url_path)
organisations = self._check_json_response(r)
if not (self.global_pythonify or pythonify) or 'errors' in organisations:
return organisations
@ -2118,12 +2138,20 @@ class PyMISP:
# ## BEGIN User ###
def users(self, pythonify: bool = False) -> Union[Dict, List[MISPUser]]:
"""Get all the users
def users(self, search: str = None, organisation: int = None, pythonify: bool = False) -> Union[Dict, List[MISPUser]]:
"""Get all the users, or a filtered set of users
:param search: The search to make against the list of users
:param organisation: The ID of an organisation to filter against
:param pythonify: Returns a list of PyMISP Objects instead of the plain json output. Warning: it might use a lot of RAM
"""
r = self._prepare_request('GET', 'admin/users/index')
urlpath = 'admin/users/index'
if search:
urlpath += f'/value:{search}'
if organisation:
organisation_id = get_uuid_or_id_from_abstract_misp(organisation)
urlpath += f"/searchorg:{organisation_id}"
r = self._prepare_request('GET', urlpath)
users = self._check_json_response(r)
if not (self.global_pythonify or pythonify) or 'errors' in users:
return users

View File

@ -986,6 +986,10 @@
"default_category": "Person",
"to_ids": 0
},
"filename-pattern": {
"default_category": "Payload installation",
"to_ids": 1
},
"pattern-in-file": {
"default_category": "Payload installation",
"to_ids": 1
@ -1391,6 +1395,7 @@
"passport-country",
"passport-expiration",
"passport-number",
"filename-pattern",
"pattern-in-file",
"pattern-in-memory",
"pattern-in-traffic",

@ -1 +1 @@
Subproject commit c8cd002a3be424531ef9ceadf00742f98820733b
Subproject commit c380279dcadb6d9e814cf8806a1c164eeeef5133

View File

@ -126,12 +126,41 @@ class MISPOrganisation(AbstractMISP):
class MISPSharingGroup(AbstractMISP):
def __init__(self):
super().__init__()
self.name: str
self.SharingGroupOrg: List[MISPOrganisation] = []
@property
def orgs(self) -> List[MISPOrganisation]:
return self.SharingGroupOrg
@orgs.setter
def orgs(self, orgs: List[MISPOrganisation]):
"""Set a list of prepared MISPSighting."""
if all(isinstance(x, MISPSighting) for x in orgs):
self.SharingGroupOrg = orgs
else:
raise PyMISPError('All the attributes have to be of type MISPOrganisation.')
def add_org(self, org):
misp_org = MISPOrganisation()
misp_org.from_dict(**org)
self.SharingGroupOrg.append(misp_org)
return misp_org
def from_dict(self, **kwargs):
if 'SharingGroupOrg' in kwargs:
[self.add_org(org) for org in kwargs.pop('SharingGroupOrg')]
if 'SharingGroup' in kwargs:
kwargs = kwargs['SharingGroup']
super().from_dict(**kwargs)
def __repr__(self) -> str:
if hasattr(self, 'name'):
return f'<{self.__class__.__name__}(name={self.name})'
return f'<{self.__class__.__name__}(NotInitialized)'
class MISPShadowAttribute(AbstractMISP):

View File

@ -45,7 +45,7 @@ class EMailObject(AbstractMISPObjectGenerator):
def parse_email(self) -> EmailMessage:
"""Convert email into EmailMessage."""
content_in_bytes = self.__pseudofile.getvalue()
content_in_bytes = self.__pseudofile.getvalue().strip()
eml = message_from_bytes(content_in_bytes,
_class=EmailMessage,
policy=policy.default)
@ -317,12 +317,6 @@ class EMailObject(AbstractMISPObjectGenerator):
if "Thread-Index" in message:
self.add_attribute("thread-index", message["Thread-Index"])
if "Received" in message:
try:
self.add_attribute("received-header-hostname", message['Received'].split(' ')[1])
except Exception:
pass
self.__generate_received()
def __add_emails(self, typ: str, data: str, insert_display_names: bool = True):
@ -351,7 +345,7 @@ class EMailObject(AbstractMISPObjectGenerator):
def __generate_received(self):
"""
Extract IP addresses from received headers that are not private.
Extract IP addresses from received headers that are not private. Also extract hostnames or domains.
"""
received_items = self.email.get_all("received")
if received_items is None:
@ -375,3 +369,11 @@ class EMailObject(AbstractMISPObjectGenerator):
continue # skip header if IP not found or is private
self.add_attribute("received-header-ip", value=str(ip), comment=fromstr)
# The hostnames and/or domains always come after the "Received: from"
# part so we can use regex to pick up those attributes.
received_from = re.findall(r'(?<=from\s)[\w\d\.\-]+\.\w{2,24}', str(received_items))
try:
[self.add_attribute("received-header-hostname", i) for i in received_from]
except Exception:
pass

View File

@ -1,6 +1,6 @@
[tool.poetry]
name = "pymisp"
version = "2.4.148"
version = "2.4.148.1"
description = "Python API for MISP."
authors = ["Raphaël Vinot <raphael.vinot@circl.lu>"]
license = "BSD-2-Clause"
@ -41,24 +41,24 @@ include = [
"Source" = "https://github.com/MISP/PyMISP"
[tool.poetry.dependencies]
python = "^3.6"
requests = "^2.25.1"
python-dateutil = "^2.8.1"
python = "^3.6.2"
requests = "^2.26.0"
python-dateutil = "^2.8.2"
jsonschema = "^3.2.0"
deprecated = "^1.2.12"
deprecated = "^1.2.13"
extract_msg = {version = "^0.28.7", optional = true}
RTFDE = {version = "^0.0.2", optional = true}
oletools = {version = "^0.56.1", optional = true}
python-magic = {version = "^0.4.22", optional = true}
oletools = {version = "^0.60", optional = true}
python-magic = {version = "^0.4.24", optional = true}
pydeep = {version = "^0.4", optional = true}
lief = {version = "^0.11.4", optional = true}
beautifulsoup4 = {version = "^4.9.3", optional = true}
lief = {version = "^0.11.5", optional = true}
beautifulsoup4 = {version = "^4.10.0", optional = true}
validators = {version = "^0.18.2", optional = true}
sphinx-autodoc-typehints = {version = "^1.12.0", optional = true}
recommonmark = {version = "^0.7.1", optional = true}
reportlab = {version = "^3.5.67", optional = true}
reportlab = {version = "^3.6.2", optional = true}
pyfaup = {version = "^1.2", optional = true}
urllib3 = {extras = ["brotli"], version = "^1.26.4", optional = true}
urllib3 = {extras = ["brotli"], version = "^1.26.7", optional = true}
[tool.poetry.extras]
@ -73,17 +73,17 @@ brotli = ['urllib3']
[tool.poetry.dev-dependencies]
nose = "^1.3.7"
coveralls = "^3.0.1"
codecov = "^2.1.11"
requests-mock = "^1.8.0"
mypy = "^0.902"
flake8 = "^3.9.0"
coveralls = "^3.2.0"
codecov = "^2.1.12"
requests-mock = "^1.9.3"
mypy = "^0.910"
flake8 = "^4.0.1"
ipython = "^7.16.1"
jupyterlab = "^2.3.1"
types-requests = "^2.25.0"
types-python-dateutil = "^0.1.4"
types-redis = "^3.5.4"
types-Flask = "^1.1.1"
jupyterlab = "^3.2"
types-requests = "^2.25.10"
types-python-dateutil = "^2.8.1"
types-redis = "^3.5.12"
types-Flask = "^1.1.4"
[build-system]
requires = ["poetry_core>=1.0", "setuptools"]

View File

@ -1176,7 +1176,7 @@ class TestComprehensive(unittest.TestCase):
try:
first = self.user_misp_connector.add_event(first)
stix = self.user_misp_connector.search(return_format='stix', eventid=first.id)
self.assertTrue(stix['related_packages'][0]['package']['incidents'][0]['related_indicators']['indicators'][0]['indicator']['observable']['object']['properties']['address_value']['value'], '8.8.8.8')
self.assertTrue(stix['related_packages']['related_packages'][0]['package']['incidents'][0]['related_indicators']['indicators'][0]['indicator']['observable']['object']['properties']['address_value']['value'], '8.8.8.8')
stix2 = self.user_misp_connector.search(return_format='stix2', eventid=first.id)
self.assertEqual(stix2['objects'][-1]['pattern'], "[network-traffic:src_ref.type = 'ipv4-addr' AND network-traffic:src_ref.value = '8.8.8.8']")
stix_xml = self.user_misp_connector.search(return_format='stix-xml', eventid=first.id)
@ -1771,6 +1771,33 @@ class TestComprehensive(unittest.TestCase):
organisation = self.admin_misp_connector.update_organisation(organisation, pythonify=True)
self.assertEqual(organisation.name, 'blah', organisation)
def test_org_search(self):
orgs = self.admin_misp_connector.organisations(pythonify=True)
org_name = 'ORGNAME'
# Search by the org name
orgs = self.admin_misp_connector.organisations(search=org_name, pythonify=True)
# There should be one org returned
self.assertTrue(len(orgs) == 1)
# This org should have the name ORGNAME
self.assertEqual(orgs[0].name, org_name)
def test_user_search(self):
users = self.admin_misp_connector.users(pythonify=True)
emailAddr = users[0].email
users = self.admin_misp_connector.users(search=emailAddr)
self.assertTrue(len(users) == 1)
self.assertEqual(users[0]['User']['email'], emailAddr)
users = self.admin_misp_connector.users(
search=emailAddr,
organisation=users[0]['Organisation']['id'],
pythonify=True
)
self.assertTrue(len(users) == 1)
self.assertEqual(users[0].email, emailAddr)
def test_attribute(self):
first = self.create_simple_event()
second = self.create_simple_event()
@ -1988,15 +2015,19 @@ class TestComprehensive(unittest.TestCase):
remote_types = remote.pop('types')
remote_categories = remote.pop('categories')
remote_category_type_mappings = remote.pop('category_type_mappings')
local = dict(self.admin_misp_connector.describe_types_local)
local_types = local.pop('types')
local_categories = local.pop('categories')
local_category_type_mappings = local.pop('category_type_mappings')
self.assertDictEqual(remote, local)
self.assertEqual(sorted(remote_types), sorted(local_types))
self.assertEqual(sorted(remote_categories), sorted(local_categories))
for category, mapping in remote_category_type_mappings.items():
self.assertEqual(sorted(local_category_type_mappings[category]), sorted(mapping))
for typ in mapping:
self.assertIn(typ, remote_types)
def test_versions(self):
self.assertEqual(self.user_misp_connector.version, self.user_misp_connector.pymisp_version_master)
@ -2177,6 +2208,30 @@ class TestComprehensive(unittest.TestCase):
self.assertFalse(self.admin_misp_connector.sharing_group_exists(sharing_group.id))
self.assertFalse(self.admin_misp_connector.sharing_group_exists(sharing_group.uuid))
def test_sharing_group(self):
# add
sg = MISPSharingGroup()
sg.name = 'Testcases SG'
sg.releasability = 'Testing'
sharing_group = self.admin_misp_connector.add_sharing_group(sg, pythonify=True)
# Add the org to the sharing group
self.admin_misp_connector.add_org_to_sharing_group(
sharing_group,
self.test_org, extend=True
)
try:
# Get the sharing group once again
sharing_group = self.admin_misp_connector.get_sharing_group(sharing_group, pythonify=True)
self.assertTrue(isinstance(sharing_group, MISPSharingGroup))
self.assertEqual(sharing_group.name, 'Testcases SG')
# Check we have the org field present and the first org is our org
self.assertTrue(isinstance(getattr(sharing_group, "orgs"), list))
self.assertEqual(sharing_group.orgs[0].id, self.test_org.id)
finally:
self.admin_misp_connector.delete_sharing_group(sharing_group.id)
def test_feeds(self):
# Add
feed = MISPFeed()
@ -2514,7 +2569,6 @@ class TestComprehensive(unittest.TestCase):
# FIXME https://github.com/MISP/MISP/issues/4892
try:
r1 = self.user_misp_connector.upload_stix('tests/stix1.xml-utf8', version='1')
print(r1.text)
event_stix_one = MISPEvent()
event_stix_one.load(r1.json())
# self.assertEqual(event_stix_one.attributes[0], '8.8.8.8')
@ -2523,10 +2577,8 @@ class TestComprehensive(unittest.TestCase):
self.assertTrue(bl['success'])
r2 = self.user_misp_connector.upload_stix('tests/stix2.json', version='2')
print(json.dumps(r2.json(), indent=2))
event_stix_two = MISPEvent()
event_stix_two.load(r2.json())
print(event_stix_two.to_json(indent=2))
# FIXME: the response is buggy.
# self.assertEqual(event_stix_two.attributes[0], '8.8.8.8')
self.admin_misp_connector.delete_event(event_stix_two)
@ -2751,7 +2803,7 @@ class TestComprehensive(unittest.TestCase):
else:
raise Exception('Unable to find UUID in Events blocklist')
first = self.user_misp_connector.add_event(first, pythonify=True)
self.assertEqual(first['errors'][1]['message'], 'Could not add Event', first)
self.assertEqual(first['errors'][1]['message'], 'Event blocked by event blocklist.', first)
ble.comment = 'This is a test'
ble.event_info = 'foo'
ble.event_orgc = 'bar'
@ -2771,7 +2823,7 @@ class TestComprehensive(unittest.TestCase):
else:
raise Exception('Unable to find UUID in Orgs blocklist')
first = self.user_misp_connector.add_event(first, pythonify=True)
self.assertEqual(first['errors'][1]['message'], 'Could not add Event', first)
self.assertEqual(first['errors'][1]['message'], 'Event blocked by organisation blocklist.', first)
blo.comment = 'This is a test'
blo.org_name = 'bar'