MISP
/
PyMISP
mirror of https://github.com/MISP/PyMISP
2
0
Fork 0
Code Issues Releases Wiki Activity

Merge branch 'main' of github.com:misp/pymisp

main
Christian Studer 2024-05-20 11:56:44 +10:00
parent afb1c8f2d0 a74dd078a6
commit 3edad4e735
No known key found for this signature in database
GPG Key ID: 6BBED1B63A6D639F
10 changed files with 685 additions and 670 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

100
CHANGELOG.txt
View File

@ -2,11 +2,111 @@ Changelog
=========
v2.4.190 (2024-04-18)
---------------------
Changes
~~~~~~~
- Bump object templates. [Raphaël Vinot]
- Bump changelog. [Raphaël Vinot]
- Bump version, deps. [Raphaël Vinot]
- Bump deps, require python 3.9+ for doc. [Raphaël Vinot]
- Bump deps. [Raphaël Vinot]
- [data] describeTypes file updated. [Alexandre Dulaunoy]
- Bump deps. [Raphaël Vinot]
Fix
~~~
- [internal] Correct way to convert bytes to string if orjson exists.
[Jakub Onderka]
v2.4.188 (2024-03-22)
---------------------
New
~~~
- Support X-MISP-AUTH Header. [Raphaël Vinot]
Also, improve HTTP headers init
Fix #1179
Changes
~~~~~~~
- Bump changelog. [Raphaël Vinot]
- Bump changelog. [Raphaël Vinot]
- Bump deps. [Raphaël Vinot]
- Bump changelog. [Raphaël Vinot]
- Bump version, templates. [Raphaël Vinot]
- Bump deps. [Raphaël Vinot]
Fix
~~~
- Strip API key before setting it. [Raphaël Vinot]
- Python 3.8 support & typing. [Raphaël Vinot]
- Typing for Python < 3.10. [Raphaël Vinot]
- Avoid issue when payload ist a list. [Raphaël Vinot]
v2.4.187 (2024-03-07)
---------------------
Changes
~~~~~~~
- Bump changelog. [Raphaël Vinot]
- Bump templates, version. [Raphaël Vinot]
- Bump deps. [Raphaël Vinot]
- Bump extract-msg. [Raphaël Vinot]
v2.4.186 (2024-02-27)
---------------------
Changes
~~~~~~~
- Bump changelog. [Raphaël Vinot]
- Bump changelog. [Raphaël Vinot]
- Bump version. [Raphaël Vinot]
- Bump objects. [Raphaël Vinot]
- Bump deps. [Raphaël Vinot]
- Bump deps. [Raphaël Vinot]
Fix
~~~
- Correct FileObject import. [Johannes Bader]
The FileObject import has been moved outside the try-except-block
related to lief, as the import is needed regardless whether lief
is available or not.
- Disable WL when calling the disable method, not toggle. [Raphaël
Vinot]
Fix #1159
Other
~~~~~
- Build(deps): bump urllib3 from 2.2.0 to 2.2.1. [dependabot[bot]]
Bumps [urllib3](https://github.com/urllib3/urllib3) from 2.2.0 to 2.2.1.
- [Release notes](https://github.com/urllib3/urllib3/releases)
- [Changelog](https://github.com/urllib3/urllib3/blob/main/CHANGES.rst)
- [Commits](https://github.com/urllib3/urllib3/compare/2.2.0...2.2.1)
---
updated-dependencies:
- dependency-name: urllib3
dependency-type: direct:production
update-type: version-update:semver-patch
...
v2.4.185 (2024-02-16)
---------------------
Changes
~~~~~~~
- Bump changelog. [Raphaël Vinot]
- Bump deps, version. [Raphaël Vinot]

1151
poetry.lock generated
View File

File diff suppressed because it is too large Load Diff

2
pymisp/abstract.py
View File

@ -252,7 +252,7 @@ class AbstractMISP(MutableMapping, MISPFileCache, metaclass=ABCMeta): # type: i
option |= orjson.OPT_INDENT_2
# orjson dumps method returns bytes instead of bytes, to keep compatibility with json
# we have to convert output to str
return str(dumps(self, default=pymisp_json_default, option=option))
return dumps(self, default=pymisp_json_default, option=option).decode()
return dumps(self, default=pymisp_json_default, sort_keys=sort_keys, indent=indent)

31
pymisp/api.py
View File

@ -57,6 +57,7 @@ except ImportError:
SearchType = TypeVar('SearchType', str, int)
# str: string to search / list: values to search (OR) / dict: {'OR': [list], 'NOT': [list], 'AND': [list]}
# NOTE: we cannot use new typing here until we drop Python 3.8 and 3.9 support
SearchParameterTypes = TypeVar('SearchParameterTypes', str, List[Union[str, int]], Dict[str, Union[str, int]])
ToIDSType = TypeVar('ToIDSType', str, int, bool)
@ -158,6 +159,7 @@ class PyMISP:
:param tool: The software using PyMISP (string), used to set a unique user-agent
:param http_headers: Arbitrary headers to pass to all the requests.
:param https_adapter: Arbitrary HTTPS adapter for the requests session.
:param http_auth_header_name: The name of the HTTP header to use for the API key. Can be either "Authorization" or "X-MISP-AUTH".
:param timeout: Timeout, as described here: https://requests.readthedocs.io/en/master/user/advanced/#timeouts
"""
@ -165,7 +167,8 @@ class PyMISP:
cert: str | tuple[str, str] | None = None, auth: AuthBase | None = None, tool: str = '',
timeout: float | tuple[float, float] | None = None,
http_headers: dict[str, str] | None = None,
https_adapter: requests.adapters.BaseAdapter | None = None
https_adapter: requests.adapters.BaseAdapter | None = None,
http_auth_header_name: str = 'Authorization'
):
if not url:
@ -174,21 +177,30 @@ class PyMISP:
raise NoKey('Please provide your authorization key.')
self.root_url: str = url
self.key: str = key
self.key: str = key.strip()
self.ssl: bool | str = ssl
self.proxies: MutableMapping[str, str] | None = proxies
self.cert: str | tuple[str, str] | None = cert
self.auth: AuthBase | None = auth
self.tool: str = tool
self.timeout: float | tuple[float, float] | None = timeout
self.__session = requests.Session() # use one session to keep connection between requests
if https_adapter is not None:
self.__session.mount('https://', https_adapter)
if brotli_supported():
self.__session.headers['Accept-Encoding'] = ', '.join(('br', 'gzip', 'deflate'))
if http_auth_header_name in ['Authorization', 'X-MISP-AUTH']:
self.__session.headers[http_auth_header_name] = self.key
else:
raise PyMISPError('http_auth_header_name should be either "Authorization" or "X-MISP-AUTH"')
user_agent = f'PyMISP {__version__} - Python {".".join(str(x) for x in sys.version_info[:2])}'
if tool:
user_agent = f'{user_agent} - {tool}'
self.__session.headers['User-Agent'] = user_agent
if http_headers:
self.__session.headers.update(http_headers)
self._user_agent = f'PyMISP {__version__} - Python {".".join(str(x) for x in sys.version_info[:2])}'
self.global_pythonify = False
@ -1542,7 +1554,7 @@ class PyMISP:
"""
galaxy_id = get_uuid_or_id_from_abstract_misp(galaxy)
allowed_context_types: list[str] = ["all", "default", "custom", "org", "deleted"]
allowed_context_types: list[str] = ["all", "default", "custom", "org", "orgc", "deleted"]
if context not in allowed_context_types:
raise PyMISPError(f"The context must be one of {', '.join(allowed_context_types)}")
kw_params = {"context": context}
@ -3708,7 +3720,7 @@ class PyMISP:
def _check_response(self, response: requests.Response, lenient_response_type: bool = False, expect_json: bool = False) -> dict[str, Any] | str:
"""Check if the response from the server is not an unexpected error"""
if response.status_code >= 500:
headers_without_auth = {i: response.request.headers[i] for i in response.request.headers if i != 'Authorization'}
headers_without_auth = {h_name: h_value for h_name, h_value in response.request.headers.items() if h_value != self.key}
logger.critical(everything_broken.format(headers_without_auth, response.request.body, response.text))
raise MISPServerError(f'Error code 500:\n{response.text}')
@ -3778,14 +3790,11 @@ class PyMISP:
url = f'{url}/{to_append_url}'
req = requests.Request(request_type, url, data=d, params=params)
user_agent = f'{self._user_agent} - {self.tool}' if self.tool else self._user_agent
req.auth = self.auth
prepped = self.__session.prepare_request(req)
prepped.headers.update(
{'Authorization': self.key,
'Accept': f'application/{output_type}',
'content-type': f'application/{content_type}',
'User-Agent': user_agent})
{'Accept': f'application/{output_type}',
'content-type': f'application/{content_type}'})
logger.debug(prepped.headers)
settings = self.__session.merge_environment_settings(req.url, proxies=self.proxies or {}, stream=None,
verify=self.ssl, cert=self.cert)

6
pymisp/data/describeTypes.json
View File

@ -545,6 +545,10 @@
"default_category": "Other",
"to_ids": 0
},
"integer": {
"default_category": "Other",
"to_ids": 0
},
"datetime": {
"default_category": "Other",
"to_ids": 0
@ -891,6 +895,7 @@
"dns-soa-email",
"size-in-bytes",
"counter",
"integer",
"datetime",
"port",
"ip-dst|port",
@ -1460,6 +1465,7 @@
"other",
"size-in-bytes",
"counter",
"integer",
"datetime",
"cpe",
"port",

2
pymisp/data/misp-objects

@ -1 +1 @@
Subproject commit 3ac509965fdbca06d8a027db22c0064588babd3c
Subproject commit 96492b9c932a4b307216550abeadddc727e17cec

3
pymisp/tools/create_misp_object.py
View File

@ -8,6 +8,7 @@ from io import BytesIO
from typing import Any, TYPE_CHECKING
from ..exceptions import MISPObjectException
from . import FileObject
logger = logging.getLogger('pymisp')
try:
@ -19,8 +20,6 @@ try:
from .peobject import make_pe_objects
from .elfobject import make_elf_objects
from .machoobject import make_macho_objects
from . import FileObject
except AttributeError:
HAS_LIEF = False
logger.critical('You need lief >= 0.11.0. The quick and dirty fix is: pip3 install --force pymisp[fileobjects]')

18
pymisp/tools/emailobject.py
View File

@ -196,11 +196,19 @@ class EMailObject(AbstractMISPObjectGenerator):
for mime_items in related_content.values():
if isinstance(mime_items[1], dict):
message.add_related(**mime_items[1])
cur_attach = message.get_payload()[-1]
if p := message.get_payload():
if isinstance(p, list):
cur_attach = p[-1]
else:
cur_attach = p
self._update_content_disp_properties(mime_items[0], cur_attach)
if body.get('text', None):
# Now add the HTML as an alternative within the related obj
related = message.get_payload()[0]
if p := message.get_payload():
if isinstance(p, list):
related = p[0]
else:
related = p
related.add_alternative(**body.get('html'))
else:
for mime_dict in body_objects:
@ -219,7 +227,11 @@ class EMailObject(AbstractMISPObjectGenerator):
subtype=subtype,
cid=attch.cid,
filename=attch.longFilename)
cur_attach = message.get_payload()[-1]
if p := message.get_payload():
if isinstance(p, list):
cur_attach = p[-1]
else:
cur_attach = p
self._update_content_disp_properties(attch, cur_attach)
if _orig_boundry is not None:
message.set_boundary(_orig_boundry) # Set back original boundary

2
pymisp/tools/vtreportobject.py
View File

@ -7,7 +7,7 @@ from typing import Any
import requests
try:
import validators # type: ignore
import validators
has_validators = True
except ImportError:
has_validators = False

38
pyproject.toml
View File

@ -1,6 +1,6 @@
[tool.poetry]
name = "pymisp"
version = "2.4.185"
version = "2.4.190"
description = "Python API for MISP."
authors = ["Raphaël Vinot <raphael.vinot@circl.lu>"]
license = "BSD-2-Clause"
@ -43,51 +43,49 @@ include = [
[tool.poetry.dependencies]
python = "^3.8"
requests = "^2.31.0"
python-dateutil = "^2.8.2"
python-dateutil = "^2.9.0.post0"
deprecated = "^1.2.14"
extract_msg = {version = "^0.47.0", optional = true}
extract_msg = {version = "^0.48.5", optional = true}
RTFDE = {version = "^0.1.1", optional = true}
oletools = {version = "^0.60.1", optional = true}
python-magic = {version = "^0.4.27", optional = true}
pydeep2 = {version = "^0.5.1", optional = true}
lief = {version = "^0.14.1", optional = true}
beautifulsoup4 = {version = "^4.12.3", optional = true}
validators = {version = "^0.22.0", optional = true}
sphinx-autodoc-typehints = {version = "^2.0.0", optional = true}
recommonmark = {version = "^0.7.1", optional = true}
reportlab = {version = "^4.1.0", optional = true}
validators = {version = "^0.28.0", optional = true}
sphinx-autodoc-typehints = {version = "^2.1.0", optional = true, python = ">=3.9"}
docutils = {version = "^0.21.1", optional = true, python = ">=3.9"}
recommonmark = {version = "^0.7.1", optional = true, python = ">=3.9"}
reportlab = {version = "^4.2.0", optional = true}
pyfaup = {version = "^1.2", optional = true}
publicsuffixlist = {version = "^0.10.0.20231214", optional = true}
publicsuffixlist = {version = "^0.10.0.20240403", optional = true}
urllib3 = {extras = ["brotli"], version = "*", optional = true}
Sphinx = [
{version = "<7.2", python = "<3.9", optional = true},
{version = "^7.2", python = ">=3.9", optional = true}
]
Sphinx = {version = "^7.3.7", python = ">=3.9", optional = true}
[tool.poetry.extras]
fileobjects = ['python-magic', 'pydeep2', 'lief']
openioc = ['beautifulsoup4']
virustotal = ['validators']
docs = ['sphinx-autodoc-typehints', 'recommonmark', 'sphinx']
docs = ['sphinx-autodoc-typehints', 'recommonmark', 'sphinx', 'docutils']
pdfexport = ['reportlab']
url = ['pyfaup']
email = ['extract_msg', "RTFDE", "oletools"]
brotli = ['urllib3']
[tool.poetry.group.dev.dependencies]
requests-mock = "^1.11.0"
mypy = "^1.8.0"
requests-mock = "^1.12.1"
mypy = "^1.10.0"
ipython = [
{version = "<8.13.0", python = "<3.9"},
{version = "^8.18.0", python = ">=3.9"},
{version = "^8.19.0", python = ">=3.10"}
]
jupyterlab = "^4.1.2"
types-requests = "^2.31.0.20240218"
types-python-dateutil = "^2.8.19.20240106"
types-redis = "^4.6.0.20240218"
jupyterlab = "^4.1.7"
types-requests = "^2.31.0.20240406"
types-python-dateutil = "^2.9.0.20240316"
types-redis = "^4.6.0.20240425"
types-Flask = "^1.1.6"
pytest-cov = "^4.1.0"
pytest-cov = "^5.0.0"
[build-system]
requires = ["poetry_core>=1.1", "setuptools"]