MISP
/
PyMISP
mirror of https://github.com/MISP/PyMISP
2
0
Fork 0
Code Issues Releases Wiki Activity

Merge branch 'main' of github.com:misp/pymisp

pull/1002/head
Christian Studer 2023-05-26 14:28:41 +02:00
parent ea67c171f7 3061912de8
commit 9fcaea8be1
19 changed files with 956 additions and 996 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.github/workflows/pytest.yml vendored
View File

@ -11,6 +11,7 @@ jobs:
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
python-version: [3.8, 3.9, '3.10', '3.11']

127
CHANGELOG.txt
View File

@ -2,11 +2,138 @@ Changelog
=========
v2.4.171 (2023-05-16)
---------------------
Changes
~~~~~~~
- Bump deps, object templates. [Raphaël Vinot]
- Bump changelog. [Raphaël Vinot]
- Bump changelog. [Raphaël Vinot]
- Bump deps. [Raphaël Vinot]
- Bump changelog. [Raphaël Vinot]
- Bump version. [Raphaël Vinot]
- Remove old setup files, bump deps. [Raphaël Vinot]
Fix
~~~
- Extra print breaking the CI on MISP side. [Raphaël Vinot]
- Properly use lief on a file. [Raphaël Vinot]
Other
~~~~~
- Allow search by 'event_tags' and improve the handling of galaxy
clusters. [Stefano Ortolani]
Changes:
- Add 'event_tags' parameter when searching for events
- Add new method to search for galaxies by value
- Add new parameter to fetch cluster information when retrieving clusters
- Add new parameter to hard-delete object references
- Using underscore name 'description_file' in setup.cfg. [Erhan]
Usage of dash-separated 'description-file' will not be supported in future versions. Please use the underscore name 'description_file' instead. By 2023-Sep-26, you need to update your project and remove deprecated calls or your builds will no longer be supported.
See https://setuptools.pypa.io/en/latest/userguide/declarative_config.html for details.
v2.4.170.2 (2023-05-04)
-----------------------
Changes
~~~~~~~
- Bump changelog. [Raphaël Vinot]
- Bump version. [Raphaël Vinot]
- Bump deps. [Raphaël Vinot]
- Bump deps. [Raphaël Vinot]
v2.4.170.1 (2023-04-19)
-----------------------
Changes
~~~~~~~
- Disable fail fast in GHA. [Raphaël Vinot]
- Bump version. [Raphaël Vinot]
- Bump deps. [Raphaël Vinot]
Fix
~~~
- Update lief code to v0.13. [Raphaël Vinot]
v2.4.170 (2023-04-12)
---------------------
Changes
~~~~~~~
- Bump changelog. [Raphaël Vinot]
- Bump version. [Raphaël Vinot]
- Bump deps. [Raphaël Vinot]
Other
~~~~~
- Add: support breakOnDuplicate option for attributes:add() [Luciano
Righetti]
- Update reportlab_generator.py. [CarlosLoureiro]
v2.4.169.3 (2023-03-27)
-----------------------
Changes
~~~~~~~
- Bump changelog. [Raphaël Vinot]
- Bump deps, version. [Raphaël Vinot]
- Bump deps. [Raphaël Vinot]
Fix
~~~
- Invalid check if taxo is enabled. [Raphaël Vinot]
v2.4.169.2 (2023-03-17)
-----------------------
Changes
~~~~~~~
- Bump changelog. [Raphaël Vinot]
- Bump version. [Raphaël Vinot]
- Include event reports by default in feed. [Raphaël Vinot]
- Bump deps. [Raphaël Vinot]
Fix
~~~
- Use proper parameter to trigger the request in search_galaxy_clusters.
[Raphaël Vinot]
- Use POST in search galaxy cluster. [Raphaël Vinot]
Other
~~~~~
- Rename include_event_reports kwarg to with_event_reports, in-line with
other kwarg naming. [UFOSmuggler]
- Add kwarg to allow the inclusion of event reports into to_feed(),
honour with_distribution and valid_distributions kwargs. [UFOSmuggler]
v2.4.169.1 (2023-03-14)
-----------------------
Changes
~~~~~~~
- Bump changelog. [Raphaël Vinot]
- Bump version. [Raphaël Vinot]
- Add greynoise-ip object. [Raphaël Vinot]
Fix #951
v2.4.169 (2023-03-10)
---------------------
Changes
~~~~~~~
- Bump changelog. [Raphaël Vinot]
- Bump version. [Raphaël Vinot]
- Bump templates. [Raphaël Vinot]
- Bump deps. [Raphaël Vinot]

2
README.md
View File

@ -46,7 +46,7 @@ pip3 install pymisp[virustotal,email]
```
git clone https://github.com/MISP/PyMISP.git && cd PyMISP
git submodule update --init
poetry install -E fileobjects -E openioc -E virustotal -E docs -E pdfexport
poetry install -E fileobjects -E openioc -E virustotal -E docs -E pdfexport -E email
```
### Running the tests

4
examples/feed-generator/settings.default.py
View File

@ -16,7 +16,7 @@ outputdir = 'output'
# you can use on the event index, such as organisation, tags, etc.
# It uses the same joining and condition rules as the API parameters
# For example:
# filters = {'tag':'tlp:white|feed-export|!privint','org':'CIRCL', 'published':1}
# filters = {'tags':['tlp:white','feed-export','!privint'],'org':'CIRCL', 'published':1}
# the above would generate a feed for all published events created by CIRCL,
# tagged tlp:white and/or feed-export but exclude anything tagged privint
filters = {'published':'true'}
@ -53,4 +53,4 @@ exclude_attribute_types = []
with_distribution = False
# Include the exportable local tags along with the global tags. The default is True.
with_local_tags = True
with_local_tags = True

1493
poetry.lock generated
View File

File diff suppressed because it is too large Load Diff

2
pymisp/__init__.py
View File

@ -1,4 +1,4 @@
__version__ = '2.4.169'
__version__ = '2.4.171'
import logging
import sys
import warnings

65
pymisp/api.py
View File

@ -588,7 +588,7 @@ class PyMISP:
:param break_on_duplicate: if True, check and reject if this object's attributes match an existing object's attributes; may require much time
"""
event_id = get_uuid_or_id_from_abstract_misp(event)
params = {'breakOnDuplicate': True} if break_on_duplicate else {}
params = {'breakOnDuplicate': 1} if break_on_duplicate else {}
r = self._prepare_request('POST', f'objects/add/{event_id}', data=misp_object, kw_params=params)
new_object = self._check_json_response(r)
if not (self.global_pythonify or pythonify) or 'errors' in new_object:
@ -643,13 +643,17 @@ class PyMISP:
ref.from_dict(**object_reference)
return ref
def delete_object_reference(self, object_reference: Union[MISPObjectReference, int, str, UUID]) -> Dict:
"""Delete a reference to an object
:param object_reference: object reference
"""
def delete_object_reference(
self,
object_reference: Union[MISPObjectReference, int, str, UUID],
hard: bool = False,
) -> Dict:
"""Delete a reference to an object."""
object_reference_id = get_uuid_or_id_from_abstract_misp(object_reference)
response = self._prepare_request('POST', f'objectReferences/delete/{object_reference_id}')
query_url = f"objectReferences/delete/{object_reference_id}"
if hard:
query_url += "/true"
response = self._prepare_request("POST", query_url)
return self._check_json_response(response)
# Object templates
@ -741,7 +745,7 @@ class PyMISP:
r = self._prepare_request('HEAD', f'attributes/view/{attribute_id}')
return self._check_head_response(r)
def add_attribute(self, event: Union[MISPEvent, int, str, UUID], attribute: Union[MISPAttribute, Iterable], pythonify: bool = False) -> Union[Dict, MISPAttribute, MISPShadowAttribute]:
def add_attribute(self, event: Union[MISPEvent, int, str, UUID], attribute: Union[MISPAttribute, Iterable], pythonify: bool = False, break_on_duplicate: bool = True) -> Union[Dict, MISPAttribute, MISPShadowAttribute]:
"""Add an attribute to an existing MISP event: https://www.misp-project.org/openapi/#tag/Attributes/operation/addAttribute
:param event: event to extend
@ -749,9 +753,11 @@ class PyMISP:
If a list is passed, the pythonified response is a dict with the following structure:
{'attributes': [MISPAttribute], 'errors': {errors by attributes}}
:param pythonify: Returns a PyMISP Object instead of the plain json output
:param break_on_duplicate: if False, do not fail if the attribute already exists, updates existing attribute instead (timestamp will be always updated)
"""
params = {'breakOnDuplicate': 0} if break_on_duplicate is not True else {}
event_id = get_uuid_or_id_from_abstract_misp(event)
r = self._prepare_request('POST', f'attributes/add/{event_id}', data=attribute)
r = self._prepare_request('POST', f'attributes/add/{event_id}', data=attribute, kw_params=params)
new_attribute = self._check_json_response(r)
if isinstance(attribute, list):
# Multiple attributes were passed at once, the handling is totally different
@ -1200,9 +1206,10 @@ class PyMISP:
"""
taxonomy_id = get_uuid_or_id_from_abstract_misp(taxonomy)
t = self.get_taxonomy(taxonomy_id)
if isinstance(t, MISPTaxonomy) and not t.enabled:
# Can happen if global pythonify is enabled.
raise PyMISPError(f"The taxonomy {t.namespace} is not enabled.")
if isinstance(t, MISPTaxonomy):
if not t.enabled:
# Can happen if global pythonify is enabled.
raise PyMISPError(f"The taxonomy {t.namespace} is not enabled.")
elif not t['Taxonomy']['enabled']:
raise PyMISPError(f"The taxonomy {t['Taxonomy']['namespace']} is not enabled.")
url = urljoin(self.root_url, 'taxonomies/addTag/{}'.format(taxonomy_id))
@ -1443,7 +1450,11 @@ class PyMISP:
# ## BEGIN Galaxy ###
def galaxies(self, pythonify: bool = False) -> Union[Dict, List[MISPGalaxy]]:
def galaxies(
self,
withCluster: bool = False,
pythonify: bool = False,
) -> Union[Dict, List[MISPGalaxy]]:
"""Get all the galaxies: https://www.misp-project.org/openapi/#tag/Galaxies/operation/getGalaxies
:param pythonify: Returns a list of PyMISP Objects instead of the plain json output. Warning: it might use a lot of RAM
@ -1455,7 +1466,25 @@ class PyMISP:
to_return = []
for galaxy in galaxies:
g = MISPGalaxy()
g.from_dict(**galaxy)
g.from_dict(**galaxy, withCluster=withCluster)
to_return.append(g)
return to_return
def search_galaxy(
self,
value: str,
withCluster: bool = False,
pythonify: bool = False,
) -> Union[Dict, List[MISPGalaxy]]:
"""Text search to find a matching galaxy name, namespace, description, or uuid."""
r = self._prepare_request("POST", "galaxies", data={"value": value})
galaxies = self._check_json_response(r)
if not (self.global_pythonify or pythonify) or "errors" in galaxies:
return galaxies
to_return = []
for galaxy in galaxies:
g = MISPGalaxy()
g.from_dict(**galaxy, withCluster=withCluster)
to_return.append(g)
return to_return
@ -1491,7 +1520,7 @@ class PyMISP:
kw_params = {"context": context}
if searchall:
kw_params["searchall"] = searchall
r = self._prepare_request('GET', f"galaxy_clusters/index/{galaxy_id}", kw_params=kw_params)
r = self._prepare_request('POST', f"galaxy_clusters/index/{galaxy_id}", data=kw_params)
clusters_j = self._check_json_response(r)
if not (self.global_pythonify or pythonify) or 'errors' in clusters_j:
return clusters_j
@ -2472,6 +2501,7 @@ class PyMISP:
category: Optional[SearchParameterTypes] = None,
org: Optional[SearchParameterTypes] = None,
tags: Optional[SearchParameterTypes] = None,
event_tags: Optional[SearchParameterTypes] = None,
quick_filter: Optional[str] = None, quickFilter: Optional[str] = None,
date_from: Optional[Union[datetime, date, int, str, float, None]] = None,
date_to: Optional[Union[datetime, date, int, str, float, None]] = None,
@ -2529,6 +2559,7 @@ class PyMISP:
:param category: The attribute category, any valid MISP attribute category is accepted.
:param org: Search by the creator organisation by supplying the organisation identifier.
:param tags: Tags to search or to exclude. You can pass a list, or the output of `build_complex_query`
:param event_tags: Tags to search or to exclude at the event level. You can pass a list, or the output of `build_complex_query`
:param quick_filter: The string passed to this field will ignore all of the other arguments. MISP will return an xml / json (depending on the header sent) of all events that have a sub-string match on value in the event info, event orgc, or any of the attribute value1 / value2 fields, or in the attribute comment.
:param date_from: Events with the date set to a date after the one specified. This filter will use the date of the event.
:param date_to: Events with the date set to a date before the one specified. This filter will use the date of the event.
@ -2571,7 +2602,8 @@ class PyMISP:
'''
return_formats = ['openioc', 'json', 'xml', 'suricata', 'snort', 'text', 'rpz', 'csv', 'cache', 'stix-xml', 'stix', 'stix2', 'yara', 'yara-json', 'attack', 'attack-sightings']
return_formats = ['openioc', 'json', 'xml', 'suricata', 'snort', 'text', 'rpz', 'csv', 'cache', 'stix-xml',
'stix', 'stix2', 'yara', 'yara-json', 'attack', 'attack-sightings', 'context', 'context-markdown']
if controller not in ['events', 'attributes', 'objects']:
raise ValueError('controller has to be in {}'.format(', '.join(['events', 'attributes', 'objects'])))
@ -2615,6 +2647,7 @@ class PyMISP:
query['category'] = category
query['org'] = org
query['tags'] = tags
query['event_tags'] = event_tags
query['quickFilter'] = quick_filter
query['from'] = self._make_timestamp(date_from)
query['to'] = self._make_timestamp(date_to)

2
pymisp/data/misp-objects

@ -1 +1 @@
Subproject commit 1da4760dcc99502a2dd5da02cba212b42068fcb8
Subproject commit 48dd45519624cabeb6fc7e22f76135312e2715b7

15
pymisp/mispevent.py
View File

@ -1594,12 +1594,13 @@ class MISPEvent(AbstractMISP):
to_return += attribute.hash_values(algorithm)
return to_return
def to_feed(self, valid_distributions: List[int] = [0, 1, 2, 3, 4, 5], with_meta: bool = False, with_distribution=False, with_local_tags: bool = True) -> Dict:
def to_feed(self, valid_distributions: List[int] = [0, 1, 2, 3, 4, 5], with_meta: bool = False, with_distribution=False, with_local_tags: bool = True, with_event_reports: bool = True) -> Dict:
""" Generate a json output for MISP Feed.
:param valid_distributions: only makes sense if the distribution key is set; i.e., the event is exported from a MISP instance.
:param with_distribution: exports distribution and Sharing Group info; otherwise all SharingGroup information is discarded (protecting privacy)
:param with_local_tags: tag export includes local exportable tags along with global exportable tags
:param with_event_reports: include event reports in the returned MISP event
"""
required = ['info', 'Orgc']
for r in required:
@ -1653,6 +1654,18 @@ class MISPEvent(AbstractMISP):
except AttributeError:
pass
if with_event_reports and self.event_reports:
to_return['EventReport'] = []
for event_report in self.event_reports:
if (valid_distributions and event_report.get('distribution') is not None and event_report.distribution not in valid_distributions):
continue
if not with_distribution:
event_report.pop('distribution', None)
event_report.pop('SharingGroup', None)
event_report.pop('sharing_group_id', None)
to_return['EventReport'].append(event_report.to_dict())
return {'Event': to_return}
@property

54
pymisp/tools/create_misp_object.py
View File

@ -11,7 +11,7 @@ from typing import Optional
logger = logging.getLogger('pymisp')
try:
import lief # type: ignore
import lief
lief.logging.disable()
HAS_LIEF = True
@ -35,40 +35,24 @@ def make_binary_objects(filepath: Optional[str] = None, pseudofile: Optional[Byt
misp_file = FileObject(filepath=filepath, pseudofile=pseudofile, filename=filename,
standalone=standalone, default_attributes_parameters=default_attributes_parameters)
if HAS_LIEF and (filepath or (pseudofile and filename)):
try:
if filepath:
lief_parsed = lief.parse(filepath=filepath)
elif pseudofile and filename:
lief_parsed = lief.parse(raw=pseudofile.getvalue(), name=filename)
else:
logger.critical('You need either a filepath, or a pseudofile and a filename.')
lief_parsed = None
if isinstance(lief_parsed, lief.PE.Binary):
return make_pe_objects(lief_parsed, misp_file, standalone, default_attributes_parameters)
elif isinstance(lief_parsed, lief.ELF.Binary):
return make_elf_objects(lief_parsed, misp_file, standalone, default_attributes_parameters)
elif isinstance(lief_parsed, lief.MachO.Binary):
return make_macho_objects(lief_parsed, misp_file, standalone, default_attributes_parameters)
except lief.bad_format as e:
logger.warning('Bad format: {}'.format(e))
except lief.bad_file as e:
logger.warning('Bad file: {}'.format(e))
except lief.conversion_error as e:
logger.warning('Conversion file: {}'.format(e))
except lief.builder_error as e:
logger.warning('Builder file: {}'.format(e))
except lief.parser_error as e:
logger.warning('Parser error: {}'.format(e))
except lief.integrity_error as e:
logger.warning('Integrity error: {}'.format(e))
except lief.pe_error as e:
logger.warning('PE error: {}'.format(e))
except lief.type_error as e:
logger.warning('Type error: {}'.format(e))
except lief.exception as e:
logger.warning('Lief exception: {}'.format(e))
except FileTypeNotImplemented as e:
logger.warning(e)
if filepath:
lief_parsed = lief.parse(filepath=filepath)
elif pseudofile and filename:
lief_parsed = lief.parse(raw=pseudofile.getvalue(), name=filename)
else:
logger.critical('You need either a filepath, or a pseudofile and a filename.')
lief_parsed = None
if isinstance(lief_parsed, lief.lief_errors):
logger.warning('Got an error parsing the file: {lief_parsed}')
elif isinstance(lief_parsed, lief.PE.Binary):
return make_pe_objects(lief_parsed, misp_file, standalone, default_attributes_parameters)
elif isinstance(lief_parsed, lief.ELF.Binary):
return make_elf_objects(lief_parsed, misp_file, standalone, default_attributes_parameters)
elif isinstance(lief_parsed, lief.MachO.Binary):
return make_macho_objects(lief_parsed, misp_file, standalone, default_attributes_parameters)
else:
logger.critical(f'Unexpected type from lief: {type(lief_parsed)}')
if not HAS_LIEF:
logger.warning('Please install lief, documentation here: https://github.com/lief-project/LIEF')
return misp_file, None, []

8
pymisp/tools/elfobject.py
View File

@ -10,7 +10,7 @@ from typing import Union, Optional
from pathlib import Path
from . import FileObject
import lief # type: ignore
import lief
try:
import pydeep # type: ignore
@ -21,7 +21,7 @@ except ImportError:
logger = logging.getLogger('pymisp')
def make_elf_objects(lief_parsed: lief.Binary, misp_file: FileObject, standalone: bool = True, default_attributes_parameters: dict = {}):
def make_elf_objects(lief_parsed: lief.ELF.Binary, misp_file: FileObject, standalone: bool = True, default_attributes_parameters: dict = {}):
elf_object = ELFObject(parsed=lief_parsed, standalone=standalone, default_attributes_parameters=default_attributes_parameters)
misp_file.add_reference(elf_object.uuid, 'includes', 'ELF indicators')
elf_sections = []
@ -32,14 +32,14 @@ def make_elf_objects(lief_parsed: lief.Binary, misp_file: FileObject, standalone
class ELFObject(AbstractMISPObjectGenerator):
def __init__(self, parsed: Optional[lief.ELF.Binary] = None, filepath: Optional[Union[Path, str]] = None, pseudofile: Optional[Union[BytesIO, bytes]] = None, **kwargs):
def __init__(self, parsed: Optional[lief.ELF.Binary] = None, filepath: Optional[Union[Path, str]] = None, pseudofile: Optional[BytesIO] = None, **kwargs):
"""Creates an ELF object, with lief"""
super().__init__('elf', **kwargs)
if not HAS_PYDEEP:
logger.warning("pydeep is missing, please install pymisp this way: pip install pymisp[fileobjects]")
if pseudofile:
if isinstance(pseudofile, BytesIO):
self.__elf = lief.ELF.parse(raw=pseudofile.getvalue())
self.__elf = lief.ELF.parse(io=pseudofile)
elif isinstance(pseudofile, bytes):
self.__elf = lief.ELF.parse(raw=pseudofile)
else:

8
pymisp/tools/emailobject.py
View File

@ -11,8 +11,7 @@ from io import BytesIO
from pathlib import Path
from typing import Union, List, Tuple, Dict, cast, Any, Optional
from extract_msg import openMsg # type: ignore
from extract_msg.message import Message as MsgObj # type: ignore
from extract_msg import openMsg, MessageBase
from RTFDE.exceptions import MalformedEncapsulatedRtf, NotEncapsulatedRtf # type: ignore
from RTFDE.deencapsulate import DeEncapsulator # type: ignore
from oletools.common.codepages import codepage2codec # type: ignore
@ -94,13 +93,14 @@ class EMailObject(AbstractMISPObjectGenerator):
def _msg_to_eml(self, msg_bytes: bytes) -> EmailMessage:
"""Converts a msg into an eml."""
msg_obj = openMsg(msg_bytes)
# NOTE: openMsg returns a MessageBase, not a MSGFile
msg_obj: MessageBase = openMsg(msg_bytes) # type: ignore
# msg obj stores the original raw header here
message, body, attachments = self._extract_msg_objects(msg_obj)
eml = self._build_eml(message, body, attachments)
return eml
def _extract_msg_objects(self, msg_obj: MsgObj) -> Tuple[EmailMessage, Dict, List[Any]]:
def _extract_msg_objects(self, msg_obj: MessageBase) -> Tuple[EmailMessage, Dict, List[Any]]:
"""Extracts email objects needed to construct an eml from a msg."""
message: EmailMessage = email.message_from_string(msg_obj.header.as_string(), policy=policy.default) # type: ignore
body = {}

6
pymisp/tools/machoobject.py
View File

@ -10,7 +10,7 @@ from typing import Optional, Union
from pathlib import Path
from . import FileObject
import lief # type: ignore
import lief
try:
import pydeep # type: ignore
@ -21,7 +21,7 @@ except ImportError:
logger = logging.getLogger('pymisp')
def make_macho_objects(lief_parsed: lief.Binary, misp_file: FileObject, standalone: bool = True, default_attributes_parameters: dict = {}):
def make_macho_objects(lief_parsed: lief.MachO.Binary, misp_file: FileObject, standalone: bool = True, default_attributes_parameters: dict = {}):
macho_object = MachOObject(parsed=lief_parsed, standalone=standalone, default_attributes_parameters=default_attributes_parameters)
misp_file.add_reference(macho_object.uuid, 'includes', 'MachO indicators')
macho_sections = []
@ -39,7 +39,7 @@ class MachOObject(AbstractMISPObjectGenerator):
logger.warning("pydeep is missing, please install pymisp this way: pip install pymisp[fileobjects]")
if pseudofile:
if isinstance(pseudofile, BytesIO):
self.__macho = lief.MachO.parse(raw=pseudofile.getvalue())
self.__macho = lief.MachO.parse(io=pseudofile)
elif isinstance(pseudofile, bytes):
self.__macho = lief.MachO.parse(raw=pseudofile)
else:

6
pymisp/tools/peobject.py
View File

@ -13,7 +13,7 @@ from base64 import b64encode
from . import FileObject
import lief # type: ignore
import lief
try:
import pydeep # type: ignore
@ -24,7 +24,7 @@ except ImportError:
logger = logging.getLogger('pymisp')
def make_pe_objects(lief_parsed: lief.Binary, misp_file: FileObject, standalone: bool = True, default_attributes_parameters: dict = {}):
def make_pe_objects(lief_parsed: lief.PE.Binary, misp_file: FileObject, standalone: bool = True, default_attributes_parameters: dict = {}):
pe_object = PEObject(parsed=lief_parsed, standalone=standalone, default_attributes_parameters=default_attributes_parameters)
misp_file.add_reference(pe_object.uuid, 'includes', 'PE indicators')
pe_sections = []
@ -42,7 +42,7 @@ class PEObject(AbstractMISPObjectGenerator):
logger.warning("pydeep is missing, please install pymisp this way: pip install pymisp[fileobjects]")
if pseudofile:
if isinstance(pseudofile, BytesIO):
self.__pe = lief.PE.parse(raw=pseudofile.getvalue())
self.__pe = lief.PE.parse(io=pseudofile)
elif isinstance(pseudofile, bytes):
self.__pe = lief.PE.parse(raw=pseudofile)
else:

2
pymisp/tools/reportlab_generator.py
View File

@ -1091,7 +1091,7 @@ class Event_Metadata():
Paragraph("Related Event #" + str(i + OFFSET), self.sample_style_sheet['Heading4']))
flowable_table.append(Indenter(left=-INDENT_SIZE_HEADING))
flowable_table += self.create_reduced_flowable_table_from_event(evt)
flowable_table += self.create_reduced_flowable_table_from_event(evt['Event'])
i += 1
else:
return flowable_table.append(self.value_formatter.get_unoverflowable_paragraph(DEFAULT_VALUE))

34
pyproject.toml
View File

@ -1,13 +1,12 @@
[tool.poetry]
name = "pymisp"
version = "2.4.169"
version = "2.4.171"
description = "Python API for MISP."
authors = ["Raphaël Vinot <raphael.vinot@circl.lu>"]
license = "BSD-2-Clause"
repository = "https://github.com/MISP/PyMISP"
documentation = "https://pymisp.readthedocs.io"
readme = "README.md"
classifiers=[
@ -43,24 +42,24 @@ include = [
[tool.poetry.dependencies]
python = "^3.8"
requests = "^2.28.2"
requests = "^2.31.0"
python-dateutil = "^2.8.2"
jsonschema = "^4.17.3"
deprecated = "^1.2.13"
extract_msg = {version = "^0.39.2", optional = true}
extract_msg = {version = "^0.41.1", optional = true}
RTFDE = {version = "^0.0.2", optional = true}
oletools = {version = "^0.60.1", optional = true}
python-magic = {version = "^0.4.27", optional = true}
pydeep2 = {version = "^0.5.1", optional = true}
lief = {version = "^0.12.3", optional = true}
beautifulsoup4 = {version = "^4.11.2", optional = true}
lief = {version = "^0.13.0", optional = true}
beautifulsoup4 = {version = "^4.12.2", optional = true}
validators = {version = "^0.20.0", optional = true}
sphinx-autodoc-typehints = {version = "^1.22", optional = true}
sphinx-autodoc-typehints = {version = "^1.23.0", optional = true}
recommonmark = {version = "^0.7.1", optional = true}
reportlab = {version = "^3.6.12", optional = true}
reportlab = {version = "^4.0.0", optional = true}
pyfaup = {version = "^1.2", optional = true}
publicsuffixlist = {version = "^0.9.3", optional = true}
urllib3 = {extras = ["brotli"], version = "^1.26.14", optional = true}
publicsuffixlist = {version = "^0.10.0.20230506", optional = true}
urllib3 = {extras = ["brotli"], version = "*", optional = true}
[tool.poetry.extras]
fileobjects = ['python-magic', 'pydeep2', 'lief']
@ -74,12 +73,15 @@ brotli = ['urllib3']
[tool.poetry.group.dev.dependencies]
requests-mock = "^1.10.0"
mypy = "^1.1.1"
ipython = "^8.11.0"
jupyterlab = "^3.6.1"
types-requests = "^2.28.11.15"
types-python-dateutil = "^2.8.19.10"
types-redis = "^4.5.1.4"
mypy = "^1.3.0"
ipython = [
{version = "<8.13.0", python = "<3.9"},
{version = "^8.13.0", python = ">=3.9"}
]
jupyterlab = "^4.0.0"
types-requests = "^2.31.0.0"
types-python-dateutil = "^2.8.19.13"
types-redis = "^4.5.5.2"
types-Flask = "^1.1.6"
pytest-cov = "^4.0.0"

2
setup.cfg
View File

@ -1,2 +0,0 @@
[metadata]
description-file = README.md

64
setup.py
View File

@ -1,64 +0,0 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from os import path
from setuptools import setup # type: ignore
import pymisp
this_directory = path.abspath(path.dirname(__file__))
with open(path.join(this_directory, 'README.md'), 'r') as f:
long_description = f.read()
setup(
name='pymisp',
version=pymisp.__version__,
author='Raphaël Vinot',
author_email='raphael.vinot@circl.lu',
maintainer='Raphaël Vinot',
url='https://github.com/MISP/PyMISP',
project_urls={
'Documentation': 'https://pymisp.readthedocs.io',
'Source': 'https://github.com/MISP/PyMISP',
'Tracker': 'https://github.com/MISP/PyMISP/issues',
},
description='Python API for MISP.',
long_description=long_description,
long_description_content_type='text/markdown',
packages=['pymisp', 'pymisp.tools'],
classifiers=[
'License :: OSI Approved :: BSD License',
'Development Status :: 5 - Production/Stable',
'Environment :: Console',
'Operating System :: POSIX :: Linux',
'Intended Audience :: Science/Research',
'Intended Audience :: Telecommunications Industry',
'Intended Audience :: Information Technology',
'Programming Language :: Python :: 3.6',
'Topic :: Security',
'Topic :: Internet',
],
install_requires=['requests',
'python-dateutil',
'jsonschema',
'deprecated'],
extras_require={'fileobjects': ['python-magic', 'pydeep2', 'lief>=0.11.0'],
'neo': ['py2neo'],
'openioc': ['beautifulsoup4'],
'virustotal': ['validators'],
'docs': ['sphinx-autodoc-typehints', 'recommonmark'],
'pdfexport': ['reportlab']},
tests_require=[
'jsonschema',
'python-magic',
'requests-mock'
],
test_suite="tests.test_mispevent",
include_package_data=True,
package_data={'pymisp': ['data/*.json',
'data/misp-objects/schema_objects.json',
'data/misp-objects/schema_relationships.json',
'data/misp-objects/objects/*/definition.json',
'data/misp-objects/relationships/definition.json',
'tools/pdf_fonts/Noto_TTF/*']},
)

57
tests/testlive_comprehensive.py
View File

@ -1596,6 +1596,47 @@ class TestComprehensive(unittest.TestCase):
# Delete event
self.admin_misp_connector.delete_event(first)
def test_add_event_with_attachment_object_controller__hard(self):
first = self.create_simple_event()
try:
first = self.user_misp_connector.add_event(first)
fo, peo, seos = make_binary_objects('tests/viper-test-files/test_files/whoami.exe')
for s in seos:
r = self.user_misp_connector.add_object(first, s)
self.assertEqual(r.name, 'pe-section', r)
r = self.user_misp_connector.add_object(first, peo, pythonify=True)
self.assertEqual(r.name, 'pe', r)
for ref in peo.ObjectReference:
r = self.user_misp_connector.add_object_reference(ref)
self.assertEqual(r.object_uuid, peo.uuid, r.to_json())
r = self.user_misp_connector.add_object(first, fo)
obj_attrs = r.get_attributes_by_relation('ssdeep')
self.assertEqual(len(obj_attrs), 1, obj_attrs)
self.assertEqual(r.name, 'file', r)
# Test break_on_duplicate at object level
fo_dup, peo_dup, _ = make_binary_objects('tests/viper-test-files/test_files/whoami.exe')
r = self.user_misp_connector.add_object(first, peo_dup, break_on_duplicate=True)
self.assertTrue("Duplicate object found" in r['errors'][1]['errors'], r)
# Test break on duplicate with breakOnDuplicate key in object
fo_dup.breakOnDuplicate = True
r = self.user_misp_connector.add_object(first, fo_dup)
self.assertTrue("Duplicate object found" in r['errors'][1]['errors'], r)
# Test refs
r = self.user_misp_connector.add_object_reference(fo.ObjectReference[0])
self.assertEqual(r.object_uuid, fo.uuid, r.to_json())
self.assertEqual(r.referenced_uuid, peo.uuid, r.to_json())
r = self.user_misp_connector.delete_object_reference(r, hard=True)
self.assertEqual(r['message'], 'ObjectReference deleted')
# TODO: verify that the reference is not soft-deleted instead
finally:
# Delete event
self.admin_misp_connector.delete_event(first)
def test_lief_and_sign(self):
first = self.create_simple_event()
try:
@ -1901,6 +1942,15 @@ class TestComprehensive(unittest.TestCase):
similar_error = self.user_misp_connector.add_attribute(first, new_similar)
self.assertEqual(similar_error['errors'][1]['errors']['value'][0], 'A similar attribute already exists for this event.')
# Test add attribute break_on_duplicate=False
time.sleep(5)
new_similar = MISPAttribute()
new_similar.value = '1.2.3.4'
new_similar.type = 'ip-dst'
new_similar = self.user_misp_connector.add_attribute(first, new_similar, break_on_duplicate=False)
self.assertTrue(isinstance(new_similar, MISPAttribute), new_similar)
self.assertGreater(new_similar.timestamp, new_attribute.timestamp)
# Test add multiple attributes at once
attr0 = MISPAttribute()
attr0.value = '0.0.0.0'
@ -2999,6 +3049,13 @@ class TestComprehensive(unittest.TestCase):
self.user_misp_connector.delete_event(event)
self.user_misp_connector.delete_event_report(new_event_report)
def test_search_galaxy(self):
self.admin_misp_connector.toggle_global_pythonify()
galaxy = self.admin_misp_connector.galaxies()[0]
ret = self.admin_misp_connector.search_galaxy(value=galaxy.name)
self.assertEqual(len(ret), 1)
self.admin_misp_connector.toggle_global_pythonify()
def test_galaxy_cluster(self):
self.admin_misp_connector.toggle_global_pythonify()
galaxy = self.admin_misp_connector.galaxies()[0]