mirror of https://github.com/MISP/PyMISP
Merge branch 'main' of github.com:misp/pymisp
commit
9fcaea8be1
|
@ -11,6 +11,7 @@ jobs:
|
|||
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
python-version: [3.8, 3.9, '3.10', '3.11']
|
||||
|
||||
|
|
127
CHANGELOG.txt
127
CHANGELOG.txt
|
@ -2,11 +2,138 @@ Changelog
|
|||
=========
|
||||
|
||||
|
||||
v2.4.171 (2023-05-16)
|
||||
---------------------
|
||||
|
||||
Changes
|
||||
~~~~~~~
|
||||
- Bump deps, object templates. [Raphaël Vinot]
|
||||
- Bump changelog. [Raphaël Vinot]
|
||||
- Bump changelog. [Raphaël Vinot]
|
||||
- Bump deps. [Raphaël Vinot]
|
||||
- Bump changelog. [Raphaël Vinot]
|
||||
- Bump version. [Raphaël Vinot]
|
||||
- Remove old setup files, bump deps. [Raphaël Vinot]
|
||||
|
||||
Fix
|
||||
~~~
|
||||
- Extra print breaking the CI on MISP side. [Raphaël Vinot]
|
||||
- Properly use lief on a file. [Raphaël Vinot]
|
||||
|
||||
Other
|
||||
~~~~~
|
||||
- Allow search by 'event_tags' and improve the handling of galaxy
|
||||
clusters. [Stefano Ortolani]
|
||||
|
||||
Changes:
|
||||
- Add 'event_tags' parameter when searching for events
|
||||
- Add new method to search for galaxies by value
|
||||
- Add new parameter to fetch cluster information when retrieving clusters
|
||||
- Add new parameter to hard-delete object references
|
||||
- Using underscore name 'description_file' in setup.cfg. [Erhan]
|
||||
|
||||
Usage of dash-separated 'description-file' will not be supported in future versions. Please use the underscore name 'description_file' instead. By 2023-Sep-26, you need to update your project and remove deprecated calls or your builds will no longer be supported.
|
||||
|
||||
See https://setuptools.pypa.io/en/latest/userguide/declarative_config.html for details.
|
||||
|
||||
|
||||
v2.4.170.2 (2023-05-04)
|
||||
-----------------------
|
||||
|
||||
Changes
|
||||
~~~~~~~
|
||||
- Bump changelog. [Raphaël Vinot]
|
||||
- Bump version. [Raphaël Vinot]
|
||||
- Bump deps. [Raphaël Vinot]
|
||||
- Bump deps. [Raphaël Vinot]
|
||||
|
||||
|
||||
v2.4.170.1 (2023-04-19)
|
||||
-----------------------
|
||||
|
||||
Changes
|
||||
~~~~~~~
|
||||
- Disable fail fast in GHA. [Raphaël Vinot]
|
||||
- Bump version. [Raphaël Vinot]
|
||||
- Bump deps. [Raphaël Vinot]
|
||||
|
||||
Fix
|
||||
~~~
|
||||
- Update lief code to v0.13. [Raphaël Vinot]
|
||||
|
||||
|
||||
v2.4.170 (2023-04-12)
|
||||
---------------------
|
||||
|
||||
Changes
|
||||
~~~~~~~
|
||||
- Bump changelog. [Raphaël Vinot]
|
||||
- Bump version. [Raphaël Vinot]
|
||||
- Bump deps. [Raphaël Vinot]
|
||||
|
||||
Other
|
||||
~~~~~
|
||||
- Add: support breakOnDuplicate option for attributes:add() [Luciano
|
||||
Righetti]
|
||||
- Update reportlab_generator.py. [CarlosLoureiro]
|
||||
|
||||
|
||||
v2.4.169.3 (2023-03-27)
|
||||
-----------------------
|
||||
|
||||
Changes
|
||||
~~~~~~~
|
||||
- Bump changelog. [Raphaël Vinot]
|
||||
- Bump deps, version. [Raphaël Vinot]
|
||||
- Bump deps. [Raphaël Vinot]
|
||||
|
||||
Fix
|
||||
~~~
|
||||
- Invalid check if taxo is enabled. [Raphaël Vinot]
|
||||
|
||||
|
||||
v2.4.169.2 (2023-03-17)
|
||||
-----------------------
|
||||
|
||||
Changes
|
||||
~~~~~~~
|
||||
- Bump changelog. [Raphaël Vinot]
|
||||
- Bump version. [Raphaël Vinot]
|
||||
- Include event reports by default in feed. [Raphaël Vinot]
|
||||
- Bump deps. [Raphaël Vinot]
|
||||
|
||||
Fix
|
||||
~~~
|
||||
- Use proper parameter to trigger the request in search_galaxy_clusters.
|
||||
[Raphaël Vinot]
|
||||
- Use POST in search galaxy cluster. [Raphaël Vinot]
|
||||
|
||||
Other
|
||||
~~~~~
|
||||
- Rename include_event_reports kwarg to with_event_reports, in-line with
|
||||
other kwarg naming. [UFOSmuggler]
|
||||
- Add kwarg to allow the inclusion of event reports into to_feed(),
|
||||
honour with_distribution and valid_distributions kwargs. [UFOSmuggler]
|
||||
|
||||
|
||||
v2.4.169.1 (2023-03-14)
|
||||
-----------------------
|
||||
|
||||
Changes
|
||||
~~~~~~~
|
||||
- Bump changelog. [Raphaël Vinot]
|
||||
- Bump version. [Raphaël Vinot]
|
||||
- Add greynoise-ip object. [Raphaël Vinot]
|
||||
|
||||
Fix #951
|
||||
|
||||
|
||||
v2.4.169 (2023-03-10)
|
||||
---------------------
|
||||
|
||||
Changes
|
||||
~~~~~~~
|
||||
- Bump changelog. [Raphaël Vinot]
|
||||
- Bump version. [Raphaël Vinot]
|
||||
- Bump templates. [Raphaël Vinot]
|
||||
- Bump deps. [Raphaël Vinot]
|
||||
|
|
|
@ -46,7 +46,7 @@ pip3 install pymisp[virustotal,email]
|
|||
```
|
||||
git clone https://github.com/MISP/PyMISP.git && cd PyMISP
|
||||
git submodule update --init
|
||||
poetry install -E fileobjects -E openioc -E virustotal -E docs -E pdfexport
|
||||
poetry install -E fileobjects -E openioc -E virustotal -E docs -E pdfexport -E email
|
||||
```
|
||||
|
||||
### Running the tests
|
||||
|
|
|
@ -16,7 +16,7 @@ outputdir = 'output'
|
|||
# you can use on the event index, such as organisation, tags, etc.
|
||||
# It uses the same joining and condition rules as the API parameters
|
||||
# For example:
|
||||
# filters = {'tag':'tlp:white|feed-export|!privint','org':'CIRCL', 'published':1}
|
||||
# filters = {'tags':['tlp:white','feed-export','!privint'],'org':'CIRCL', 'published':1}
|
||||
# the above would generate a feed for all published events created by CIRCL,
|
||||
# tagged tlp:white and/or feed-export but exclude anything tagged privint
|
||||
filters = {'published':'true'}
|
||||
|
@ -53,4 +53,4 @@ exclude_attribute_types = []
|
|||
with_distribution = False
|
||||
|
||||
# Include the exportable local tags along with the global tags. The default is True.
|
||||
with_local_tags = True
|
||||
with_local_tags = True
|
||||
|
|
File diff suppressed because it is too large
Load Diff
|
@ -1,4 +1,4 @@
|
|||
__version__ = '2.4.169'
|
||||
__version__ = '2.4.171'
|
||||
import logging
|
||||
import sys
|
||||
import warnings
|
||||
|
|
|
@ -588,7 +588,7 @@ class PyMISP:
|
|||
:param break_on_duplicate: if True, check and reject if this object's attributes match an existing object's attributes; may require much time
|
||||
"""
|
||||
event_id = get_uuid_or_id_from_abstract_misp(event)
|
||||
params = {'breakOnDuplicate': True} if break_on_duplicate else {}
|
||||
params = {'breakOnDuplicate': 1} if break_on_duplicate else {}
|
||||
r = self._prepare_request('POST', f'objects/add/{event_id}', data=misp_object, kw_params=params)
|
||||
new_object = self._check_json_response(r)
|
||||
if not (self.global_pythonify or pythonify) or 'errors' in new_object:
|
||||
|
@ -643,13 +643,17 @@ class PyMISP:
|
|||
ref.from_dict(**object_reference)
|
||||
return ref
|
||||
|
||||
def delete_object_reference(self, object_reference: Union[MISPObjectReference, int, str, UUID]) -> Dict:
|
||||
"""Delete a reference to an object
|
||||
|
||||
:param object_reference: object reference
|
||||
"""
|
||||
def delete_object_reference(
|
||||
self,
|
||||
object_reference: Union[MISPObjectReference, int, str, UUID],
|
||||
hard: bool = False,
|
||||
) -> Dict:
|
||||
"""Delete a reference to an object."""
|
||||
object_reference_id = get_uuid_or_id_from_abstract_misp(object_reference)
|
||||
response = self._prepare_request('POST', f'objectReferences/delete/{object_reference_id}')
|
||||
query_url = f"objectReferences/delete/{object_reference_id}"
|
||||
if hard:
|
||||
query_url += "/true"
|
||||
response = self._prepare_request("POST", query_url)
|
||||
return self._check_json_response(response)
|
||||
|
||||
# Object templates
|
||||
|
@ -741,7 +745,7 @@ class PyMISP:
|
|||
r = self._prepare_request('HEAD', f'attributes/view/{attribute_id}')
|
||||
return self._check_head_response(r)
|
||||
|
||||
def add_attribute(self, event: Union[MISPEvent, int, str, UUID], attribute: Union[MISPAttribute, Iterable], pythonify: bool = False) -> Union[Dict, MISPAttribute, MISPShadowAttribute]:
|
||||
def add_attribute(self, event: Union[MISPEvent, int, str, UUID], attribute: Union[MISPAttribute, Iterable], pythonify: bool = False, break_on_duplicate: bool = True) -> Union[Dict, MISPAttribute, MISPShadowAttribute]:
|
||||
"""Add an attribute to an existing MISP event: https://www.misp-project.org/openapi/#tag/Attributes/operation/addAttribute
|
||||
|
||||
:param event: event to extend
|
||||
|
@ -749,9 +753,11 @@ class PyMISP:
|
|||
If a list is passed, the pythonified response is a dict with the following structure:
|
||||
{'attributes': [MISPAttribute], 'errors': {errors by attributes}}
|
||||
:param pythonify: Returns a PyMISP Object instead of the plain json output
|
||||
:param break_on_duplicate: if False, do not fail if the attribute already exists, updates existing attribute instead (timestamp will be always updated)
|
||||
"""
|
||||
params = {'breakOnDuplicate': 0} if break_on_duplicate is not True else {}
|
||||
event_id = get_uuid_or_id_from_abstract_misp(event)
|
||||
r = self._prepare_request('POST', f'attributes/add/{event_id}', data=attribute)
|
||||
r = self._prepare_request('POST', f'attributes/add/{event_id}', data=attribute, kw_params=params)
|
||||
new_attribute = self._check_json_response(r)
|
||||
if isinstance(attribute, list):
|
||||
# Multiple attributes were passed at once, the handling is totally different
|
||||
|
@ -1200,9 +1206,10 @@ class PyMISP:
|
|||
"""
|
||||
taxonomy_id = get_uuid_or_id_from_abstract_misp(taxonomy)
|
||||
t = self.get_taxonomy(taxonomy_id)
|
||||
if isinstance(t, MISPTaxonomy) and not t.enabled:
|
||||
# Can happen if global pythonify is enabled.
|
||||
raise PyMISPError(f"The taxonomy {t.namespace} is not enabled.")
|
||||
if isinstance(t, MISPTaxonomy):
|
||||
if not t.enabled:
|
||||
# Can happen if global pythonify is enabled.
|
||||
raise PyMISPError(f"The taxonomy {t.namespace} is not enabled.")
|
||||
elif not t['Taxonomy']['enabled']:
|
||||
raise PyMISPError(f"The taxonomy {t['Taxonomy']['namespace']} is not enabled.")
|
||||
url = urljoin(self.root_url, 'taxonomies/addTag/{}'.format(taxonomy_id))
|
||||
|
@ -1443,7 +1450,11 @@ class PyMISP:
|
|||
|
||||
# ## BEGIN Galaxy ###
|
||||
|
||||
def galaxies(self, pythonify: bool = False) -> Union[Dict, List[MISPGalaxy]]:
|
||||
def galaxies(
|
||||
self,
|
||||
withCluster: bool = False,
|
||||
pythonify: bool = False,
|
||||
) -> Union[Dict, List[MISPGalaxy]]:
|
||||
"""Get all the galaxies: https://www.misp-project.org/openapi/#tag/Galaxies/operation/getGalaxies
|
||||
|
||||
:param pythonify: Returns a list of PyMISP Objects instead of the plain json output. Warning: it might use a lot of RAM
|
||||
|
@ -1455,7 +1466,25 @@ class PyMISP:
|
|||
to_return = []
|
||||
for galaxy in galaxies:
|
||||
g = MISPGalaxy()
|
||||
g.from_dict(**galaxy)
|
||||
g.from_dict(**galaxy, withCluster=withCluster)
|
||||
to_return.append(g)
|
||||
return to_return
|
||||
|
||||
def search_galaxy(
|
||||
self,
|
||||
value: str,
|
||||
withCluster: bool = False,
|
||||
pythonify: bool = False,
|
||||
) -> Union[Dict, List[MISPGalaxy]]:
|
||||
"""Text search to find a matching galaxy name, namespace, description, or uuid."""
|
||||
r = self._prepare_request("POST", "galaxies", data={"value": value})
|
||||
galaxies = self._check_json_response(r)
|
||||
if not (self.global_pythonify or pythonify) or "errors" in galaxies:
|
||||
return galaxies
|
||||
to_return = []
|
||||
for galaxy in galaxies:
|
||||
g = MISPGalaxy()
|
||||
g.from_dict(**galaxy, withCluster=withCluster)
|
||||
to_return.append(g)
|
||||
return to_return
|
||||
|
||||
|
@ -1491,7 +1520,7 @@ class PyMISP:
|
|||
kw_params = {"context": context}
|
||||
if searchall:
|
||||
kw_params["searchall"] = searchall
|
||||
r = self._prepare_request('GET', f"galaxy_clusters/index/{galaxy_id}", kw_params=kw_params)
|
||||
r = self._prepare_request('POST', f"galaxy_clusters/index/{galaxy_id}", data=kw_params)
|
||||
clusters_j = self._check_json_response(r)
|
||||
if not (self.global_pythonify or pythonify) or 'errors' in clusters_j:
|
||||
return clusters_j
|
||||
|
@ -2472,6 +2501,7 @@ class PyMISP:
|
|||
category: Optional[SearchParameterTypes] = None,
|
||||
org: Optional[SearchParameterTypes] = None,
|
||||
tags: Optional[SearchParameterTypes] = None,
|
||||
event_tags: Optional[SearchParameterTypes] = None,
|
||||
quick_filter: Optional[str] = None, quickFilter: Optional[str] = None,
|
||||
date_from: Optional[Union[datetime, date, int, str, float, None]] = None,
|
||||
date_to: Optional[Union[datetime, date, int, str, float, None]] = None,
|
||||
|
@ -2529,6 +2559,7 @@ class PyMISP:
|
|||
:param category: The attribute category, any valid MISP attribute category is accepted.
|
||||
:param org: Search by the creator organisation by supplying the organisation identifier.
|
||||
:param tags: Tags to search or to exclude. You can pass a list, or the output of `build_complex_query`
|
||||
:param event_tags: Tags to search or to exclude at the event level. You can pass a list, or the output of `build_complex_query`
|
||||
:param quick_filter: The string passed to this field will ignore all of the other arguments. MISP will return an xml / json (depending on the header sent) of all events that have a sub-string match on value in the event info, event orgc, or any of the attribute value1 / value2 fields, or in the attribute comment.
|
||||
:param date_from: Events with the date set to a date after the one specified. This filter will use the date of the event.
|
||||
:param date_to: Events with the date set to a date before the one specified. This filter will use the date of the event.
|
||||
|
@ -2571,7 +2602,8 @@ class PyMISP:
|
|||
|
||||
'''
|
||||
|
||||
return_formats = ['openioc', 'json', 'xml', 'suricata', 'snort', 'text', 'rpz', 'csv', 'cache', 'stix-xml', 'stix', 'stix2', 'yara', 'yara-json', 'attack', 'attack-sightings']
|
||||
return_formats = ['openioc', 'json', 'xml', 'suricata', 'snort', 'text', 'rpz', 'csv', 'cache', 'stix-xml',
|
||||
'stix', 'stix2', 'yara', 'yara-json', 'attack', 'attack-sightings', 'context', 'context-markdown']
|
||||
|
||||
if controller not in ['events', 'attributes', 'objects']:
|
||||
raise ValueError('controller has to be in {}'.format(', '.join(['events', 'attributes', 'objects'])))
|
||||
|
@ -2615,6 +2647,7 @@ class PyMISP:
|
|||
query['category'] = category
|
||||
query['org'] = org
|
||||
query['tags'] = tags
|
||||
query['event_tags'] = event_tags
|
||||
query['quickFilter'] = quick_filter
|
||||
query['from'] = self._make_timestamp(date_from)
|
||||
query['to'] = self._make_timestamp(date_to)
|
||||
|
|
|
@ -1 +1 @@
|
|||
Subproject commit 1da4760dcc99502a2dd5da02cba212b42068fcb8
|
||||
Subproject commit 48dd45519624cabeb6fc7e22f76135312e2715b7
|
|
@ -1594,12 +1594,13 @@ class MISPEvent(AbstractMISP):
|
|||
to_return += attribute.hash_values(algorithm)
|
||||
return to_return
|
||||
|
||||
def to_feed(self, valid_distributions: List[int] = [0, 1, 2, 3, 4, 5], with_meta: bool = False, with_distribution=False, with_local_tags: bool = True) -> Dict:
|
||||
def to_feed(self, valid_distributions: List[int] = [0, 1, 2, 3, 4, 5], with_meta: bool = False, with_distribution=False, with_local_tags: bool = True, with_event_reports: bool = True) -> Dict:
|
||||
""" Generate a json output for MISP Feed.
|
||||
|
||||
:param valid_distributions: only makes sense if the distribution key is set; i.e., the event is exported from a MISP instance.
|
||||
:param with_distribution: exports distribution and Sharing Group info; otherwise all SharingGroup information is discarded (protecting privacy)
|
||||
:param with_local_tags: tag export includes local exportable tags along with global exportable tags
|
||||
:param with_event_reports: include event reports in the returned MISP event
|
||||
"""
|
||||
required = ['info', 'Orgc']
|
||||
for r in required:
|
||||
|
@ -1653,6 +1654,18 @@ class MISPEvent(AbstractMISP):
|
|||
except AttributeError:
|
||||
pass
|
||||
|
||||
if with_event_reports and self.event_reports:
|
||||
to_return['EventReport'] = []
|
||||
for event_report in self.event_reports:
|
||||
if (valid_distributions and event_report.get('distribution') is not None and event_report.distribution not in valid_distributions):
|
||||
continue
|
||||
if not with_distribution:
|
||||
event_report.pop('distribution', None)
|
||||
event_report.pop('SharingGroup', None)
|
||||
event_report.pop('sharing_group_id', None)
|
||||
to_return['EventReport'].append(event_report.to_dict())
|
||||
|
||||
|
||||
return {'Event': to_return}
|
||||
|
||||
@property
|
||||
|
|
|
@ -11,7 +11,7 @@ from typing import Optional
|
|||
logger = logging.getLogger('pymisp')
|
||||
|
||||
try:
|
||||
import lief # type: ignore
|
||||
import lief
|
||||
lief.logging.disable()
|
||||
HAS_LIEF = True
|
||||
|
||||
|
@ -35,40 +35,24 @@ def make_binary_objects(filepath: Optional[str] = None, pseudofile: Optional[Byt
|
|||
misp_file = FileObject(filepath=filepath, pseudofile=pseudofile, filename=filename,
|
||||
standalone=standalone, default_attributes_parameters=default_attributes_parameters)
|
||||
if HAS_LIEF and (filepath or (pseudofile and filename)):
|
||||
try:
|
||||
if filepath:
|
||||
lief_parsed = lief.parse(filepath=filepath)
|
||||
elif pseudofile and filename:
|
||||
lief_parsed = lief.parse(raw=pseudofile.getvalue(), name=filename)
|
||||
else:
|
||||
logger.critical('You need either a filepath, or a pseudofile and a filename.')
|
||||
lief_parsed = None
|
||||
if isinstance(lief_parsed, lief.PE.Binary):
|
||||
return make_pe_objects(lief_parsed, misp_file, standalone, default_attributes_parameters)
|
||||
elif isinstance(lief_parsed, lief.ELF.Binary):
|
||||
return make_elf_objects(lief_parsed, misp_file, standalone, default_attributes_parameters)
|
||||
elif isinstance(lief_parsed, lief.MachO.Binary):
|
||||
return make_macho_objects(lief_parsed, misp_file, standalone, default_attributes_parameters)
|
||||
except lief.bad_format as e:
|
||||
logger.warning('Bad format: {}'.format(e))
|
||||
except lief.bad_file as e:
|
||||
logger.warning('Bad file: {}'.format(e))
|
||||
except lief.conversion_error as e:
|
||||
logger.warning('Conversion file: {}'.format(e))
|
||||
except lief.builder_error as e:
|
||||
logger.warning('Builder file: {}'.format(e))
|
||||
except lief.parser_error as e:
|
||||
logger.warning('Parser error: {}'.format(e))
|
||||
except lief.integrity_error as e:
|
||||
logger.warning('Integrity error: {}'.format(e))
|
||||
except lief.pe_error as e:
|
||||
logger.warning('PE error: {}'.format(e))
|
||||
except lief.type_error as e:
|
||||
logger.warning('Type error: {}'.format(e))
|
||||
except lief.exception as e:
|
||||
logger.warning('Lief exception: {}'.format(e))
|
||||
except FileTypeNotImplemented as e:
|
||||
logger.warning(e)
|
||||
if filepath:
|
||||
lief_parsed = lief.parse(filepath=filepath)
|
||||
elif pseudofile and filename:
|
||||
lief_parsed = lief.parse(raw=pseudofile.getvalue(), name=filename)
|
||||
else:
|
||||
logger.critical('You need either a filepath, or a pseudofile and a filename.')
|
||||
lief_parsed = None
|
||||
|
||||
if isinstance(lief_parsed, lief.lief_errors):
|
||||
logger.warning('Got an error parsing the file: {lief_parsed}')
|
||||
elif isinstance(lief_parsed, lief.PE.Binary):
|
||||
return make_pe_objects(lief_parsed, misp_file, standalone, default_attributes_parameters)
|
||||
elif isinstance(lief_parsed, lief.ELF.Binary):
|
||||
return make_elf_objects(lief_parsed, misp_file, standalone, default_attributes_parameters)
|
||||
elif isinstance(lief_parsed, lief.MachO.Binary):
|
||||
return make_macho_objects(lief_parsed, misp_file, standalone, default_attributes_parameters)
|
||||
else:
|
||||
logger.critical(f'Unexpected type from lief: {type(lief_parsed)}')
|
||||
if not HAS_LIEF:
|
||||
logger.warning('Please install lief, documentation here: https://github.com/lief-project/LIEF')
|
||||
return misp_file, None, []
|
||||
|
|
|
@ -10,7 +10,7 @@ from typing import Union, Optional
|
|||
from pathlib import Path
|
||||
from . import FileObject
|
||||
|
||||
import lief # type: ignore
|
||||
import lief
|
||||
|
||||
try:
|
||||
import pydeep # type: ignore
|
||||
|
@ -21,7 +21,7 @@ except ImportError:
|
|||
logger = logging.getLogger('pymisp')
|
||||
|
||||
|
||||
def make_elf_objects(lief_parsed: lief.Binary, misp_file: FileObject, standalone: bool = True, default_attributes_parameters: dict = {}):
|
||||
def make_elf_objects(lief_parsed: lief.ELF.Binary, misp_file: FileObject, standalone: bool = True, default_attributes_parameters: dict = {}):
|
||||
elf_object = ELFObject(parsed=lief_parsed, standalone=standalone, default_attributes_parameters=default_attributes_parameters)
|
||||
misp_file.add_reference(elf_object.uuid, 'includes', 'ELF indicators')
|
||||
elf_sections = []
|
||||
|
@ -32,14 +32,14 @@ def make_elf_objects(lief_parsed: lief.Binary, misp_file: FileObject, standalone
|
|||
|
||||
class ELFObject(AbstractMISPObjectGenerator):
|
||||
|
||||
def __init__(self, parsed: Optional[lief.ELF.Binary] = None, filepath: Optional[Union[Path, str]] = None, pseudofile: Optional[Union[BytesIO, bytes]] = None, **kwargs):
|
||||
def __init__(self, parsed: Optional[lief.ELF.Binary] = None, filepath: Optional[Union[Path, str]] = None, pseudofile: Optional[BytesIO] = None, **kwargs):
|
||||
"""Creates an ELF object, with lief"""
|
||||
super().__init__('elf', **kwargs)
|
||||
if not HAS_PYDEEP:
|
||||
logger.warning("pydeep is missing, please install pymisp this way: pip install pymisp[fileobjects]")
|
||||
if pseudofile:
|
||||
if isinstance(pseudofile, BytesIO):
|
||||
self.__elf = lief.ELF.parse(raw=pseudofile.getvalue())
|
||||
self.__elf = lief.ELF.parse(io=pseudofile)
|
||||
elif isinstance(pseudofile, bytes):
|
||||
self.__elf = lief.ELF.parse(raw=pseudofile)
|
||||
else:
|
||||
|
|
|
@ -11,8 +11,7 @@ from io import BytesIO
|
|||
from pathlib import Path
|
||||
from typing import Union, List, Tuple, Dict, cast, Any, Optional
|
||||
|
||||
from extract_msg import openMsg # type: ignore
|
||||
from extract_msg.message import Message as MsgObj # type: ignore
|
||||
from extract_msg import openMsg, MessageBase
|
||||
from RTFDE.exceptions import MalformedEncapsulatedRtf, NotEncapsulatedRtf # type: ignore
|
||||
from RTFDE.deencapsulate import DeEncapsulator # type: ignore
|
||||
from oletools.common.codepages import codepage2codec # type: ignore
|
||||
|
@ -94,13 +93,14 @@ class EMailObject(AbstractMISPObjectGenerator):
|
|||
|
||||
def _msg_to_eml(self, msg_bytes: bytes) -> EmailMessage:
|
||||
"""Converts a msg into an eml."""
|
||||
msg_obj = openMsg(msg_bytes)
|
||||
# NOTE: openMsg returns a MessageBase, not a MSGFile
|
||||
msg_obj: MessageBase = openMsg(msg_bytes) # type: ignore
|
||||
# msg obj stores the original raw header here
|
||||
message, body, attachments = self._extract_msg_objects(msg_obj)
|
||||
eml = self._build_eml(message, body, attachments)
|
||||
return eml
|
||||
|
||||
def _extract_msg_objects(self, msg_obj: MsgObj) -> Tuple[EmailMessage, Dict, List[Any]]:
|
||||
def _extract_msg_objects(self, msg_obj: MessageBase) -> Tuple[EmailMessage, Dict, List[Any]]:
|
||||
"""Extracts email objects needed to construct an eml from a msg."""
|
||||
message: EmailMessage = email.message_from_string(msg_obj.header.as_string(), policy=policy.default) # type: ignore
|
||||
body = {}
|
||||
|
|
|
@ -10,7 +10,7 @@ from typing import Optional, Union
|
|||
from pathlib import Path
|
||||
from . import FileObject
|
||||
|
||||
import lief # type: ignore
|
||||
import lief
|
||||
|
||||
try:
|
||||
import pydeep # type: ignore
|
||||
|
@ -21,7 +21,7 @@ except ImportError:
|
|||
logger = logging.getLogger('pymisp')
|
||||
|
||||
|
||||
def make_macho_objects(lief_parsed: lief.Binary, misp_file: FileObject, standalone: bool = True, default_attributes_parameters: dict = {}):
|
||||
def make_macho_objects(lief_parsed: lief.MachO.Binary, misp_file: FileObject, standalone: bool = True, default_attributes_parameters: dict = {}):
|
||||
macho_object = MachOObject(parsed=lief_parsed, standalone=standalone, default_attributes_parameters=default_attributes_parameters)
|
||||
misp_file.add_reference(macho_object.uuid, 'includes', 'MachO indicators')
|
||||
macho_sections = []
|
||||
|
@ -39,7 +39,7 @@ class MachOObject(AbstractMISPObjectGenerator):
|
|||
logger.warning("pydeep is missing, please install pymisp this way: pip install pymisp[fileobjects]")
|
||||
if pseudofile:
|
||||
if isinstance(pseudofile, BytesIO):
|
||||
self.__macho = lief.MachO.parse(raw=pseudofile.getvalue())
|
||||
self.__macho = lief.MachO.parse(io=pseudofile)
|
||||
elif isinstance(pseudofile, bytes):
|
||||
self.__macho = lief.MachO.parse(raw=pseudofile)
|
||||
else:
|
||||
|
|
|
@ -13,7 +13,7 @@ from base64 import b64encode
|
|||
|
||||
from . import FileObject
|
||||
|
||||
import lief # type: ignore
|
||||
import lief
|
||||
|
||||
try:
|
||||
import pydeep # type: ignore
|
||||
|
@ -24,7 +24,7 @@ except ImportError:
|
|||
logger = logging.getLogger('pymisp')
|
||||
|
||||
|
||||
def make_pe_objects(lief_parsed: lief.Binary, misp_file: FileObject, standalone: bool = True, default_attributes_parameters: dict = {}):
|
||||
def make_pe_objects(lief_parsed: lief.PE.Binary, misp_file: FileObject, standalone: bool = True, default_attributes_parameters: dict = {}):
|
||||
pe_object = PEObject(parsed=lief_parsed, standalone=standalone, default_attributes_parameters=default_attributes_parameters)
|
||||
misp_file.add_reference(pe_object.uuid, 'includes', 'PE indicators')
|
||||
pe_sections = []
|
||||
|
@ -42,7 +42,7 @@ class PEObject(AbstractMISPObjectGenerator):
|
|||
logger.warning("pydeep is missing, please install pymisp this way: pip install pymisp[fileobjects]")
|
||||
if pseudofile:
|
||||
if isinstance(pseudofile, BytesIO):
|
||||
self.__pe = lief.PE.parse(raw=pseudofile.getvalue())
|
||||
self.__pe = lief.PE.parse(io=pseudofile)
|
||||
elif isinstance(pseudofile, bytes):
|
||||
self.__pe = lief.PE.parse(raw=pseudofile)
|
||||
else:
|
||||
|
|
|
@ -1091,7 +1091,7 @@ class Event_Metadata():
|
|||
Paragraph("Related Event #" + str(i + OFFSET), self.sample_style_sheet['Heading4']))
|
||||
flowable_table.append(Indenter(left=-INDENT_SIZE_HEADING))
|
||||
|
||||
flowable_table += self.create_reduced_flowable_table_from_event(evt)
|
||||
flowable_table += self.create_reduced_flowable_table_from_event(evt['Event'])
|
||||
i += 1
|
||||
else:
|
||||
return flowable_table.append(self.value_formatter.get_unoverflowable_paragraph(DEFAULT_VALUE))
|
||||
|
|
|
@ -1,13 +1,12 @@
|
|||
[tool.poetry]
|
||||
name = "pymisp"
|
||||
version = "2.4.169"
|
||||
version = "2.4.171"
|
||||
description = "Python API for MISP."
|
||||
authors = ["Raphaël Vinot <raphael.vinot@circl.lu>"]
|
||||
license = "BSD-2-Clause"
|
||||
repository = "https://github.com/MISP/PyMISP"
|
||||
documentation = "https://pymisp.readthedocs.io"
|
||||
|
||||
|
||||
readme = "README.md"
|
||||
|
||||
classifiers=[
|
||||
|
@ -43,24 +42,24 @@ include = [
|
|||
|
||||
[tool.poetry.dependencies]
|
||||
python = "^3.8"
|
||||
requests = "^2.28.2"
|
||||
requests = "^2.31.0"
|
||||
python-dateutil = "^2.8.2"
|
||||
jsonschema = "^4.17.3"
|
||||
deprecated = "^1.2.13"
|
||||
extract_msg = {version = "^0.39.2", optional = true}
|
||||
extract_msg = {version = "^0.41.1", optional = true}
|
||||
RTFDE = {version = "^0.0.2", optional = true}
|
||||
oletools = {version = "^0.60.1", optional = true}
|
||||
python-magic = {version = "^0.4.27", optional = true}
|
||||
pydeep2 = {version = "^0.5.1", optional = true}
|
||||
lief = {version = "^0.12.3", optional = true}
|
||||
beautifulsoup4 = {version = "^4.11.2", optional = true}
|
||||
lief = {version = "^0.13.0", optional = true}
|
||||
beautifulsoup4 = {version = "^4.12.2", optional = true}
|
||||
validators = {version = "^0.20.0", optional = true}
|
||||
sphinx-autodoc-typehints = {version = "^1.22", optional = true}
|
||||
sphinx-autodoc-typehints = {version = "^1.23.0", optional = true}
|
||||
recommonmark = {version = "^0.7.1", optional = true}
|
||||
reportlab = {version = "^3.6.12", optional = true}
|
||||
reportlab = {version = "^4.0.0", optional = true}
|
||||
pyfaup = {version = "^1.2", optional = true}
|
||||
publicsuffixlist = {version = "^0.9.3", optional = true}
|
||||
urllib3 = {extras = ["brotli"], version = "^1.26.14", optional = true}
|
||||
publicsuffixlist = {version = "^0.10.0.20230506", optional = true}
|
||||
urllib3 = {extras = ["brotli"], version = "*", optional = true}
|
||||
|
||||
[tool.poetry.extras]
|
||||
fileobjects = ['python-magic', 'pydeep2', 'lief']
|
||||
|
@ -74,12 +73,15 @@ brotli = ['urllib3']
|
|||
|
||||
[tool.poetry.group.dev.dependencies]
|
||||
requests-mock = "^1.10.0"
|
||||
mypy = "^1.1.1"
|
||||
ipython = "^8.11.0"
|
||||
jupyterlab = "^3.6.1"
|
||||
types-requests = "^2.28.11.15"
|
||||
types-python-dateutil = "^2.8.19.10"
|
||||
types-redis = "^4.5.1.4"
|
||||
mypy = "^1.3.0"
|
||||
ipython = [
|
||||
{version = "<8.13.0", python = "<3.9"},
|
||||
{version = "^8.13.0", python = ">=3.9"}
|
||||
]
|
||||
jupyterlab = "^4.0.0"
|
||||
types-requests = "^2.31.0.0"
|
||||
types-python-dateutil = "^2.8.19.13"
|
||||
types-redis = "^4.5.5.2"
|
||||
types-Flask = "^1.1.6"
|
||||
pytest-cov = "^4.0.0"
|
||||
|
||||
|
|
64
setup.py
64
setup.py
|
@ -1,64 +0,0 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
from os import path
|
||||
|
||||
from setuptools import setup # type: ignore
|
||||
|
||||
import pymisp
|
||||
|
||||
this_directory = path.abspath(path.dirname(__file__))
|
||||
with open(path.join(this_directory, 'README.md'), 'r') as f:
|
||||
long_description = f.read()
|
||||
|
||||
setup(
|
||||
name='pymisp',
|
||||
version=pymisp.__version__,
|
||||
author='Raphaël Vinot',
|
||||
author_email='raphael.vinot@circl.lu',
|
||||
maintainer='Raphaël Vinot',
|
||||
url='https://github.com/MISP/PyMISP',
|
||||
project_urls={
|
||||
'Documentation': 'https://pymisp.readthedocs.io',
|
||||
'Source': 'https://github.com/MISP/PyMISP',
|
||||
'Tracker': 'https://github.com/MISP/PyMISP/issues',
|
||||
},
|
||||
description='Python API for MISP.',
|
||||
long_description=long_description,
|
||||
long_description_content_type='text/markdown',
|
||||
packages=['pymisp', 'pymisp.tools'],
|
||||
classifiers=[
|
||||
'License :: OSI Approved :: BSD License',
|
||||
'Development Status :: 5 - Production/Stable',
|
||||
'Environment :: Console',
|
||||
'Operating System :: POSIX :: Linux',
|
||||
'Intended Audience :: Science/Research',
|
||||
'Intended Audience :: Telecommunications Industry',
|
||||
'Intended Audience :: Information Technology',
|
||||
'Programming Language :: Python :: 3.6',
|
||||
'Topic :: Security',
|
||||
'Topic :: Internet',
|
||||
],
|
||||
install_requires=['requests',
|
||||
'python-dateutil',
|
||||
'jsonschema',
|
||||
'deprecated'],
|
||||
extras_require={'fileobjects': ['python-magic', 'pydeep2', 'lief>=0.11.0'],
|
||||
'neo': ['py2neo'],
|
||||
'openioc': ['beautifulsoup4'],
|
||||
'virustotal': ['validators'],
|
||||
'docs': ['sphinx-autodoc-typehints', 'recommonmark'],
|
||||
'pdfexport': ['reportlab']},
|
||||
tests_require=[
|
||||
'jsonschema',
|
||||
'python-magic',
|
||||
'requests-mock'
|
||||
],
|
||||
test_suite="tests.test_mispevent",
|
||||
include_package_data=True,
|
||||
package_data={'pymisp': ['data/*.json',
|
||||
'data/misp-objects/schema_objects.json',
|
||||
'data/misp-objects/schema_relationships.json',
|
||||
'data/misp-objects/objects/*/definition.json',
|
||||
'data/misp-objects/relationships/definition.json',
|
||||
'tools/pdf_fonts/Noto_TTF/*']},
|
||||
)
|
|
@ -1596,6 +1596,47 @@ class TestComprehensive(unittest.TestCase):
|
|||
# Delete event
|
||||
self.admin_misp_connector.delete_event(first)
|
||||
|
||||
def test_add_event_with_attachment_object_controller__hard(self):
|
||||
first = self.create_simple_event()
|
||||
try:
|
||||
first = self.user_misp_connector.add_event(first)
|
||||
fo, peo, seos = make_binary_objects('tests/viper-test-files/test_files/whoami.exe')
|
||||
for s in seos:
|
||||
r = self.user_misp_connector.add_object(first, s)
|
||||
self.assertEqual(r.name, 'pe-section', r)
|
||||
|
||||
r = self.user_misp_connector.add_object(first, peo, pythonify=True)
|
||||
self.assertEqual(r.name, 'pe', r)
|
||||
for ref in peo.ObjectReference:
|
||||
r = self.user_misp_connector.add_object_reference(ref)
|
||||
self.assertEqual(r.object_uuid, peo.uuid, r.to_json())
|
||||
|
||||
r = self.user_misp_connector.add_object(first, fo)
|
||||
obj_attrs = r.get_attributes_by_relation('ssdeep')
|
||||
self.assertEqual(len(obj_attrs), 1, obj_attrs)
|
||||
self.assertEqual(r.name, 'file', r)
|
||||
|
||||
# Test break_on_duplicate at object level
|
||||
fo_dup, peo_dup, _ = make_binary_objects('tests/viper-test-files/test_files/whoami.exe')
|
||||
r = self.user_misp_connector.add_object(first, peo_dup, break_on_duplicate=True)
|
||||
self.assertTrue("Duplicate object found" in r['errors'][1]['errors'], r)
|
||||
|
||||
# Test break on duplicate with breakOnDuplicate key in object
|
||||
fo_dup.breakOnDuplicate = True
|
||||
r = self.user_misp_connector.add_object(first, fo_dup)
|
||||
self.assertTrue("Duplicate object found" in r['errors'][1]['errors'], r)
|
||||
|
||||
# Test refs
|
||||
r = self.user_misp_connector.add_object_reference(fo.ObjectReference[0])
|
||||
self.assertEqual(r.object_uuid, fo.uuid, r.to_json())
|
||||
self.assertEqual(r.referenced_uuid, peo.uuid, r.to_json())
|
||||
r = self.user_misp_connector.delete_object_reference(r, hard=True)
|
||||
self.assertEqual(r['message'], 'ObjectReference deleted')
|
||||
# TODO: verify that the reference is not soft-deleted instead
|
||||
finally:
|
||||
# Delete event
|
||||
self.admin_misp_connector.delete_event(first)
|
||||
|
||||
def test_lief_and_sign(self):
|
||||
first = self.create_simple_event()
|
||||
try:
|
||||
|
@ -1901,6 +1942,15 @@ class TestComprehensive(unittest.TestCase):
|
|||
similar_error = self.user_misp_connector.add_attribute(first, new_similar)
|
||||
self.assertEqual(similar_error['errors'][1]['errors']['value'][0], 'A similar attribute already exists for this event.')
|
||||
|
||||
# Test add attribute break_on_duplicate=False
|
||||
time.sleep(5)
|
||||
new_similar = MISPAttribute()
|
||||
new_similar.value = '1.2.3.4'
|
||||
new_similar.type = 'ip-dst'
|
||||
new_similar = self.user_misp_connector.add_attribute(first, new_similar, break_on_duplicate=False)
|
||||
self.assertTrue(isinstance(new_similar, MISPAttribute), new_similar)
|
||||
self.assertGreater(new_similar.timestamp, new_attribute.timestamp)
|
||||
|
||||
# Test add multiple attributes at once
|
||||
attr0 = MISPAttribute()
|
||||
attr0.value = '0.0.0.0'
|
||||
|
@ -2999,6 +3049,13 @@ class TestComprehensive(unittest.TestCase):
|
|||
self.user_misp_connector.delete_event(event)
|
||||
self.user_misp_connector.delete_event_report(new_event_report)
|
||||
|
||||
def test_search_galaxy(self):
|
||||
self.admin_misp_connector.toggle_global_pythonify()
|
||||
galaxy = self.admin_misp_connector.galaxies()[0]
|
||||
ret = self.admin_misp_connector.search_galaxy(value=galaxy.name)
|
||||
self.assertEqual(len(ret), 1)
|
||||
self.admin_misp_connector.toggle_global_pythonify()
|
||||
|
||||
def test_galaxy_cluster(self):
|
||||
self.admin_misp_connector.toggle_global_pythonify()
|
||||
galaxy = self.admin_misp_connector.galaxies()[0]
|
||||
|
|
Loading…
Reference in New Issue