From 164791e980a8bac9ecceaef021ac64198facb25a Mon Sep 17 00:00:00 2001 From: Tom King Date: Sat, 16 Jan 2021 15:56:30 +0000 Subject: [PATCH 01/28] new: MISP Galaxy 2.0 capability --- pymisp/__init__.py | 2 +- pymisp/api.py | 69 ++++++++- pymisp/data/misp-objects | 2 +- pymisp/exceptions.py | 12 ++ pymisp/mispevent.py | 324 +++++++++++++++++++++++++++++++++++++-- 5 files changed, 395 insertions(+), 14 deletions(-) diff --git a/pymisp/__init__.py b/pymisp/__init__.py index dfdf831..c2e3050 100644 --- a/pymisp/__init__.py +++ b/pymisp/__init__.py @@ -24,7 +24,7 @@ Response (if any): try: from .exceptions import PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey, InvalidMISPObject, UnknownMISPObjectTemplate, PyMISPInvalidFormat, MISPServerError, PyMISPNotImplementedYet, PyMISPUnexpectedResponse, PyMISPEmptyResponse # noqa from .abstract import AbstractMISP, MISPEncode, pymisp_json_default, MISPTag, Distribution, ThreatLevel, Analysis # noqa - from .mispevent import MISPEvent, MISPAttribute, MISPObjectReference, MISPObjectAttribute, MISPObject, MISPUser, MISPOrganisation, MISPSighting, MISPLog, MISPShadowAttribute, MISPWarninglist, MISPTaxonomy, MISPNoticelist, MISPObjectTemplate, MISPSharingGroup, MISPRole, MISPServer, MISPFeed, MISPEventDelegation, MISPUserSetting, MISPInbox, MISPEventBlocklist, MISPOrganisationBlocklist # noqa + from .mispevent import MISPEvent, MISPAttribute, MISPObjectReference, MISPObjectAttribute, MISPObject, MISPUser, MISPOrganisation, MISPSighting, MISPLog, MISPShadowAttribute, MISPWarninglist, MISPTaxonomy, MISPNoticelist, MISPObjectTemplate, MISPSharingGroup, MISPRole, MISPServer, MISPFeed, MISPEventDelegation, MISPUserSetting, MISPInbox, MISPEventBlocklist, MISPOrganisationBlocklist, MISPGalaxyCluster # noqa from .tools import AbstractMISPObjectGenerator # noqa from .tools import Neo4j # noqa from .tools import stix # noqa diff --git a/pymisp/api.py b/pymisp/api.py index a7811a4..ab823b1 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -23,7 +23,8 @@ from .mispevent import MISPEvent, MISPAttribute, MISPSighting, MISPLog, MISPObje MISPUser, MISPOrganisation, MISPShadowAttribute, MISPWarninglist, MISPTaxonomy, \ MISPGalaxy, MISPNoticelist, MISPObjectReference, MISPObjectTemplate, MISPSharingGroup, \ MISPRole, MISPServer, MISPFeed, MISPEventDelegation, MISPCommunity, MISPUserSetting, \ - MISPInbox, MISPEventBlocklist, MISPOrganisationBlocklist + MISPInbox, MISPEventBlocklist, MISPOrganisationBlocklist, MISPGalaxyCluster, \ + MISPGalaxyClusterRelation from .abstract import pymisp_json_default, MISPTag, AbstractMISP, describe_types SearchType = TypeVar('SearchType', str, int) @@ -1191,10 +1192,11 @@ class PyMISP: to_return.append(g) return to_return - def get_galaxy(self, galaxy: Union[MISPGalaxy, int, str, UUID], pythonify: bool = False) -> Union[Dict, MISPGalaxy]: + def get_galaxy(self, galaxy: Union[MISPGalaxy, int, str, UUID], withCluster: bool = False, pythonify: bool = False) -> Union[Dict, MISPGalaxy]: """Get a galaxy by id :param galaxy: galaxy to get + :param withCluster: Include the clusters associated with the galaxy :param pythonify: Returns a PyMISP Object instead of the plain json output """ galaxy_id = get_uuid_or_id_from_abstract_misp(galaxy) @@ -1203,7 +1205,7 @@ class PyMISP: if not (self.global_pythonify or pythonify) or 'errors' in galaxy_j: return galaxy_j g = MISPGalaxy() - g.from_dict(**galaxy_j) + g.from_dict(**galaxy_j, withCluster=withCluster) return g def update_galaxies(self) -> Dict: @@ -1211,6 +1213,67 @@ class PyMISP: response = self._prepare_request('POST', 'galaxies/update') return self._check_json_response(response) + def get_galaxy_cluster(self, galaxy_cluster: Union[MISPGalaxyCluster, int, str, UUID], pythonify: bool = False) -> Union[Dict, MISPGalaxyCluster]: + cluster_id = get_uuid_or_id_from_abstract_misp(galaxy_cluster) + r = self._prepare_request('GET', f'galaxy_clusters/view/{cluster_id}') + cluster_j = self._check_json_response(r) + if not (self.global_pythonify or pythonify) or 'errors' in cluster_j: + return cluster_j + gc = MISPGalaxyCluster() + gc.from_dict(**cluster_j) + return gc + + def update_galaxy_cluster(self, galaxy_cluster: MISPGalaxyCluster, pythonify: bool = False) -> Union[Dict, MISPGalaxyCluster]: + """Update a custom galaxy cluster.""" + if galaxy_cluster.default: + # We can't edit default galaxies + raise PyMISPError('You are not able to update a default galaxy cluster') + cluster_id = get_uuid_or_id_from_abstract_misp(galaxy_cluster) + r = self._prepare_request('POST', f'galaxy_clusters/edit/{cluster_id}', data=galaxy_cluster) + cluster_j = self._check_json_response(r) + if not (self.global_pythonify or pythonify) or 'errors' in cluster_j: + return cluster_j + gc = MISPGalaxyCluster() + gc.from_dict(**cluster_j) + return gc + + def publish_galaxy_cluster(self, galaxy_cluster: Union[MISPGalaxyClusterRelation, int, str, UUID]) -> Dict: + cluster_id = get_uuid_or_id_from_abstract_misp(galaxy_cluster) + r = self._prepare_request('POST', f'galaxy_clusters/publish/{cluster_id}') + response = self._check_json_response(r) + return response + + def fork_galaxy_cluster(self, galaxy: Union[MISPGalaxyClusterRelation, int, str, UUID], galaxy_cluster: Union[MISPGalaxyClusterRelation, int, str, UUID], pythonify: bool = False) -> Union[Dict, MISPGalaxyCluster]: + galaxy_id = get_uuid_or_id_from_abstract_misp(galaxy) + cluster_id = get_uuid_or_id_from_abstract_misp(galaxy_cluster) + # Create a duplicate cluster from the cluster to fork + forked_galaxy_cluster = MISPGalaxyCluster() + forked_galaxy_cluster.from_dict(**galaxy_cluster) + # Set the UUID and version it extends from the existing galaxy cluster + forked_galaxy_cluster.extends_uuid = forked_galaxy_cluster.pop('uuid') + forked_galaxy_cluster.extends_version = forked_galaxy_cluster.pop('version') + r = self._prepare_request('POST', f'galaxy_clusters/add/{galaxy_id}/forkUUID:{cluster_id}', data=galaxy_cluster) + cluster_j = self._check_json_response(r) + if not (self.global_pythonify or pythonify) or 'errors' in cluster_j: + return cluster_j + gc = MISPGalaxyCluster() + gc.from_dict(**cluster_j) + return gc + + def update_galaxy_cluster_relation(self, galaxy_cluster_relation: MISPGalaxyClusterRelation) -> Dict: + """Update a galaxy cluster relation.""" + cluster_relation_id = get_uuid_or_id_from_abstract_misp(galaxy_cluster_relation) + r = self._prepare_request('POST', f'galaxy_cluster_relations/edit/{cluster_relation_id}', data=galaxy_cluster_relation) + cluster_rel_j = self._check_json_response(r) + return cluster_rel_j + + def delete_galaxy_cluster_relation(self, galaxy_cluster_relation: Union[MISPGalaxyClusterRelation, int, str, UUID]) -> Dict: + """Delete a galaxy cluster relation""" + cluster_relation_id = get_uuid_or_id_from_abstract_misp(galaxy_cluster_relation) + r = self._prepare_request('POST', f'galaxy_cluster_relations/delete/{cluster_relation_id}') + cluster_rel_j = self._check_json_response(r) + return cluster_rel_j + # ## END Galaxy ### # ## BEGIN Feed ### diff --git a/pymisp/data/misp-objects b/pymisp/data/misp-objects index 8921a0c..27a554a 160000 --- a/pymisp/data/misp-objects +++ b/pymisp/data/misp-objects @@ -1 +1 @@ -Subproject commit 8921a0c8a26f1e5a7ce77fca7e96d8523ad5fffe +Subproject commit 27a554ab12acbc1242f801b5682364b2047cf9e0 diff --git a/pymisp/exceptions.py b/pymisp/exceptions.py index 8a809cc..7a1afd4 100644 --- a/pymisp/exceptions.py +++ b/pymisp/exceptions.py @@ -23,6 +23,18 @@ class UpdateAttributeError(PyMISPError): pass +class NewGalaxyError(PyMISPError): + pass + + +class NewGalaxyClusterError(PyMISPError): + pass + + +class NewGalaxyClusterRelationError(PyMISPError): + pass + + class SearchError(PyMISPError): pass diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 04c33f9..1c61134 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -16,7 +16,7 @@ from pathlib import Path from typing import List, Optional, Union, IO, Dict, Any from .abstract import AbstractMISP, MISPTag -from .exceptions import UnknownMISPObjectTemplate, InvalidMISPObject, PyMISPError, NewEventError, NewAttributeError +from .exceptions import UnknownMISPObjectTemplate, InvalidMISPObject, PyMISPError, NewEventError, NewAttributeError, NewGalaxyClusterError, NewGalaxyClusterRelationError logger = logging.getLogger('pymisp') @@ -982,6 +982,305 @@ class MISPObject(AbstractMISP): return '<{self.__class__.__name__}(NotInitialized)'.format(self=self) +class MISPGalaxyClusterElement(AbstractMISP): + def __repr__(self) -> str: + if hasattr(self, 'key') and hasattr(self, 'value'): + return '<{self.__class__.__name__}(key={self.key}, value={self.value})'.format(self=self) + return '<{self.__class__.__name__}(NotInitialized)'.format(self=self) + + def __setattr__(self, key, value): + if key == "value" and isinstance(value, list): + raise PyMISPError("You tried to set a list to a cluster element's value. " + "Instead, create seperate elements for each value") + super().__setattr__(key, value) + + +class MISPGalaxyClusterRelation(AbstractMISP): + """A MISP Galaxy cluster relation, linking one cluster to another + + :param distribution: The distribution of the relation, one of 0, 1, 2, 3, default 0 + :type distribution: int + :param galaxy_cluster_uuid: The UUID of the galaxy the relation links to + :type galaxy_cluster_uuid: uuid + :param referenced_galaxy_cluster_type: The relation type, e.g. dropped-by + :type referenced_galaxy_cluster_type: str + :param referenced_galaxy_cluster_uuid: The UUID of the related galaxy + :type referenced_galaxy_cluster_uuid: uuid + :param referenced_galaxy_cluster_id: The ID of the related galaxy + :type referenced_galaxy_cluster_id: int, optional + :param galaxy_cluster_id: The ID of the galaxy cluster + :type galaxy_cluster_id: id, optional + :param id: The ID of the cluster relation + :type id: int, optional + :param default: Whether the relation is a default + :type default: bool, optional + """ + + def __repr__(self) -> str: + if hasattr(self, "referenced_galaxy_cluster_type"): + return '<{self.__class__.__name__}(referenced_galaxy_cluster_type={self.referenced_galaxy_cluster_type})'.format(self=self) + return '<{self.__class__.__name__}(NotInitialized)'.format(self=self) + + def __init__(self): + super().__init__() + self.galaxy_cluster_uuid: uuid + self.referenced_galaxy_cluster_uuid: uuid + self.distribution: int = 0 + self.referenced_galaxy_cluster_type: str + self.Tag: MISPTag = [] + + def from_dict(self, **kwargs): + # Default values for a valid event to send to a MISP instance + self.distribution = kwargs.pop('distribution', 0) + self.distribution = int(self.distribution) + if self.distribution not in [0, 1, 2, 3, 4, 5]: + raise NewGalaxyClusterRelationError(f'{self.distribution} is invalid, the distribution has to be in 0, 1, 2, 3, 4') + + if kwargs.get('id'): + self.id = int(kwargs.pop('id')) + if kwargs.get('orgc_id'): + self.orgc_id = int(kwargs.pop('orgc_id')) + if kwargs.get('org_id'): + self.org_id = int(kwargs.pop('org_id')) + if kwargs.get('galaxy_id'): + self.galaxy_id = int(kwargs.pop('galaxy_id')) + if kwargs.get('tag_id'): + self.tag_id = int(kwargs.pop('tag_id')) + if kwargs.get('sharing_group_id'): + self.sharing_group_id = int(kwargs.pop('sharing_group_id')) + if kwargs.get('Tag'): + [self.add_tag(**t) for t in kwargs.pop('Tag')] + if kwargs.get('SharingGroup'): + self.SharingGroup = MISPSharingGroup() + self.SharingGroup.from_dict(**kwargs.pop('SharingGroup')) + super().from_dict(**kwargs) + + def add_tag(self, tag: Optional[Union[str, MISPTag, Dict]] = None, **kwargs) -> MISPTag: + return super()._add_tag(tag, **kwargs) + + @property + def tags(self) -> List[MISPTag]: + """Returns a list of tags associated to this Attribute""" + return self.Tag + + @tags.setter + def tags(self, tags: List[MISPTag]): + """Set a list of prepared MISPTag.""" + super()._set_tags(tags) + + +class MISPGalaxyCluster(AbstractMISP): + """A MISP galaxy cluster, storing respective galaxy elements and relations. + Used to view default galaxy clusters and add/edit/update/delete Galaxy 2.0 clusters + + :param Org: The organisation as a MISPOrganisation + :type Org: MISPOrganisation + :param Orgc: The creator organisation as a MISPOrganisation + :type Orgc: MISPOrganisation + :param SharingGroup: The SharingGroup applied to the cluster, if any + :type SharingGroup: MISPSharingGroup, optional + :param default: Whether the galaxy cluster is a default or custom cluster, default clusters cannot be edited + :type default: bool + :param deleted: Whether the galaxy cluster is deleted or not + :type deleted: bool + :param description: The description of the galaxy cluster + :type description: str + :param distribution: The distribution type, one of 1, 2, 3, 4, 5 + :type distribution: int + :param sharing_group_id: The sharing group ID, if distribution is set to 4 + :type sharing_group_id: int, optional + :param extends_uuid: The UUID of the galaxy cluster it extends + :type extends_uuid: uuid, optional + :param galaxy_id: The ID of the galaxy + :type galaxy_id: int + :param id: The ID of the galaxy cluster + :type id: int + :param org_id: The org's ID + :type org_id: int + :param orgc_id: The creating org's ID + :type orgc_id: int + :param published: Whether the cluster is published or not + :type published: bool + :param source: The source of the galaxy cluster + :type source: str + :param tag_count: The count of events using this galaxy cluster + :type tag_count: int + :param tag_id: The tag ID + :type tag_id: int + :param tag_name: The galaxy cluster's tag + :type tag_name: str + :param type: The type of the galaxy cluster, must match the housing galaxies type + :type type: str + :param value: The value of the galaxy cluster + :type value: str + :param authors: A list of authors of the galaxy cluster + :type authors: list, optional + :param cluster_elements: List of MISPGalaxyClusterElement + :type cluster_elements: list, optional + :param cluster_relations: List of MISPGalaxyClusterRelation, changes must be made through PyMISP instance + :type cluster_relations: list, optional + """ + + def __init__(self): + super().__init__() + self.Galaxy: MISPGalaxy + self.GalaxyElement: List[MISPGalaxyClusterElement] = [] + self.GalaxyClusterRelation: List[MISPGalaxyClusterRelation] = [] + self.Org: MISPOrganisation + self.Orgc: MISPOrganisation + self.SharingGroup: MISPSharingGroup + + @property + def cluster_elements(self) -> List[MISPGalaxyClusterElement]: + return self.GalaxyElement + + @cluster_elements.setter + def cluster_elements(self, cluster_elements: List[MISPGalaxyClusterElement]): + self.GalaxyElement = cluster_elements + + @property + def cluster_relations(self) -> MISPGalaxyClusterRelation: + return self.GalaxyClusterRelation + + @cluster_relations.setter + def cluster_relations(self, cluster_relations: List[MISPGalaxyClusterRelation]): + self.GalaxyClusterRelation = cluster_relations + + def from_dict(self, **kwargs): + # If the default field is set, we shouldn't have distribution or sharing group ID set + if 'GalaxyCluster' in kwargs: + kwargs = kwargs['GalaxyCluster'] + + if kwargs.get('default', False): + blocked_fields = ["distribution" "sharing_group_id"] + for field in blocked_fields: + if kwargs.get(field, None): + raise NewGalaxyClusterError( + f"One of the following fields are set for a default galaxy cluster: {', '.join(blocked_fields)}" + ) + if 'uuid' in kwargs: + self.uuid = kwargs.pop('uuid') + if 'meta' in kwargs: + # Parse the cluster elements from the kwargs meta fields + for key, value in kwargs.pop('meta').items(): + # The meta will merge fields together, i.e. Two 'countries' will be a list, so split these up + if not isinstance(value, list): + value = [value] + for v in value: + self.add_cluster_element(key=key, value=v) + if 'Galaxy' in kwargs: + self.Galaxy = MISPGalaxy() + self.Galaxy.from_dict(**kwargs.pop('Galaxy')) + if 'GalaxyElement' in kwargs: + [self.add_cluster_element(**e) for e in kwargs.pop('GalaxyElement')] + if 'Org' in kwargs: + self.Org = MISPOrganisation() + self.Org.from_dict(**kwargs.pop('Org')) + if 'Orgc' in kwargs: + self.Orgc = MISPOrganisation() + self.Orgc.from_dict(**kwargs.pop('Orgc')) + if 'GalaxyClusterRelation' in kwargs: + [self.add_cluster_relation(**r) for r in kwargs.pop('GalaxyClusterRelation')] + if 'SharingGroup' in kwargs: + self.SharingGroup = MISPSharingGroup() + self.SharingGroup.from_dict(**kwargs.pop('SharingGroup')) + super().from_dict(**kwargs) + + def add_cluster_element(self, key: str, value: str, **kwargs): + """Add a cluster relation to a MISPGalaxyCluster, key and value are required + + :param key: The key name of the element + :param value: The value of the element + """ + cluster_element = MISPGalaxyClusterElement() + cluster_element.from_dict(key=key, value=value, **kwargs) + self.cluster_elements.append(cluster_element) + return cluster_element + + def add_cluster_relation(self, referenced_galaxy_cluster_uuid: uuid, referenced_galaxy_cluster_type: str, galaxy_cluster_uuid: str = None, **kwargs): + """Add a cluster relation to a MISPGalaxyCluster + + :param referenced_galaxy_cluster_uuid: UUID of the related cluster + :param referenced_galaxy_cluster_type: Relation type + """ + if not getattr(self, "uuid", None): + raise PyMISPError("The cluster does not have a UUID, make sure it is a valid galaxy cluster") + cluster_relation = MISPGalaxyClusterRelation() + cluster_relation.from_dict( + galaxy_cluster_uuid=self.uuid, + referenced_galaxy_cluster_uuid=referenced_galaxy_cluster_uuid, + referenced_galaxy_cluster_type=referenced_galaxy_cluster_type, + **kwargs + ) + self.cluster_relations.append(cluster_relation) + return cluster_relation + + def __repr__(self) -> str: + if hasattr(self, 'value'): + return '<{self.__class__.__name__}(value={self.value})'.format(self=self) + return '<{self.__class__.__name__}(NotInitialized)'.format(self=self) + + +class MISPGalaxy(AbstractMISP): + """Galaxy class, used to view a galaxy and respective clusters, supports the following fields + + :param id: Galaxy ID + :type id: int + :param uuid: Galaxy UUID + :type uuuid: uuid, str + :param name: Galaxy name + :type name: str + :param type: Galaxy type + :type type: str + :param description: Galaxy description + :type description: str + :param version: Galaxy version number + :type version: int + :param icon: Galaxy icon + :type icon: str + :param namespace: Galaxy namespace + :type namespace: str + :param clusters: List of MISPGalaxyCluster + :type clusters: list + """ + + def __init__(self): + super().__init__() + self.GalaxyCluster: List[MISPGalaxyCluster] = [] + + def from_dict(self, **kwargs): + """Galaxy could be in one of the following formats: + {'Galaxy': {}, 'GalaxyCluster': []} + {'Galaxy': {'GalaxyCluster': []}} + """ + + if 'GalaxyCluster' in kwargs and kwargs.get("withCluster", True): + # Parse the cluster from the kwargs + [self.add_galaxy_cluster(**e) for e in kwargs.pop('GalaxyCluster')] + + if 'Galaxy' in kwargs: + kwargs = kwargs['Galaxy'] + super().from_dict(**kwargs) + + @property + def clusters(self) -> List[MISPGalaxyCluster]: + return self.GalaxyCluster + + def add_galaxy_cluster(self, **kwargs) -> MISPGalaxyCluster: + """Add a MISP galaxy and sub-clusters into an event. + Supports all other parameters supported by MISPGalaxy""" + + galaxy_cluster = MISPGalaxyCluster() + galaxy_cluster.from_dict(**kwargs) + self.clusters.append(galaxy_cluster) + return galaxy_cluster + + def __repr__(self) -> str: + if hasattr(self, 'name'): + return '<{self.__class__.__name__}(name={self.name})'.format(self=self) + return '<{self.__class__.__name__}(NotInitialized)'.format(self=self) + + class MISPEvent(AbstractMISP): _fields_for_feed: set = {'uuid', 'info', 'threat_level_id', 'analysis', 'timestamp', @@ -1006,6 +1305,7 @@ class MISPEvent(AbstractMISP): self.ShadowAttribute: List[MISPShadowAttribute] = [] self.SharingGroup: MISPSharingGroup self.Tag: List[MISPTag] = [] + self.Galaxy: List[MISPGalaxy] = [] def add_tag(self, tag: Optional[Union[str, MISPTag, dict]] = None, **kwargs) -> MISPTag: return super()._add_tag(tag, **kwargs) @@ -1168,6 +1468,10 @@ class MISPEvent(AbstractMISP): def objects(self) -> List[MISPObject]: return self.Object + @property + def galaxies(self) -> List[MISPGalaxy]: + return self.Galaxy + @objects.setter def objects(self, objects: List[MISPObject]): if all(isinstance(x, MISPObject) for x in objects): @@ -1272,6 +1576,8 @@ class MISPEvent(AbstractMISP): self.set_date(kwargs.pop('date')) if kwargs.get('Attribute'): [self.add_attribute(**a) for a in kwargs.pop('Attribute')] + if kwargs.get('Galaxy'): + [self.add_galaxy(**e) for e in kwargs.pop('Galaxy')] # All other keys if kwargs.get('id'): @@ -1412,6 +1718,14 @@ class MISPEvent(AbstractMISP): return attr_list return attribute + def add_galaxy(self, **kwargs) -> MISPGalaxy: + """Add a MISP galaxy and sub-clusters into an event. + Supports all other parameters supported by MISPGalaxy""" + galaxy = MISPGalaxy() + galaxy.from_dict(**kwargs) + self.galaxies.append(galaxy) + return galaxy + def get_object_by_id(self, object_id: Union[str, int]) -> MISPObject: """Get an object by ID @@ -1610,14 +1924,6 @@ class MISPTaxonomy(AbstractMISP): super().from_dict(**kwargs) -class MISPGalaxy(AbstractMISP): - - def from_dict(self, **kwargs): - if 'Galaxy' in kwargs: - kwargs = kwargs['Galaxy'] - super().from_dict(**kwargs) - - class MISPNoticelist(AbstractMISP): def from_dict(self, **kwargs): From cff7e7b28550bb0911485430eb79d6389c250056 Mon Sep 17 00:00:00 2001 From: Tom King Date: Sat, 16 Jan 2021 16:11:41 +0000 Subject: [PATCH 02/28] new: Add in ability to add a new cluster relation --- pymisp/api.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pymisp/api.py b/pymisp/api.py index ab823b1..03f7868 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -1260,6 +1260,12 @@ class PyMISP: gc.from_dict(**cluster_j) return gc + def add_galaxy_cluster_relation(self, galaxy_cluster_relation: MISPGalaxyClusterRelation) -> Dict: + """Add a galaxy cluster relation""" + r = self._prepare_request('POST', 'galaxy_cluster_relations/add/', data=galaxy_cluster_relation) + cluster_rel_j = self._check_json_response(r) + return cluster_rel_j + def update_galaxy_cluster_relation(self, galaxy_cluster_relation: MISPGalaxyClusterRelation) -> Dict: """Update a galaxy cluster relation.""" cluster_relation_id = get_uuid_or_id_from_abstract_misp(galaxy_cluster_relation) From cc102675bbf72006ed2391aa74ebbba542d8e314 Mon Sep 17 00:00:00 2001 From: Tom King Date: Mon, 25 Jan 2021 13:18:12 +0000 Subject: [PATCH 03/28] chg: Add in add_cluster function and ability to search clusters within a galaxy --- pymisp/api.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/pymisp/api.py b/pymisp/api.py index 03f7868..8808266 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -1208,6 +1208,25 @@ class PyMISP: g.from_dict(**galaxy_j, withCluster=withCluster) return g + def search_galaxy_clusters(self, galaxy: Union[MISPGalaxy, int, str, UUID], context: str = "all", searchall: str = None, pythonify: bool = False) -> Union[Dict, List[MISPGalaxyCluster]]: + galaxy_id = get_uuid_or_id_from_abstract_misp(galaxy) + allowed_context_types = ["all", "default", "custom", "org", "deleted"] + if context not in allowed_context_types: + raise PyMISPError(f"The context must be one of {allowed_context_types.join(', ')}") + kw_params = {"context": context} + if searchall: + kw_params["searchall"] = searchall + r = self._prepare_request('GET', f"galaxy_clusters/index/{galaxy_id}", kw_params=kw_params) + clusters_j = self._check_json_response(r) + if not (self.global_pythonify or pythonify) or 'errors' in clusters_j: + return clusters_j + response = [] + for cluster in clusters_j: + c = MISPGalaxyCluster() + c.from_dict(**cluster) + response.append(c) + return response + def update_galaxies(self) -> Dict: """Update all the galaxies.""" response = self._prepare_request('POST', 'galaxies/update') @@ -1223,6 +1242,16 @@ class PyMISP: gc.from_dict(**cluster_j) return gc + def add_galaxy_cluster(self, galaxy: MISPGalaxy, galaxy_cluster: MISPGalaxyCluster, pythonify: bool = False) -> Union[Dict, MISPGalaxyCluster]: + galaxy_id = get_uuid_or_id_from_abstract_misp(galaxy) + r = self._prepare_request('POST', f'galaxy_clusters/add/{galaxy_id}', data=galaxy_cluster) + cluster_j = self._check_json_response(r) + if not (self.global_pythonify or pythonify) or 'errors' in cluster_j: + return cluster_j + gc = MISPGalaxyCluster() + gc.from_dict(**cluster_j) + return gc + def update_galaxy_cluster(self, galaxy_cluster: MISPGalaxyCluster, pythonify: bool = False) -> Union[Dict, MISPGalaxyCluster]: """Update a custom galaxy cluster.""" if galaxy_cluster.default: From 96636639c4d688bd064a24537fe078e99f428145 Mon Sep 17 00:00:00 2001 From: Tom King Date: Sat, 30 Jan 2021 13:56:40 +0000 Subject: [PATCH 04/28] chg: Add in more Galaxy 2.0 functions and code cleanup --- pymisp/__init__.py | 2 +- pymisp/api.py | 89 +++++++++++++++++++--- pymisp/exceptions.py | 4 - pymisp/mispevent.py | 173 +++++++++++++++++++++++-------------------- 4 files changed, 172 insertions(+), 96 deletions(-) diff --git a/pymisp/__init__.py b/pymisp/__init__.py index e23bc30..11b36cc 100644 --- a/pymisp/__init__.py +++ b/pymisp/__init__.py @@ -24,7 +24,7 @@ Response (if any): try: from .exceptions import PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey, InvalidMISPObject, UnknownMISPObjectTemplate, PyMISPInvalidFormat, MISPServerError, PyMISPNotImplementedYet, PyMISPUnexpectedResponse, PyMISPEmptyResponse # noqa from .abstract import AbstractMISP, MISPEncode, pymisp_json_default, MISPTag, Distribution, ThreatLevel, Analysis # noqa - from .mispevent import MISPEvent, MISPAttribute, MISPObjectReference, MISPObjectAttribute, MISPObject, MISPUser, MISPOrganisation, MISPSighting, MISPLog, MISPShadowAttribute, MISPWarninglist, MISPTaxonomy, MISPNoticelist, MISPObjectTemplate, MISPSharingGroup, MISPRole, MISPServer, MISPFeed, MISPEventDelegation, MISPUserSetting, MISPInbox, MISPEventBlocklist, MISPOrganisationBlocklist, MISPEventReport, MISPGalaxyCluster # noqa + from .mispevent import MISPEvent, MISPAttribute, MISPObjectReference, MISPObjectAttribute, MISPObject, MISPUser, MISPOrganisation, MISPSighting, MISPLog, MISPShadowAttribute, MISPWarninglist, MISPTaxonomy, MISPNoticelist, MISPObjectTemplate, MISPSharingGroup, MISPRole, MISPServer, MISPFeed, MISPEventDelegation, MISPUserSetting, MISPInbox, MISPEventBlocklist, MISPOrganisationBlocklist, MISPEventReport, MISPGalaxyCluster, MISPGalaxyClusterElement, MISPGalaxyClusterRelation # noqa from .tools import AbstractMISPObjectGenerator # noqa from .tools import Neo4j # noqa from .tools import stix # noqa diff --git a/pymisp/api.py b/pymisp/api.py index 48191d9..6f39a24 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -471,9 +471,10 @@ class PyMISP: """ event_report_id = get_uuid_or_id_from_abstract_misp(event_report) request_url = f'eventReports/delete/{event_report_id}' + data = {} if hard: - request_url += "/1" - r = self._prepare_request('POST', request_url) + data['hard'] = 1 + r = self._prepare_request('POST', request_url, data=data) return self._check_json_response(r) # ## END Event Report ### @@ -1302,7 +1303,15 @@ class PyMISP: g.from_dict(**galaxy_j, withCluster=withCluster) return g - def search_galaxy_clusters(self, galaxy: Union[MISPGalaxy, int, str, UUID], context: str = "all", searchall: str = None, pythonify: bool = False) -> Union[Dict, List[MISPGalaxyCluster]]: + def search_galaxy_clusters(self, galaxy: Union[MISPGalaxy, int, str, UUID], context: str = "all", searchall: str = None, pythonify: bool = False) -> Union[List[Dict], List[MISPGalaxyCluster]]: + """Searches the galaxy clusters within a specific galaxy + + :param galaxy: The MISPGalaxy you wish to search in + :param context: The context of how you want to search within the galaxy_ + :param searchall: The search you want to make against the galaxy and context + :param pythonify: Returns a PyMISP Object instead of the plain json output + """ + galaxy_id = get_uuid_or_id_from_abstract_misp(galaxy) allowed_context_types = ["all", "default", "custom", "org", "deleted"] if context not in allowed_context_types: @@ -1327,6 +1336,12 @@ class PyMISP: return self._check_json_response(response) def get_galaxy_cluster(self, galaxy_cluster: Union[MISPGalaxyCluster, int, str, UUID], pythonify: bool = False) -> Union[Dict, MISPGalaxyCluster]: + """Gets a specific galaxy cluster + + :param galaxy_cluster: The MISPGalaxyCluster you want to get + :param pythonify: Returns a PyMISP Object instead of the plain json output + """ + cluster_id = get_uuid_or_id_from_abstract_misp(galaxy_cluster) r = self._prepare_request('GET', f'galaxy_clusters/view/{cluster_id}') cluster_j = self._check_json_response(r) @@ -1336,7 +1351,17 @@ class PyMISP: gc.from_dict(**cluster_j) return gc - def add_galaxy_cluster(self, galaxy: MISPGalaxy, galaxy_cluster: MISPGalaxyCluster, pythonify: bool = False) -> Union[Dict, MISPGalaxyCluster]: + def add_galaxy_cluster(self, galaxy: Union[MISPGalaxy, str, UUID], galaxy_cluster: MISPGalaxyCluster, pythonify: bool = False) -> Union[Dict, MISPGalaxyCluster]: + """Add a new galaxy cluster to a MISP Galaxy + + :param galaxy: A MISPGalaxy (or UUID) where you wish to add the galaxy cluster + :param galaxy_cluster: A MISPGalaxyCluster you wish to add + :param pythonify: Returns a PyMISP Object instead of the plain json output + """ + + if galaxy_cluster.default: + # We can't add default galaxies + raise PyMISPError('You are not able add a default galaxy cluster') galaxy_id = get_uuid_or_id_from_abstract_misp(galaxy) r = self._prepare_request('POST', f'galaxy_clusters/add/{galaxy_id}', data=galaxy_cluster) cluster_j = self._check_json_response(r) @@ -1347,7 +1372,12 @@ class PyMISP: return gc def update_galaxy_cluster(self, galaxy_cluster: MISPGalaxyCluster, pythonify: bool = False) -> Union[Dict, MISPGalaxyCluster]: - """Update a custom galaxy cluster.""" + """Update a custom galaxy cluster. + + ;param galaxy_cluster: The MISPGalaxyCluster you wish to update + :param pythonify: Returns a PyMISP Object instead of the plain json output + """ + if galaxy_cluster.default: # We can't edit default galaxies raise PyMISPError('You are not able to update a default galaxy cluster') @@ -1360,13 +1390,26 @@ class PyMISP: gc.from_dict(**cluster_j) return gc - def publish_galaxy_cluster(self, galaxy_cluster: Union[MISPGalaxyClusterRelation, int, str, UUID]) -> Dict: + def publish_galaxy_cluster(self, galaxy_cluster: Union[MISPGalaxyCluster, int, str, UUID]) -> Dict: + """Publishes a galaxy cluster + + :param galaxy_cluster: The galaxy cluster you wish to publish + """ + if isinstance(galaxy_cluster, MISPGalaxyCluster) and galaxy_cluster.default: + raise PyMISPError('You are not able to publish a default galaxy cluster') cluster_id = get_uuid_or_id_from_abstract_misp(galaxy_cluster) r = self._prepare_request('POST', f'galaxy_clusters/publish/{cluster_id}') response = self._check_json_response(r) return response - def fork_galaxy_cluster(self, galaxy: Union[MISPGalaxyClusterRelation, int, str, UUID], galaxy_cluster: Union[MISPGalaxyClusterRelation, int, str, UUID], pythonify: bool = False) -> Union[Dict, MISPGalaxyCluster]: + def fork_galaxy_cluster(self, galaxy: Union[MISPGalaxy, int, str, UUID], galaxy_cluster: Union[MISPGalaxyClusterRelation, int, str, UUID], pythonify: bool = False) -> Union[Dict, MISPGalaxyCluster]: + """Forks an existing galaxy cluster, creating a new one with matching attributes + + :param galaxy: The galaxy (or galaxy ID) where the cluster you want to fork resides + :param galaxy_cluster: The galaxy cluster you wish to fork + :param pythonify: Returns a PyMISP Object instead of the plain json output + """ + galaxy_id = get_uuid_or_id_from_abstract_misp(galaxy) cluster_id = get_uuid_or_id_from_abstract_misp(galaxy_cluster) # Create a duplicate cluster from the cluster to fork @@ -1383,21 +1426,47 @@ class PyMISP: gc.from_dict(**cluster_j) return gc + def delete_galaxy_cluster(self, galaxy_cluster: Union[MISPGalaxyCluster, id, str, UUID], hard=False) -> Dict: + """Deletes a galaxy cluster from MISP + + :param galaxy_cluster: The MISPGalaxyCluster you wish to delete from MISP + :param hard: flag for hard delete + """ + + if isinstance(galaxy_cluster, MISPGalaxyCluster) and galaxy_cluster.default: + raise PyMISPError('You are not able to delete a default galaxy cluster') + data = {} + if hard: + data['hard'] = 1 + cluster_id = get_uuid_or_id_from_abstract_misp(galaxy_cluster) + r = self._prepare_request('POST', f'galaxy_clusters/delete/{cluster_id}', data=data) + return self._check_json_response(r) + def add_galaxy_cluster_relation(self, galaxy_cluster_relation: MISPGalaxyClusterRelation) -> Dict: - """Add a galaxy cluster relation""" + """Add a galaxy cluster relation, cluster relation must include + cluster UUIDs in both directions + + :param galaxy_cluster_relation: The MISPGalaxyClusterRelation to add + """ r = self._prepare_request('POST', 'galaxy_cluster_relations/add/', data=galaxy_cluster_relation) cluster_rel_j = self._check_json_response(r) return cluster_rel_j def update_galaxy_cluster_relation(self, galaxy_cluster_relation: MISPGalaxyClusterRelation) -> Dict: - """Update a galaxy cluster relation.""" + """Update a galaxy cluster relation + + :param galaxy_cluster_relation: The MISPGalaxyClusterRelation to update + """ cluster_relation_id = get_uuid_or_id_from_abstract_misp(galaxy_cluster_relation) r = self._prepare_request('POST', f'galaxy_cluster_relations/edit/{cluster_relation_id}', data=galaxy_cluster_relation) cluster_rel_j = self._check_json_response(r) return cluster_rel_j def delete_galaxy_cluster_relation(self, galaxy_cluster_relation: Union[MISPGalaxyClusterRelation, int, str, UUID]) -> Dict: - """Delete a galaxy cluster relation""" + """Delete a galaxy cluster relation + + :param galaxy_cluster_relation: The MISPGalaxyClusterRelation to delete + """ cluster_relation_id = get_uuid_or_id_from_abstract_misp(galaxy_cluster_relation) r = self._prepare_request('POST', f'galaxy_cluster_relations/delete/{cluster_relation_id}') cluster_rel_j = self._check_json_response(r) diff --git a/pymisp/exceptions.py b/pymisp/exceptions.py index a9857ce..58d3a52 100644 --- a/pymisp/exceptions.py +++ b/pymisp/exceptions.py @@ -27,10 +27,6 @@ class UpdateAttributeError(PyMISPError): pass -class NewGalaxyError(PyMISPError): - pass - - class NewGalaxyClusterError(PyMISPError): pass diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index e7905e8..a3a2437 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -1054,6 +1054,16 @@ class MISPEventReport(AbstractMISP): class MISPGalaxyClusterElement(AbstractMISP): + """A MISP Galaxy cluster element, providing further info on a cluster + + Creating a new galaxy cluster element can take the following parameters + + :param key: The key/identifier of the element + :type key: str + :param value: The value of the element + :type value: str + """ + def __repr__(self) -> str: if hasattr(self, 'key') and hasattr(self, 'value'): return '<{self.__class__.__name__}(key={self.key}, value={self.value})'.format(self=self) @@ -1065,26 +1075,30 @@ class MISPGalaxyClusterElement(AbstractMISP): "Instead, create seperate elements for each value") super().__setattr__(key, value) + def from_dict(self, **kwargs): + if kwargs.get('id'): + self.id = int(kwargs.pop('id')) + if kwargs.get('galaxy_cluster_id'): + self.galaxy_cluster_id = int(kwargs.pop('galaxy_cluster_id')) + + super().from_dict(**kwargs) + class MISPGalaxyClusterRelation(AbstractMISP): """A MISP Galaxy cluster relation, linking one cluster to another - :param distribution: The distribution of the relation, one of 0, 1, 2, 3, default 0 - :type distribution: int + Creating a new galaxy cluster can take the following parameters + :param galaxy_cluster_uuid: The UUID of the galaxy the relation links to :type galaxy_cluster_uuid: uuid :param referenced_galaxy_cluster_type: The relation type, e.g. dropped-by :type referenced_galaxy_cluster_type: str :param referenced_galaxy_cluster_uuid: The UUID of the related galaxy :type referenced_galaxy_cluster_uuid: uuid - :param referenced_galaxy_cluster_id: The ID of the related galaxy - :type referenced_galaxy_cluster_id: int, optional - :param galaxy_cluster_id: The ID of the galaxy cluster - :type galaxy_cluster_id: id, optional - :param id: The ID of the cluster relation - :type id: int, optional - :param default: Whether the relation is a default - :type default: bool, optional + :param distribution: The distribution of the relation, one of 0, 1, 2, 3, 4, default 0 + :type distribution: int + :param sharing_group_id: The sharing group of the relation, only when distribution is 4 + :type sharing_group_id: int, optional """ def __repr__(self) -> str: @@ -1102,11 +1116,21 @@ class MISPGalaxyClusterRelation(AbstractMISP): def from_dict(self, **kwargs): # Default values for a valid event to send to a MISP instance - self.distribution = kwargs.pop('distribution', 0) - self.distribution = int(self.distribution) + self.distribution = int(kwargs.pop('distribution', 0)) if self.distribution not in [0, 1, 2, 3, 4, 5]: raise NewGalaxyClusterRelationError(f'{self.distribution} is invalid, the distribution has to be in 0, 1, 2, 3, 4') + if kwargs.get('sharing_group_id'): + self.sharing_group_id = int(kwargs.pop('sharing_group_id')) + + if self.distribution == 4: + # The distribution is set to sharing group, a sharing_group_id is required. + if not hasattr(self, 'sharing_group_id'): + raise NewGalaxyClusterRelationError('If the distribution is set to sharing group, a sharing group ID is required.') + elif not self.sharing_group_id: + # Cannot be None or 0 either. + raise NewGalaxyClusterRelationError('If the distribution is set to sharing group, a sharing group ID is required (cannot be {}).'.format(self.sharing_group_id)) + if kwargs.get('id'): self.id = int(kwargs.pop('id')) if kwargs.get('orgc_id'): @@ -1144,52 +1168,22 @@ class MISPGalaxyCluster(AbstractMISP): """A MISP galaxy cluster, storing respective galaxy elements and relations. Used to view default galaxy clusters and add/edit/update/delete Galaxy 2.0 clusters - :param Org: The organisation as a MISPOrganisation - :type Org: MISPOrganisation - :param Orgc: The creator organisation as a MISPOrganisation - :type Orgc: MISPOrganisation - :param SharingGroup: The SharingGroup applied to the cluster, if any - :type SharingGroup: MISPSharingGroup, optional - :param default: Whether the galaxy cluster is a default or custom cluster, default clusters cannot be edited - :type default: bool - :param deleted: Whether the galaxy cluster is deleted or not - :type deleted: bool + Creating a new galaxy cluster can take the following parameters + + :param value: The value of the galaxy cluster + :type value: str :param description: The description of the galaxy cluster :type description: str - :param distribution: The distribution type, one of 1, 2, 3, 4, 5 + :param distribution: The distribution type, one of 0, 1, 2, 3, 4 :type distribution: int :param sharing_group_id: The sharing group ID, if distribution is set to 4 :type sharing_group_id: int, optional - :param extends_uuid: The UUID of the galaxy cluster it extends - :type extends_uuid: uuid, optional - :param galaxy_id: The ID of the galaxy - :type galaxy_id: int - :param id: The ID of the galaxy cluster - :type id: int - :param org_id: The org's ID - :type org_id: int - :param orgc_id: The creating org's ID - :type orgc_id: int - :param published: Whether the cluster is published or not - :type published: bool - :param source: The source of the galaxy cluster - :type source: str - :param tag_count: The count of events using this galaxy cluster - :type tag_count: int - :param tag_id: The tag ID - :type tag_id: int - :param tag_name: The galaxy cluster's tag - :type tag_name: str - :param type: The type of the galaxy cluster, must match the housing galaxies type - :type type: str - :param value: The value of the galaxy cluster - :type value: str :param authors: A list of authors of the galaxy cluster - :type authors: list, optional + :type authors: list[str], optional :param cluster_elements: List of MISPGalaxyClusterElement - :type cluster_elements: list, optional - :param cluster_relations: List of MISPGalaxyClusterRelation, changes must be made through PyMISP instance - :type cluster_relations: list, optional + :type cluster_elements: list[MISPGalaxyClusterElement], optional + :param cluster_relations: List of MISPGalaxyClusterRelation + :type cluster_relations: list[MISPGalaxyClusterRelation], optional """ def __init__(self): @@ -1200,6 +1194,8 @@ class MISPGalaxyCluster(AbstractMISP): self.Org: MISPOrganisation self.Orgc: MISPOrganisation self.SharingGroup: MISPSharingGroup + # Set any inititialized cluster to be False + self.default = False @property def cluster_elements(self) -> List[MISPGalaxyClusterElement]: @@ -1217,18 +1213,45 @@ class MISPGalaxyCluster(AbstractMISP): def cluster_relations(self, cluster_relations: List[MISPGalaxyClusterRelation]): self.GalaxyClusterRelation = cluster_relations + @property + def meta(self) -> Dict: + """Function to return the galaxy cluster elements as a dictionary structure of lists + that comes from a MISPGalaxy within a MISPEvent. Lossy, you lose the element ID + """ + response = defaultdict(list) + for element in self.cluster_elements: + response[element.key].append(element.value) + return dict(response) + def from_dict(self, **kwargs): - # If the default field is set, we shouldn't have distribution or sharing group ID set if 'GalaxyCluster' in kwargs: kwargs = kwargs['GalaxyCluster'] - if kwargs.get('default', False): + self.default = kwargs.pop('default', False) + # If the default field is set, we shouldn't have distribution or sharing group ID set + if self.default: blocked_fields = ["distribution" "sharing_group_id"] for field in blocked_fields: if kwargs.get(field, None): raise NewGalaxyClusterError( - f"One of the following fields are set for a default galaxy cluster: {', '.join(blocked_fields)}" + f"The field '{field}' cannot be set on a default galaxy cluster" ) + + self.distribution = int(kwargs.pop('distribution', 0)) + if self.distribution not in [0, 1, 2, 3, 4]: + raise NewGalaxyClusterError(f'{self.distribution} is invalid, the distribution has to be in 0, 1, 2, 3, 4') + + if kwargs.get('sharing_group_id'): + self.sharing_group_id = int(kwargs.pop('sharing_group_id')) + + if self.distribution == 4: + # The distribution is set to sharing group, a sharing_group_id is required. + if not hasattr(self, 'sharing_group_id'): + raise NewGalaxyClusterError('If the distribution is set to sharing group, a sharing group ID is required.') + elif not self.sharing_group_id: + # Cannot be None or 0 either. + raise NewGalaxyClusterError('If the distribution is set to sharing group, a sharing group ID is required (cannot be {}).'.format(self.sharing_group_id)) + if 'uuid' in kwargs: self.uuid = kwargs.pop('uuid') if 'meta' in kwargs: @@ -1257,30 +1280,38 @@ class MISPGalaxyCluster(AbstractMISP): self.SharingGroup.from_dict(**kwargs.pop('SharingGroup')) super().from_dict(**kwargs) - def add_cluster_element(self, key: str, value: str, **kwargs): + def add_cluster_element(self, key: str, value: str, **kwargs) -> MISPGalaxyClusterElement: """Add a cluster relation to a MISPGalaxyCluster, key and value are required :param key: The key name of the element + :type key: str :param value: The value of the element + :type value: str """ + cluster_element = MISPGalaxyClusterElement() cluster_element.from_dict(key=key, value=value, **kwargs) self.cluster_elements.append(cluster_element) return cluster_element - def add_cluster_relation(self, referenced_galaxy_cluster_uuid: uuid, referenced_galaxy_cluster_type: str, galaxy_cluster_uuid: str = None, **kwargs): - """Add a cluster relation to a MISPGalaxyCluster + def add_cluster_relation(self, referenced_galaxy_cluster_uuid: uuid, referenced_galaxy_cluster_type: str, galaxy_cluster_uuid: str = None, **kwargs) -> MISPGalaxyClusterRelation: + """Add a cluster relation to a MISPGalaxyCluster. :param referenced_galaxy_cluster_uuid: UUID of the related cluster + :type referenced_galaxy_cluster_uuid: uuid :param referenced_galaxy_cluster_type: Relation type + :type referenced_galaxy_cluster_type: uuid + :param galaxy_cluster_uuid: UUID of this cluster, leave blank to use the stored UUID + :param galaxy_cluster_uuid: uuid, Optional """ + if not getattr(self, "uuid", None): raise PyMISPError("The cluster does not have a UUID, make sure it is a valid galaxy cluster") cluster_relation = MISPGalaxyClusterRelation() cluster_relation.from_dict( - galaxy_cluster_uuid=self.uuid, referenced_galaxy_cluster_uuid=referenced_galaxy_cluster_uuid, referenced_galaxy_cluster_type=referenced_galaxy_cluster_type, + galaxy_cluster_uuid=galaxy_cluster_uuid or self.uuid, **kwargs ) self.cluster_relations.append(cluster_relation) @@ -1293,27 +1324,7 @@ class MISPGalaxyCluster(AbstractMISP): class MISPGalaxy(AbstractMISP): - """Galaxy class, used to view a galaxy and respective clusters, supports the following fields - - :param id: Galaxy ID - :type id: int - :param uuid: Galaxy UUID - :type uuuid: uuid, str - :param name: Galaxy name - :type name: str - :param type: Galaxy type - :type type: str - :param description: Galaxy description - :type description: str - :param version: Galaxy version number - :type version: int - :param icon: Galaxy icon - :type icon: str - :param namespace: Galaxy namespace - :type namespace: str - :param clusters: List of MISPGalaxyCluster - :type clusters: list - """ + """Galaxy class, used to view a galaxy and respective clusters""" def __init__(self): super().__init__() @@ -1338,8 +1349,8 @@ class MISPGalaxy(AbstractMISP): return self.GalaxyCluster def add_galaxy_cluster(self, **kwargs) -> MISPGalaxyCluster: - """Add a MISP galaxy and sub-clusters into an event. - Supports all other parameters supported by MISPGalaxy""" + """Add a MISP galaxy cluster into a MISPGalaxy. + Supports all other parameters supported by MISPGalaxyCluster""" galaxy_cluster = MISPGalaxyCluster() galaxy_cluster.from_dict(**kwargs) From 7d4cfc40b7349022a3af85d0cad5cb75fddea006 Mon Sep 17 00:00:00 2001 From: Tom King Date: Sat, 30 Jan 2021 15:34:29 +0000 Subject: [PATCH 05/28] chg: Add in nosetests for MISP Galaxy functions, check default key as a dict attribute not MISPAbstract attribute --- pymisp/api.py | 9 +-- pymisp/mispevent.py | 7 ++- tests/testlive_comprehensive.py | 105 +++++++++++++++++++++++++++++++- 3 files changed, 115 insertions(+), 6 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index 6f39a24..5475705 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -1359,7 +1359,7 @@ class PyMISP: :param pythonify: Returns a PyMISP Object instead of the plain json output """ - if galaxy_cluster.default: + if getattr(galaxy_cluster, "default", False): # We can't add default galaxies raise PyMISPError('You are not able add a default galaxy cluster') galaxy_id = get_uuid_or_id_from_abstract_misp(galaxy) @@ -1378,10 +1378,11 @@ class PyMISP: :param pythonify: Returns a PyMISP Object instead of the plain json output """ - if galaxy_cluster.default: + if getattr(galaxy_cluster, "default", False): # We can't edit default galaxies raise PyMISPError('You are not able to update a default galaxy cluster') cluster_id = get_uuid_or_id_from_abstract_misp(galaxy_cluster) + print(cluster_id) r = self._prepare_request('POST', f'galaxy_clusters/edit/{cluster_id}', data=galaxy_cluster) cluster_j = self._check_json_response(r) if not (self.global_pythonify or pythonify) or 'errors' in cluster_j: @@ -1395,7 +1396,7 @@ class PyMISP: :param galaxy_cluster: The galaxy cluster you wish to publish """ - if isinstance(galaxy_cluster, MISPGalaxyCluster) and galaxy_cluster.default: + if isinstance(galaxy_cluster, MISPGalaxyCluster) and getattr(galaxy_cluster, "default", False): raise PyMISPError('You are not able to publish a default galaxy cluster') cluster_id = get_uuid_or_id_from_abstract_misp(galaxy_cluster) r = self._prepare_request('POST', f'galaxy_clusters/publish/{cluster_id}') @@ -1433,7 +1434,7 @@ class PyMISP: :param hard: flag for hard delete """ - if isinstance(galaxy_cluster, MISPGalaxyCluster) and galaxy_cluster.default: + if isinstance(galaxy_cluster, MISPGalaxyCluster) and getattr(galaxy_cluster, "default", False): raise PyMISPError('You are not able to delete a default galaxy cluster') data = {} if hard: diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index a3a2437..3665f57 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -9,6 +9,7 @@ import sys from io import BytesIO, BufferedIOBase, TextIOBase from zipfile import ZipFile import uuid +from uuid import UUID from collections import defaultdict import logging import hashlib @@ -1294,7 +1295,7 @@ class MISPGalaxyCluster(AbstractMISP): self.cluster_elements.append(cluster_element) return cluster_element - def add_cluster_relation(self, referenced_galaxy_cluster_uuid: uuid, referenced_galaxy_cluster_type: str, galaxy_cluster_uuid: str = None, **kwargs) -> MISPGalaxyClusterRelation: + def add_cluster_relation(self, referenced_galaxy_cluster_uuid: Union["MISPGalaxyCluster", str, UUID], referenced_galaxy_cluster_type: str, galaxy_cluster_uuid: str = None, **kwargs) -> MISPGalaxyClusterRelation: """Add a cluster relation to a MISPGalaxyCluster. :param referenced_galaxy_cluster_uuid: UUID of the related cluster @@ -1308,6 +1309,10 @@ class MISPGalaxyCluster(AbstractMISP): if not getattr(self, "uuid", None): raise PyMISPError("The cluster does not have a UUID, make sure it is a valid galaxy cluster") cluster_relation = MISPGalaxyClusterRelation() + + if isinstance(referenced_galaxy_cluster_uuid, MISPGalaxyCluster): + referenced_galaxy_cluster_uuid = referenced_galaxy_cluster_uuid.uuid + cluster_relation.from_dict( referenced_galaxy_cluster_uuid=referenced_galaxy_cluster_uuid, referenced_galaxy_cluster_type=referenced_galaxy_cluster_type, diff --git a/tests/testlive_comprehensive.py b/tests/testlive_comprehensive.py index 1478686..92e97b5 100644 --- a/tests/testlive_comprehensive.py +++ b/tests/testlive_comprehensive.py @@ -27,7 +27,7 @@ logger = logging.getLogger('pymisp') try: - from pymisp import register_user, PyMISP, MISPEvent, MISPOrganisation, MISPUser, Distribution, ThreatLevel, Analysis, MISPObject, MISPAttribute, MISPSighting, MISPShadowAttribute, MISPTag, MISPSharingGroup, MISPFeed, MISPServer, MISPUserSetting, MISPEventBlocklist, MISPEventReport + from pymisp import register_user, PyMISP, MISPEvent, MISPOrganisation, MISPUser, Distribution, ThreatLevel, Analysis, MISPObject, MISPAttribute, MISPSighting, MISPShadowAttribute, MISPTag, MISPSharingGroup, MISPFeed, MISPServer, MISPUserSetting, MISPEventBlocklist, MISPEventReport, MISPGalaxyCluster from pymisp.tools import CSVLoader, DomainIPObject, ASNObject, GenericObjectGenerator from pymisp.exceptions import MISPServerError except ImportError: @@ -2690,6 +2690,109 @@ class TestComprehensive(unittest.TestCase): self.user_misp_connector.delete_event(event) self.user_misp_connector.delete_event_report(new_event_report) + def test_galaxy_cluster(self): + self.admin_misp_connector.toggle_global_pythonify() + galaxy = self.admin_misp_connector.galaxies()[0] + new_galaxy_cluster = MISPGalaxyCluster() + new_galaxy_cluster.value = "Test Cluster" + new_galaxy_cluster.authors = ["MISP"] + new_galaxy_cluster.distribution = 1 + new_galaxy_cluster.description = "Example test cluster" + try: + galaxy = self.admin_misp_connector.get_galaxy(galaxy.id, withCluster=True) + existing_galaxy_cluster = galaxy.clusters[0] + + new_galaxy_cluster = self.admin_misp_connector.add_galaxy_cluster(galaxy.id, new_galaxy_cluster) + # The new galaxy cluster should be under the selected galaxy + self.assertEqual(galaxy.id, new_galaxy_cluster.galaxy_id) + # The cluster should have the right value + self.assertEqual(new_galaxy_cluster.value, "Test Cluster") + + new_galaxy_cluster.add_cluster_element("synonyms", "Test2") + new_galaxy_cluster = self.admin_misp_connector.update_galaxy_cluster(new_galaxy_cluster) + + # The cluster should have one element that is a synonym + self.assertEqual(len(new_galaxy_cluster.cluster_elements), 1) + element = new_galaxy_cluster.cluster_elements[0] + self.assertEqual(element.key, "synonyms") + self.assertEqual(element.value, "Test2") + + # The cluster should have the old meta as a prop + self.assertEqual(new_galaxy_cluster.meta, {'synonyms': ['Test2']}) + + # The cluster element should be updatable + element.value = "Test3" + new_galaxy_cluster = self.admin_misp_connector.update_galaxy_cluster(new_galaxy_cluster) + element = new_galaxy_cluster.cluster_elements[0] + self.assertEqual(element.value, "Test3") + + new_galaxy_cluster.add_cluster_element("synonyms", "ToDelete") + new_galaxy_cluster = self.admin_misp_connector.update_galaxy_cluster(new_galaxy_cluster) + # The cluster should have two elements + self.assertEqual(len(new_galaxy_cluster.cluster_elements), 2) + + new_galaxy_cluster.cluster_elements = [e for e in new_galaxy_cluster.cluster_elements if e.value != "ToDelete"] + new_galaxy_cluster = self.admin_misp_connector.update_galaxy_cluster(new_galaxy_cluster) + # The cluster elements should be deletable + self.assertEqual(len(new_galaxy_cluster.cluster_elements), 1) + + new_galaxy_cluster.add_cluster_relation(existing_galaxy_cluster, "is-tested-by") + new_galaxy_cluster = self.admin_misp_connector.update_galaxy_cluster(new_galaxy_cluster) + # The cluster should have a relationship + self.assertEqual(len(new_galaxy_cluster.cluster_relations), 1) + relation = new_galaxy_cluster.cluster_relations[0] + self.assertEqual(relation.referenced_galaxy_cluster_type, "is-tested-by") + self.assertEqual(relation.referenced_galaxy_cluster_uuid, existing_galaxy_cluster.uuid) + + relation.add_tag("tlp:amber") + new_galaxy_cluster = self.admin_misp_connector.update_galaxy_cluster(new_galaxy_cluster) + relation = new_galaxy_cluster.cluster_relations[0] + # The relationship should have a tag of tlp:amber + self.assertEqual(len(relation.tags), 1) + self.assertEqual(relation.tags[0].name, "tlp:amber") + + # The cluster relations should be deletable + resp = self.admin_misp_connector.delete_galaxy_cluster_relation(relation) + self.assertTrue(resp['success']) + # The cluster relation should no longer be present + new_galaxy_cluster = self.admin_misp_connector.get_galaxy_cluster(new_galaxy_cluster) + self.assertEqual(len(new_galaxy_cluster.cluster_relations), 0) + + resp = self.admin_misp_connector.delete_galaxy_cluster(new_galaxy_cluster) + # Galaxy clusters should be soft deletable + self.assertTrue(resp['success']) + new_galaxy_cluster = self.admin_misp_connector.get_galaxy_cluster(new_galaxy_cluster) + self.assertTrue(isinstance(new_galaxy_cluster, MISPGalaxyCluster)) + + resp = self.admin_misp_connector.delete_galaxy_cluster(new_galaxy_cluster, hard=True) + # Galaxy clusters should be hard deletable + self.assertTrue(resp['success']) + resp = self.admin_misp_connector.get_galaxy_cluster(new_galaxy_cluster) + self.assertTrue("errors" in resp) + finally: + self.admin_misp_connector.delete_galaxy_cluster(new_galaxy_cluster) + self.admin_misp_connector.toggle_global_pythonify() + + def test_event_galaxy(self): + self.admin_misp_connector.toggle_global_pythonify() + event = self.create_simple_event() + try: + galaxy = self.admin_misp_connector.galaxies()[0] + galaxy = self.admin_misp_connector.get_galaxy(galaxy.id, withCluster=True) + galaxy_cluster = galaxy.clusters[0] + event.add_tag(galaxy_cluster.tag_name) + event = self.admin_misp_connector.add_event(event) + # The event should have a galaxy attached + self.assertEqual(len(event.galaxies), 1) + event_galaxy = event.galaxies[0] + # The galaxy ID should equal the galaxy from which the cluster came from + self.assertEqual(event_galaxy.id, galaxy.id) + # The galaxy cluster should equal the cluster added + self.assertEqual(event_galaxy.clusters[0].id, galaxy_cluster.id) + finally: + self.admin_misp_connector.delete_event(event) + self.admin_misp_connector.toggle_global_pythonify() + @unittest.skip("Internal use only") def missing_methods(self): skip = [ From 544547996096f512096d35861d8b1186da71de5c Mon Sep 17 00:00:00 2001 From: Tom King Date: Mon, 8 Feb 2021 11:52:08 +0000 Subject: [PATCH 06/28] chg: Don't parse the meta key into cluster elements on a MISPEvent, but allow users to manually perform this action --- pymisp/mispevent.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index f39cd2f..bc0253d 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -1195,6 +1195,7 @@ class MISPGalaxyCluster(AbstractMISP): super().__init__() self.Galaxy: MISPGalaxy self.GalaxyElement: List[MISPGalaxyClusterElement] = [] + self.meta = {} self.GalaxyClusterRelation: List[MISPGalaxyClusterRelation] = [] self.Org: MISPOrganisation self.Orgc: MISPOrganisation @@ -1218,8 +1219,18 @@ class MISPGalaxyCluster(AbstractMISP): def cluster_relations(self, cluster_relations: List[MISPGalaxyClusterRelation]): self.GalaxyClusterRelation = cluster_relations + def parse_meta_as_elements(self): + """Function to parse the meta field into GalaxyClusterElements""" + # Parse the cluster elements from the kwargs meta fields + for key, value in self.meta.items(): + # The meta will merge fields together, i.e. Two 'countries' will be a list, so split these up + if not isinstance(value, list): + value = [value] + for v in value: + self.add_cluster_element(key=key, value=v) + @property - def meta(self) -> Dict: + def elements_meta(self) -> Dict: """Function to return the galaxy cluster elements as a dictionary structure of lists that comes from a MISPGalaxy within a MISPEvent. Lossy, you lose the element ID """ @@ -1260,13 +1271,7 @@ class MISPGalaxyCluster(AbstractMISP): if 'uuid' in kwargs: self.uuid = kwargs.pop('uuid') if 'meta' in kwargs: - # Parse the cluster elements from the kwargs meta fields - for key, value in kwargs.pop('meta').items(): - # The meta will merge fields together, i.e. Two 'countries' will be a list, so split these up - if not isinstance(value, list): - value = [value] - for v in value: - self.add_cluster_element(key=key, value=v) + self.meta = kwargs.pop('meta') if 'Galaxy' in kwargs: self.Galaxy = MISPGalaxy() self.Galaxy.from_dict(**kwargs.pop('Galaxy')) From ea86dd0d57ea66f65917bfdb1adc99178f455adb Mon Sep 17 00:00:00 2001 From: Tom King Date: Sat, 27 Feb 2021 12:35:24 +0000 Subject: [PATCH 07/28] fix: Fix mispevent edit test by including default and distribution keys on a GalaxyCluster --- tests/mispevent_testfiles/existing_event.json | 28 +++++++++++++++++++ .../existing_event_edited.json | 28 +++++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/tests/mispevent_testfiles/existing_event.json b/tests/mispevent_testfiles/existing_event.json index 073aa88..6d04a38 100644 --- a/tests/mispevent_testfiles/existing_event.json +++ b/tests/mispevent_testfiles/existing_event.json @@ -192,7 +192,9 @@ "Timo Steffens", "Various" ], + "default": false, "description": "The Sofacy Group (also known as APT28, Pawn Storm, Fancy Bear and Sednit) is a cyber espionage group believed to have ties to the Russian government. Likely operating since 2007, the group is known to target government, military, and security organizations. It has been characterized as an advanced persistent threat.", + "distribution": "0", "galaxy_id": "366", "id": "45563", "meta": { @@ -248,7 +250,9 @@ "Will Metcalf", "KahuSecurity" ], + "default": false, "description": "Sednit EK is the exploit kit used by APT28", + "distribution": "0", "galaxy_id": "370", "id": "38813", "meta": { @@ -274,7 +278,9 @@ "Will Metcalf", "KahuSecurity" ], + "default": false, "description": "DealersChoice is a Flash Player Exploit platform triggered by RTF", + "distribution": "0", "galaxy_id": "370", "id": "38805", "meta": { @@ -315,7 +321,9 @@ "Timo Steffens", "Christophe Vandeplas" ], + "default": false, "description": "backdoor", + "distribution": "0", "galaxy_id": "367", "id": "46592", "meta": { @@ -347,7 +355,9 @@ "Timo Steffens", "Christophe Vandeplas" ], + "default": false, "description": "", + "distribution": "0", "galaxy_id": "367", "id": "46670", "meta": { @@ -370,7 +380,9 @@ "Timo Steffens", "Christophe Vandeplas" ], + "default": false, "description": "backdoor used by apt28\n\nSedreco serves as a spying backdoor; its functionalities can be extended with dynamically loaded plugins. It is made up of two distinct components: a dropper and the persistent payload installed by this dropper. We have not seen this component since April 2016.", + "distribution": "0", "galaxy_id": "367", "id": "46591", "meta": { @@ -405,7 +417,9 @@ "Timo Steffens", "Christophe Vandeplas" ], + "default": false, "description": "This backdoor component is known to have a modular structure featuring various espionage functionalities, such as key-logging, screen grabbing and file exfiltration. This component is available for Osx, Windows, Linux and iOS operating systems.\n\nXagent is a modular backdoor with spying functionalities such as keystroke logging and file exfiltration. Xagent is the group’s flagship backdoor and heavily used in their operations. Early versions for Linux and Windows were seen years ago, then in 2015 an iOS version came out. One year later, an Android version was discovered and finally, in the beginning of 2017, an Xagent sample for OS X was described.", + "distribution": "0", "galaxy_id": "367", "id": "46669", "meta": { @@ -444,7 +458,9 @@ "authors": [ "MITRE" ], + "default": false, "description": "JHUHUGIT is malware used by APT28. It is based on Carberp source code and serves as reconnaissance malware.[[Citation: Kaspersky Sofacy]][[Citation: F-Secure Sofacy 2015]][[Citation: ESET Sednit Part 1]][[Citation: FireEye APT28 January 2017]]\n\nAliases: JHUHUGIT, Seduploader, JKEYSKW, Sednit, GAMEFISH", + "distribution": "0", "galaxy_id": "365", "id": "41618", "meta": { @@ -478,7 +494,9 @@ "authors": [ "MITRE" ], + "default": false, "description": "XTunnel a VPN-like network proxy tool that can relay traffic between a C2 server and a victim. It was first seen in May 2013 and reportedly used by APT28 during the compromise of the Democratic National Committee.[[Citation: Crowdstrike DNC June 2016]][[Citation: Invincea XTunnel]][[Citation: ESET Sednit Part 2]]\n\nAliases: XTunnel, X-Tunnel, XAPS", + "distribution": "0", "galaxy_id": "365", "id": "41543", "meta": { @@ -509,7 +527,9 @@ "authors": [ "MITRE" ], + "default": false, "description": "ADVSTORESHELL is a spying backdoor that has been used by APT28 from at least 2012 to 2016. It is generally used for long-term espionage and is deployed on targets deemed interesting after a reconnaissance phase.[[Citation: Kaspersky Sofacy]][[Citation: ESET Sednit Part 2]]\n\nAliases: ADVSTORESHELL, NETUI, EVILTOSS, AZZY, Sedreco", + "distribution": "0", "galaxy_id": "365", "id": "41582", "meta": { @@ -541,7 +561,9 @@ "authors": [ "MITRE" ], + "default": false, "description": "USBStealer is malware that has used by APT28 since at least 2005 to extract information from air-gapped networks. It does not have the capability to communicate over the Internet and has been used in conjunction with ADVSTORESHELL.[[Citation: ESET Sednit USBStealer 2014]][[Citation: Kaspersky Sofacy]]\n\nAliases: USBStealer, USB Stealer, Win32/USBStealer", + "distribution": "0", "galaxy_id": "365", "id": "41549", "meta": { @@ -571,7 +593,9 @@ "authors": [ "MITRE" ], + "default": false, "description": "is a trojan that has been used by APT28 on OS X and appears to be a port of their standard CHOPSTICK or XAgent trojan.[[Citation: XAgentOSX]]", + "distribution": "0", "galaxy_id": "365", "id": "41551", "meta": { @@ -595,7 +619,9 @@ "authors": [ "MITRE" ], + "default": false, "description": "CHOPSTICK is malware family of modular backdoors used by APT28. It has been used from at least November 2012 to August 2016 and is usually dropped on victims as second-stage malware, though it has been used as first-stage malware in several cases.[[Citation: FireEye APT28]][[Citation: ESET Sednit Part 2]][[Citation: FireEye APT28 January 2017]]\n\nAliases: CHOPSTICK, SPLM, Xagent, X-Agent, webhp", + "distribution": "0", "galaxy_id": "365", "id": "41559", "meta": { @@ -628,7 +654,9 @@ "authors": [ "MITRE" ], + "default": false, "description": "Downdelph is a first-stage downloader written in Delphi that has been used by APT28 in rare instances between 2013 and 2015.[[Citation: ESET Sednit Part 3]]\n\nAliases: Downdelph, Delphacy", + "distribution": "0", "galaxy_id": "365", "id": "41504", "meta": { diff --git a/tests/mispevent_testfiles/existing_event_edited.json b/tests/mispevent_testfiles/existing_event_edited.json index d869362..b5937d3 100644 --- a/tests/mispevent_testfiles/existing_event_edited.json +++ b/tests/mispevent_testfiles/existing_event_edited.json @@ -192,7 +192,9 @@ "Timo Steffens", "Various" ], + "default": false, "description": "The Sofacy Group (also known as APT28, Pawn Storm, Fancy Bear and Sednit) is a cyber espionage group believed to have ties to the Russian government. Likely operating since 2007, the group is known to target government, military, and security organizations. It has been characterized as an advanced persistent threat.", + "distribution": "0", "galaxy_id": "366", "id": "45563", "meta": { @@ -248,7 +250,9 @@ "Will Metcalf", "KahuSecurity" ], + "default": false, "description": "Sednit EK is the exploit kit used by APT28", + "distribution": "0", "galaxy_id": "370", "id": "38813", "meta": { @@ -274,7 +278,9 @@ "Will Metcalf", "KahuSecurity" ], + "default": false, "description": "DealersChoice is a Flash Player Exploit platform triggered by RTF", + "distribution": "0", "galaxy_id": "370", "id": "38805", "meta": { @@ -315,7 +321,9 @@ "Timo Steffens", "Christophe Vandeplas" ], + "default": false, "description": "backdoor", + "distribution": "0", "galaxy_id": "367", "id": "46592", "meta": { @@ -347,7 +355,9 @@ "Timo Steffens", "Christophe Vandeplas" ], + "default": false, "description": "", + "distribution": "0", "galaxy_id": "367", "id": "46670", "meta": { @@ -370,7 +380,9 @@ "Timo Steffens", "Christophe Vandeplas" ], + "default": false, "description": "backdoor used by apt28\n\nSedreco serves as a spying backdoor; its functionalities can be extended with dynamically loaded plugins. It is made up of two distinct components: a dropper and the persistent payload installed by this dropper. We have not seen this component since April 2016.", + "distribution": "0", "galaxy_id": "367", "id": "46591", "meta": { @@ -405,7 +417,9 @@ "Timo Steffens", "Christophe Vandeplas" ], + "default": false, "description": "This backdoor component is known to have a modular structure featuring various espionage functionalities, such as key-logging, screen grabbing and file exfiltration. This component is available for Osx, Windows, Linux and iOS operating systems.\n\nXagent is a modular backdoor with spying functionalities such as keystroke logging and file exfiltration. Xagent is the group’s flagship backdoor and heavily used in their operations. Early versions for Linux and Windows were seen years ago, then in 2015 an iOS version came out. One year later, an Android version was discovered and finally, in the beginning of 2017, an Xagent sample for OS X was described.", + "distribution": "0", "galaxy_id": "367", "id": "46669", "meta": { @@ -444,7 +458,9 @@ "authors": [ "MITRE" ], + "default": false, "description": "JHUHUGIT is malware used by APT28. It is based on Carberp source code and serves as reconnaissance malware.[[Citation: Kaspersky Sofacy]][[Citation: F-Secure Sofacy 2015]][[Citation: ESET Sednit Part 1]][[Citation: FireEye APT28 January 2017]]\n\nAliases: JHUHUGIT, Seduploader, JKEYSKW, Sednit, GAMEFISH", + "distribution": "0", "galaxy_id": "365", "id": "41618", "meta": { @@ -478,7 +494,9 @@ "authors": [ "MITRE" ], + "default": false, "description": "XTunnel a VPN-like network proxy tool that can relay traffic between a C2 server and a victim. It was first seen in May 2013 and reportedly used by APT28 during the compromise of the Democratic National Committee.[[Citation: Crowdstrike DNC June 2016]][[Citation: Invincea XTunnel]][[Citation: ESET Sednit Part 2]]\n\nAliases: XTunnel, X-Tunnel, XAPS", + "distribution": "0", "galaxy_id": "365", "id": "41543", "meta": { @@ -509,7 +527,9 @@ "authors": [ "MITRE" ], + "default": false, "description": "ADVSTORESHELL is a spying backdoor that has been used by APT28 from at least 2012 to 2016. It is generally used for long-term espionage and is deployed on targets deemed interesting after a reconnaissance phase.[[Citation: Kaspersky Sofacy]][[Citation: ESET Sednit Part 2]]\n\nAliases: ADVSTORESHELL, NETUI, EVILTOSS, AZZY, Sedreco", + "distribution": "0", "galaxy_id": "365", "id": "41582", "meta": { @@ -541,7 +561,9 @@ "authors": [ "MITRE" ], + "default": false, "description": "USBStealer is malware that has used by APT28 since at least 2005 to extract information from air-gapped networks. It does not have the capability to communicate over the Internet and has been used in conjunction with ADVSTORESHELL.[[Citation: ESET Sednit USBStealer 2014]][[Citation: Kaspersky Sofacy]]\n\nAliases: USBStealer, USB Stealer, Win32/USBStealer", + "distribution": "0", "galaxy_id": "365", "id": "41549", "meta": { @@ -571,7 +593,9 @@ "authors": [ "MITRE" ], + "default": false, "description": "is a trojan that has been used by APT28 on OS X and appears to be a port of their standard CHOPSTICK or XAgent trojan.[[Citation: XAgentOSX]]", + "distribution": "0", "galaxy_id": "365", "id": "41551", "meta": { @@ -595,7 +619,9 @@ "authors": [ "MITRE" ], + "default": false, "description": "CHOPSTICK is malware family of modular backdoors used by APT28. It has been used from at least November 2012 to August 2016 and is usually dropped on victims as second-stage malware, though it has been used as first-stage malware in several cases.[[Citation: FireEye APT28]][[Citation: ESET Sednit Part 2]][[Citation: FireEye APT28 January 2017]]\n\nAliases: CHOPSTICK, SPLM, Xagent, X-Agent, webhp", + "distribution": "0", "galaxy_id": "365", "id": "41559", "meta": { @@ -628,7 +654,9 @@ "authors": [ "MITRE" ], + "default": false, "description": "Downdelph is a first-stage downloader written in Delphi that has been used by APT28 in rare instances between 2013 and 2015.[[Citation: ESET Sednit Part 3]]\n\nAliases: Downdelph, Delphacy", + "distribution": "0", "galaxy_id": "365", "id": "41504", "meta": { From 5035911d4525c630d55d98194753728c3116db82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 10 Feb 2021 21:04:02 +0100 Subject: [PATCH 08/28] chg: Bump deps --- poetry.lock | 66 +++++++++++------------------------------------------ 1 file changed, 13 insertions(+), 53 deletions(-) diff --git a/poetry.lock b/poetry.lock index afe33a6..43bfe4e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -214,7 +214,7 @@ yaml = ["PyYAML (>=3.10)"] [[package]] name = "cryptography" -version = "3.4.1" +version = "3.4.4" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." category = "main" optional = true @@ -222,12 +222,12 @@ python-versions = ">=3.6" [package.dependencies] cffi = ">=1.12" -setuptools-rust = ">=0.11.4" [package.extras] docs = ["sphinx (>=1.6.5,!=1.8.0,!=3.1.0,!=3.1.1)", "sphinx-rtd-theme"] docstest = ["doc8", "pyenchant (>=1.6.11)", "twine (>=1.12.0)", "sphinxcontrib-spelling (>=4.0.1)"] pep8test = ["black", "flake8", "flake8-import-order", "pep8-naming"] +sdist = ["setuptools-rust (>=0.11.4)"] ssh = ["bcrypt (>=3.1.5)"] test = ["pytest (>=6.0)", "pytest-cov", "pytest-subtests", "pytest-xdist", "pretend", "iso8601", "pytz", "hypothesis (>=1.11.4,!=3.79.2)"] @@ -1087,14 +1087,6 @@ oletools = ">=0.56" dev = ["lxml (>=4.6)"] msg_parse = ["extract-msg (>=0.27)"] -[[package]] -name = "semantic-version" -version = "2.8.5" -description = "A library implementing the 'SemVer' scheme." -category = "main" -optional = true -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" - [[package]] name = "send2trash" version = "1.5.0" @@ -1103,18 +1095,6 @@ category = "dev" optional = false python-versions = "*" -[[package]] -name = "setuptools-rust" -version = "0.11.6" -description = "Setuptools rust extension plugin" -category = "main" -optional = true -python-versions = ">=3.5" - -[package.dependencies] -semantic-version = ">=2.6.0" -toml = ">=0.9.0" - [[package]] name = "six" version = "1.15.0" @@ -1133,11 +1113,11 @@ python-versions = "*" [[package]] name = "soupsieve" -version = "2.1" +version = "2.2" description = "A modern CSS selector implementation for Beautiful Soup." category = "main" optional = true -python-versions = ">=3.5" +python-versions = ">=3.6" [[package]] name = "sphinx" @@ -1280,14 +1260,6 @@ python-versions = "*" [package.extras] test = ["pathlib2"] -[[package]] -name = "toml" -version = "0.10.2" -description = "Python Library for Tom's Obvious, Minimal Language" -category = "main" -optional = true -python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" - [[package]] name = "tornado" version = "6.1" @@ -1638,13 +1610,13 @@ coveralls = [ {file = "coveralls-3.0.0.tar.gz", hash = "sha256:5399c0565ab822a70a477f7031f6c88a9dd196b3de2877b3facb43b51bd13434"}, ] cryptography = [ - {file = "cryptography-3.4.1-cp36-abi3-macosx_10_10_x86_64.whl", hash = "sha256:e63b8da1d77ff60a73d72db68cb72e8ffbe9f7319e5ffa23f6bfe2757d6871e3"}, - {file = "cryptography-3.4.1-cp36-abi3-manylinux2010_x86_64.whl", hash = "sha256:4d8df1f5b6b172fe53a465d8fc32134a07ccd4dc677f19af85562bcbc7e97504"}, - {file = "cryptography-3.4.1-cp36-abi3-manylinux2014_aarch64.whl", hash = "sha256:cd1d14b6c52d9372a7f4682576fa7d9c9d256afa6cc50828b337dbb8a9596066"}, - {file = "cryptography-3.4.1-cp36-abi3-manylinux2014_x86_64.whl", hash = "sha256:85c7d84beacf32bf629767f06c78af0350eb869e2830e4ebe05b239c695eca38"}, - {file = "cryptography-3.4.1-cp36-abi3-win32.whl", hash = "sha256:007edf78a0b96513b94c1c7cc515286aec72f787bdcd11892edcdec9e27e936d"}, - {file = "cryptography-3.4.1-cp36-abi3-win_amd64.whl", hash = "sha256:423c12d04df7ed3323e74745cba91056a411bd8f57609a6a64562845ccc5541a"}, - {file = "cryptography-3.4.1.tar.gz", hash = "sha256:be70bdaa29bcacf70896dae3a6f3eef91daf51bfba8a49dbfb9c23bb2cc914ba"}, + {file = "cryptography-3.4.4-cp36-abi3-macosx_10_10_x86_64.whl", hash = "sha256:287032b6a7d86abc98e8e977b20138c53fea40e5b24e29090d5a675a973dcd10"}, + {file = "cryptography-3.4.4-cp36-abi3-manylinux2010_x86_64.whl", hash = "sha256:7eed937ad9b53280a5f53570d3a7dc93cb4412b6a3d58d4c6bb78cc26319c729"}, + {file = "cryptography-3.4.4-cp36-abi3-manylinux2014_aarch64.whl", hash = "sha256:f21be9ec6b44c223b2024bbe59d394fadc7be320d18a8d595419afadb6cd5620"}, + {file = "cryptography-3.4.4-cp36-abi3-manylinux2014_x86_64.whl", hash = "sha256:dab437c2e84628703e3358f0f06555a6259bc5039209d51aa3b05af667ff4fd0"}, + {file = "cryptography-3.4.4-cp36-abi3-win32.whl", hash = "sha256:f6ea140d2736b7e1f0de4f988c43f76b0b3f3d365080e091715429ba218dce28"}, + {file = "cryptography-3.4.4-cp36-abi3-win_amd64.whl", hash = "sha256:288c65eea20bd89b11102c47b118bc1e0749386b0a0dfebba414076c5d4c8188"}, + {file = "cryptography-3.4.4.tar.gz", hash = "sha256:ee5e19f0856b6fbbdbab15c2787ca65d203801d2d65d0b8de6218f424206c848"}, ] decorator = [ {file = "decorator-4.4.2-py2.py3-none-any.whl", hash = "sha256:41fa54c2a0cc4ba648be4fd43cff00aedf5b9465c9bf18d64325bc225f08f760"}, @@ -2092,18 +2064,10 @@ rtfde = [ {file = "RTFDE-0.0.2-py3-none-any.whl", hash = "sha256:18386e4f060cee12a2a8035b0acf0cc99689f5dff1bf347bab7e92351860a21d"}, {file = "RTFDE-0.0.2.tar.gz", hash = "sha256:b86b5d734950fe8745a5b89133f50554252dbd67c6d1b9265e23ee140e7ea8a2"}, ] -semantic-version = [ - {file = "semantic_version-2.8.5-py2.py3-none-any.whl", hash = "sha256:45e4b32ee9d6d70ba5f440ec8cc5221074c7f4b0e8918bdab748cc37912440a9"}, - {file = "semantic_version-2.8.5.tar.gz", hash = "sha256:d2cb2de0558762934679b9a104e82eca7af448c9f4974d1f3eeccff651df8a54"}, -] send2trash = [ {file = "Send2Trash-1.5.0-py3-none-any.whl", hash = "sha256:f1691922577b6fa12821234aeb57599d887c4900b9ca537948d2dac34aea888b"}, {file = "Send2Trash-1.5.0.tar.gz", hash = "sha256:60001cc07d707fe247c94f74ca6ac0d3255aabcb930529690897ca2a39db28b2"}, ] -setuptools-rust = [ - {file = "setuptools-rust-0.11.6.tar.gz", hash = "sha256:a5b5954909cbc5d66b914ee6763f81fa2610916041c7266105a469f504a7c4ca"}, - {file = "setuptools_rust-0.11.6-py3-none-any.whl", hash = "sha256:5acf8cd8e89d57f0cd3cc942f60fa2ccfdede4c7a0b0d4b28eb7ab756df30347"}, -] six = [ {file = "six-1.15.0-py2.py3-none-any.whl", hash = "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced"}, {file = "six-1.15.0.tar.gz", hash = "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259"}, @@ -2113,8 +2077,8 @@ snowballstemmer = [ {file = "snowballstemmer-2.1.0.tar.gz", hash = "sha256:e997baa4f2e9139951b6f4c631bad912dfd3c792467e2f03d7239464af90e914"}, ] soupsieve = [ - {file = "soupsieve-2.1-py3-none-any.whl", hash = "sha256:4bb21a6ee4707bf43b61230e80740e71bfe56e55d1f1f50924b087bb2975c851"}, - {file = "soupsieve-2.1.tar.gz", hash = "sha256:6dc52924dc0bc710a5d16794e6b3480b2c7c08b07729505feab2b2c16661ff6e"}, + {file = "soupsieve-2.2-py3-none-any.whl", hash = "sha256:d3a5ea5b350423f47d07639f74475afedad48cf41c0ad7a82ca13a3928af34f6"}, + {file = "soupsieve-2.2.tar.gz", hash = "sha256:407fa1e8eb3458d1b5614df51d9651a1180ea5fedf07feb46e45d7e25e6d6cdd"}, ] sphinx = [ {file = "Sphinx-3.4.3-py3-none-any.whl", hash = "sha256:c314c857e7cd47c856d2c5adff514ac2e6495f8b8e0f886a8a37e9305dfea0d8"}, @@ -2156,10 +2120,6 @@ testpath = [ {file = "testpath-0.4.4-py2.py3-none-any.whl", hash = "sha256:bfcf9411ef4bf3db7579063e0546938b1edda3d69f4e1fb8756991f5951f85d4"}, {file = "testpath-0.4.4.tar.gz", hash = "sha256:60e0a3261c149755f4399a1fff7d37523179a70fdc3abdf78de9fc2604aeec7e"}, ] -toml = [ - {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, - {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, -] tornado = [ {file = "tornado-6.1-cp35-cp35m-macosx_10_9_x86_64.whl", hash = "sha256:d371e811d6b156d82aa5f9a4e08b58debf97c302a35714f6f45e35139c332e32"}, {file = "tornado-6.1-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:0d321a39c36e5f2c4ff12b4ed58d41390460f798422c4504e09eb5678e09998c"}, From 59bb0a7bb64804fcf498d31f237c0536f8603410 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 15 Feb 2021 12:00:06 +0100 Subject: [PATCH 09/28] fix: urllib3.__version__ may not have a patch number fix https://github.com/MISP/PyMISP/issues/698 --- pymisp/api.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pymisp/api.py b/pymisp/api.py index a920e93..7bc3329 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -95,7 +95,12 @@ def brotli_supported() -> bool: """ # urllib >= 1.25.1 includes brotli support - major, minor, patch = urllib3.__version__.split('.') # noqa: F811 + version_splitted = urllib3.__version__.split('.') # noqa: F811 + if len(version_splitted) == 2: + major, minor = version_splitted + patch = 0 + else: + major, minor, patch = version_splitted major, minor, patch = int(major), int(minor), int(patch) urllib3_with_brotli = (major == 1 and ((minor == 25 and patch >= 1) or (minor >= 26))) or major >= 2 From c87c0654b9fd4cd9ea49e706f794163b9762f435 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 15 Feb 2021 12:02:44 +0100 Subject: [PATCH 10/28] chg: Bump deps --- poetry.lock | 170 ++++++++++++++++++++++++++-------------------------- 1 file changed, 86 insertions(+), 84 deletions(-) diff --git a/poetry.lock b/poetry.lock index 43bfe4e..e2209a5 100644 --- a/poetry.lock +++ b/poetry.lock @@ -121,7 +121,7 @@ python-versions = "*" [[package]] name = "cffi" -version = "1.14.4" +version = "1.14.5" description = "Foreign Function Interface for Python calling C code." category = "main" optional = false @@ -214,7 +214,7 @@ yaml = ["PyYAML (>=3.10)"] [[package]] name = "cryptography" -version = "3.4.4" +version = "3.4.5" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." category = "main" optional = true @@ -936,7 +936,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [[package]] name = "pygments" -version = "2.7.4" +version = "2.8.0" description = "Pygments is a syntax highlighting package written in Python." category = "main" optional = false @@ -1003,7 +1003,7 @@ python-versions = "*" [[package]] name = "pyzmq" -version = "22.0.2" +version = "22.0.3" description = "Python bindings for 0MQ" category = "dev" optional = false @@ -1121,7 +1121,7 @@ python-versions = ">=3.6" [[package]] name = "sphinx" -version = "3.4.3" +version = "3.5.0" description = "Python documentation generator" category = "main" optional = true @@ -1146,9 +1146,9 @@ sphinxcontrib-qthelp = "*" sphinxcontrib-serializinghtml = "*" [package.extras] -docs = ["sphinxcontrib-websupport"] -lint = ["flake8 (>=3.5.0)", "isort", "mypy (>=0.790)", "docutils-stubs"] +lint = ["docutils-stubs", "flake8 (>=3.5.0)", "isort", "mypy (>=0.800)"] test = ["pytest", "pytest-cov", "html5lib", "cython", "typed-ast"] +docs = ["sphinxcontrib-websupport"] [[package]] name = "sphinx-autodoc-typehints" @@ -1497,40 +1497,43 @@ certifi = [ {file = "certifi-2020.12.5.tar.gz", hash = "sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c"}, ] cffi = [ - {file = "cffi-1.14.4-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:ebb253464a5d0482b191274f1c8bf00e33f7e0b9c66405fbffc61ed2c839c775"}, - {file = "cffi-1.14.4-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:2c24d61263f511551f740d1a065eb0212db1dbbbbd241db758f5244281590c06"}, - {file = "cffi-1.14.4-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:9f7a31251289b2ab6d4012f6e83e58bc3b96bd151f5b5262467f4bb6b34a7c26"}, - {file = "cffi-1.14.4-cp27-cp27m-win32.whl", hash = "sha256:5cf4be6c304ad0b6602f5c4e90e2f59b47653ac1ed9c662ed379fe48a8f26b0c"}, - {file = "cffi-1.14.4-cp27-cp27m-win_amd64.whl", hash = "sha256:f60567825f791c6f8a592f3c6e3bd93dd2934e3f9dac189308426bd76b00ef3b"}, - {file = "cffi-1.14.4-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:c6332685306b6417a91b1ff9fae889b3ba65c2292d64bd9245c093b1b284809d"}, - {file = "cffi-1.14.4-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:d9efd8b7a3ef378dd61a1e77367f1924375befc2eba06168b6ebfa903a5e59ca"}, - {file = "cffi-1.14.4-cp35-cp35m-macosx_10_9_x86_64.whl", hash = "sha256:51a8b381b16ddd370178a65360ebe15fbc1c71cf6f584613a7ea08bfad946698"}, - {file = "cffi-1.14.4-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:1d2c4994f515e5b485fd6d3a73d05526aa0fcf248eb135996b088d25dfa1865b"}, - {file = "cffi-1.14.4-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:af5c59122a011049aad5dd87424b8e65a80e4a6477419c0c1015f73fb5ea0293"}, - {file = "cffi-1.14.4-cp35-cp35m-win32.whl", hash = "sha256:594234691ac0e9b770aee9fcdb8fa02c22e43e5c619456efd0d6c2bf276f3eb2"}, - {file = "cffi-1.14.4-cp35-cp35m-win_amd64.whl", hash = "sha256:64081b3f8f6f3c3de6191ec89d7dc6c86a8a43911f7ecb422c60e90c70be41c7"}, - {file = "cffi-1.14.4-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:f803eaa94c2fcda012c047e62bc7a51b0bdabda1cad7a92a522694ea2d76e49f"}, - {file = "cffi-1.14.4-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:105abaf8a6075dc96c1fe5ae7aae073f4696f2905fde6aeada4c9d2926752362"}, - {file = "cffi-1.14.4-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:0638c3ae1a0edfb77c6765d487fee624d2b1ee1bdfeffc1f0b58c64d149e7eec"}, - {file = "cffi-1.14.4-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:7c6b1dece89874d9541fc974917b631406233ea0440d0bdfbb8e03bf39a49b3b"}, - {file = "cffi-1.14.4-cp36-cp36m-win32.whl", hash = "sha256:155136b51fd733fa94e1c2ea5211dcd4c8879869008fc811648f16541bf99668"}, - {file = "cffi-1.14.4-cp36-cp36m-win_amd64.whl", hash = "sha256:6bc25fc545a6b3d57b5f8618e59fc13d3a3a68431e8ca5fd4c13241cd70d0009"}, - {file = "cffi-1.14.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a7711edca4dcef1a75257b50a2fbfe92a65187c47dab5a0f1b9b332c5919a3fb"}, - {file = "cffi-1.14.4-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:00e28066507bfc3fe865a31f325c8391a1ac2916219340f87dfad602c3e48e5d"}, - {file = "cffi-1.14.4-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:798caa2a2384b1cbe8a2a139d80734c9db54f9cc155c99d7cc92441a23871c03"}, - {file = "cffi-1.14.4-cp37-cp37m-win32.whl", hash = "sha256:00a1ba5e2e95684448de9b89888ccd02c98d512064b4cb987d48f4b40aa0421e"}, - {file = "cffi-1.14.4-cp37-cp37m-win_amd64.whl", hash = "sha256:9cc46bc107224ff5b6d04369e7c595acb700c3613ad7bcf2e2012f62ece80c35"}, - {file = "cffi-1.14.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:df5169c4396adc04f9b0a05f13c074df878b6052430e03f50e68adf3a57aa28d"}, - {file = "cffi-1.14.4-cp38-cp38-manylinux1_i686.whl", hash = "sha256:9ffb888f19d54a4d4dfd4b3f29bc2c16aa4972f1c2ab9c4ab09b8ab8685b9c2b"}, - {file = "cffi-1.14.4-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:8d6603078baf4e11edc4168a514c5ce5b3ba6e3e9c374298cb88437957960a53"}, - {file = "cffi-1.14.4-cp38-cp38-win32.whl", hash = "sha256:b4e248d1087abf9f4c10f3c398896c87ce82a9856494a7155823eb45a892395d"}, - {file = "cffi-1.14.4-cp38-cp38-win_amd64.whl", hash = "sha256:ec80dc47f54e6e9a78181ce05feb71a0353854cc26999db963695f950b5fb375"}, - {file = "cffi-1.14.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:840793c68105fe031f34d6a086eaea153a0cd5c491cde82a74b420edd0a2b909"}, - {file = "cffi-1.14.4-cp39-cp39-manylinux1_i686.whl", hash = "sha256:b18e0a9ef57d2b41f5c68beefa32317d286c3d6ac0484efd10d6e07491bb95dd"}, - {file = "cffi-1.14.4-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:045d792900a75e8b1e1b0ab6787dd733a8190ffcf80e8c8ceb2fb10a29ff238a"}, - {file = "cffi-1.14.4-cp39-cp39-win32.whl", hash = "sha256:ba4e9e0ae13fc41c6b23299545e5ef73055213e466bd107953e4a013a5ddd7e3"}, - {file = "cffi-1.14.4-cp39-cp39-win_amd64.whl", hash = "sha256:f032b34669220030f905152045dfa27741ce1a6db3324a5bc0b96b6c7420c87b"}, - {file = "cffi-1.14.4.tar.gz", hash = "sha256:1a465cbe98a7fd391d47dce4b8f7e5b921e6cd805ef421d04f5f66ba8f06086c"}, + {file = "cffi-1.14.5-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:bb89f306e5da99f4d922728ddcd6f7fcebb3241fc40edebcb7284d7514741991"}, + {file = "cffi-1.14.5-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:34eff4b97f3d982fb93e2831e6750127d1355a923ebaeeb565407b3d2f8d41a1"}, + {file = "cffi-1.14.5-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:99cd03ae7988a93dd00bcd9d0b75e1f6c426063d6f03d2f90b89e29b25b82dfa"}, + {file = "cffi-1.14.5-cp27-cp27m-win32.whl", hash = "sha256:65fa59693c62cf06e45ddbb822165394a288edce9e276647f0046e1ec26920f3"}, + {file = "cffi-1.14.5-cp27-cp27m-win_amd64.whl", hash = "sha256:51182f8927c5af975fece87b1b369f722c570fe169f9880764b1ee3bca8347b5"}, + {file = "cffi-1.14.5-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:43e0b9d9e2c9e5d152946b9c5fe062c151614b262fda2e7b201204de0b99e482"}, + {file = "cffi-1.14.5-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:cbde590d4faaa07c72bf979734738f328d239913ba3e043b1e98fe9a39f8b2b6"}, + {file = "cffi-1.14.5-cp35-cp35m-macosx_10_9_x86_64.whl", hash = "sha256:5de7970188bb46b7bf9858eb6890aad302577a5f6f75091fd7cdd3ef13ef3045"}, + {file = "cffi-1.14.5-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:a465da611f6fa124963b91bf432d960a555563efe4ed1cc403ba5077b15370aa"}, + {file = "cffi-1.14.5-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:d42b11d692e11b6634f7613ad8df5d6d5f8875f5d48939520d351007b3c13406"}, + {file = "cffi-1.14.5-cp35-cp35m-win32.whl", hash = "sha256:72d8d3ef52c208ee1c7b2e341f7d71c6fd3157138abf1a95166e6165dd5d4369"}, + {file = "cffi-1.14.5-cp35-cp35m-win_amd64.whl", hash = "sha256:29314480e958fd8aab22e4a58b355b629c59bf5f2ac2492b61e3dc06d8c7a315"}, + {file = "cffi-1.14.5-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:3d3dd4c9e559eb172ecf00a2a7517e97d1e96de2a5e610bd9b68cea3925b4892"}, + {file = "cffi-1.14.5-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:48e1c69bbacfc3d932221851b39d49e81567a4d4aac3b21258d9c24578280058"}, + {file = "cffi-1.14.5-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:69e395c24fc60aad6bb4fa7e583698ea6cc684648e1ffb7fe85e3c1ca131a7d5"}, + {file = "cffi-1.14.5-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:9e93e79c2551ff263400e1e4be085a1210e12073a31c2011dbbda14bda0c6132"}, + {file = "cffi-1.14.5-cp36-cp36m-win32.whl", hash = "sha256:58e3f59d583d413809d60779492342801d6e82fefb89c86a38e040c16883be53"}, + {file = "cffi-1.14.5-cp36-cp36m-win_amd64.whl", hash = "sha256:005a36f41773e148deac64b08f233873a4d0c18b053d37da83f6af4d9087b813"}, + {file = "cffi-1.14.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:2894f2df484ff56d717bead0a5c2abb6b9d2bf26d6960c4604d5c48bbc30ee73"}, + {file = "cffi-1.14.5-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:0857f0ae312d855239a55c81ef453ee8fd24136eaba8e87a2eceba644c0d4c06"}, + {file = "cffi-1.14.5-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:cd2868886d547469123fadc46eac7ea5253ea7fcb139f12e1dfc2bbd406427d1"}, + {file = "cffi-1.14.5-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:35f27e6eb43380fa080dccf676dece30bef72e4a67617ffda586641cd4508d49"}, + {file = "cffi-1.14.5-cp37-cp37m-win32.whl", hash = "sha256:9ff227395193126d82e60319a673a037d5de84633f11279e336f9c0f189ecc62"}, + {file = "cffi-1.14.5-cp37-cp37m-win_amd64.whl", hash = "sha256:9cf8022fb8d07a97c178b02327b284521c7708d7c71a9c9c355c178ac4bbd3d4"}, + {file = "cffi-1.14.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8b198cec6c72df5289c05b05b8b0969819783f9418e0409865dac47288d2a053"}, + {file = "cffi-1.14.5-cp38-cp38-manylinux1_i686.whl", hash = "sha256:ad17025d226ee5beec591b52800c11680fca3df50b8b29fe51d882576e039ee0"}, + {file = "cffi-1.14.5-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:6c97d7350133666fbb5cf4abdc1178c812cb205dc6f41d174a7b0f18fb93337e"}, + {file = "cffi-1.14.5-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:8ae6299f6c68de06f136f1f9e69458eae58f1dacf10af5c17353eae03aa0d827"}, + {file = "cffi-1.14.5-cp38-cp38-win32.whl", hash = "sha256:b85eb46a81787c50650f2392b9b4ef23e1f126313b9e0e9013b35c15e4288e2e"}, + {file = "cffi-1.14.5-cp38-cp38-win_amd64.whl", hash = "sha256:1f436816fc868b098b0d63b8920de7d208c90a67212546d02f84fe78a9c26396"}, + {file = "cffi-1.14.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:1071534bbbf8cbb31b498d5d9db0f274f2f7a865adca4ae429e147ba40f73dea"}, + {file = "cffi-1.14.5-cp39-cp39-manylinux1_i686.whl", hash = "sha256:9de2e279153a443c656f2defd67769e6d1e4163952b3c622dcea5b08a6405322"}, + {file = "cffi-1.14.5-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:6e4714cc64f474e4d6e37cfff31a814b509a35cb17de4fb1999907575684479c"}, + {file = "cffi-1.14.5-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:158d0d15119b4b7ff6b926536763dc0714313aa59e320ddf787502c70c4d4bee"}, + {file = "cffi-1.14.5-cp39-cp39-win32.whl", hash = "sha256:afb29c1ba2e5a3736f1c301d9d0abe3ec8b86957d04ddfa9d7a6a42b9367e396"}, + {file = "cffi-1.14.5-cp39-cp39-win_amd64.whl", hash = "sha256:f2d45f97ab6bb54753eab54fffe75aaf3de4ff2341c9daee1987ee1837636f1d"}, + {file = "cffi-1.14.5.tar.gz", hash = "sha256:fd78e5fee591709f32ef6edb9a015b4aa1a5022598e36227500c8f4e02328d9c"}, ] chardet = [ {file = "chardet-4.0.0-py2.py3-none-any.whl", hash = "sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5"}, @@ -1610,13 +1613,13 @@ coveralls = [ {file = "coveralls-3.0.0.tar.gz", hash = "sha256:5399c0565ab822a70a477f7031f6c88a9dd196b3de2877b3facb43b51bd13434"}, ] cryptography = [ - {file = "cryptography-3.4.4-cp36-abi3-macosx_10_10_x86_64.whl", hash = "sha256:287032b6a7d86abc98e8e977b20138c53fea40e5b24e29090d5a675a973dcd10"}, - {file = "cryptography-3.4.4-cp36-abi3-manylinux2010_x86_64.whl", hash = "sha256:7eed937ad9b53280a5f53570d3a7dc93cb4412b6a3d58d4c6bb78cc26319c729"}, - {file = "cryptography-3.4.4-cp36-abi3-manylinux2014_aarch64.whl", hash = "sha256:f21be9ec6b44c223b2024bbe59d394fadc7be320d18a8d595419afadb6cd5620"}, - {file = "cryptography-3.4.4-cp36-abi3-manylinux2014_x86_64.whl", hash = "sha256:dab437c2e84628703e3358f0f06555a6259bc5039209d51aa3b05af667ff4fd0"}, - {file = "cryptography-3.4.4-cp36-abi3-win32.whl", hash = "sha256:f6ea140d2736b7e1f0de4f988c43f76b0b3f3d365080e091715429ba218dce28"}, - {file = "cryptography-3.4.4-cp36-abi3-win_amd64.whl", hash = "sha256:288c65eea20bd89b11102c47b118bc1e0749386b0a0dfebba414076c5d4c8188"}, - {file = "cryptography-3.4.4.tar.gz", hash = "sha256:ee5e19f0856b6fbbdbab15c2787ca65d203801d2d65d0b8de6218f424206c848"}, + {file = "cryptography-3.4.5-cp36-abi3-macosx_10_10_x86_64.whl", hash = "sha256:18d6f3ac1da14a01c95f4590ee58e96aa6628d231ce738e9eca330b9997127b6"}, + {file = "cryptography-3.4.5-cp36-abi3-manylinux2010_x86_64.whl", hash = "sha256:c8dc9859c5a046e1ca22da360dfd609c09064a4f974881cb5cba977c581088ec"}, + {file = "cryptography-3.4.5-cp36-abi3-manylinux2014_aarch64.whl", hash = "sha256:0bf49d5b38e4f3745a0eab0597fa97720dd49b30d65f534b49a82e303f149deb"}, + {file = "cryptography-3.4.5-cp36-abi3-manylinux2014_x86_64.whl", hash = "sha256:9c6f7552d4f2130542d488b9d9e5b1546204b5d1aa90c823d50cce8eed421363"}, + {file = "cryptography-3.4.5-cp36-abi3-win32.whl", hash = "sha256:b0873ac0c0e6bc6882cd285930cc382ec4e78786be71bdc113c06246eea61294"}, + {file = "cryptography-3.4.5-cp36-abi3-win_amd64.whl", hash = "sha256:8b3b79af57e12aabbc3db81e563eaa07870293a1ffdcc891d107035ce9a0dbff"}, + {file = "cryptography-3.4.5.tar.gz", hash = "sha256:4f6761a82b51fe02cda8f45af1c2f698a10f50003dc9c2572d8a49eda2e6d35b"}, ] decorator = [ {file = "decorator-4.4.2-py2.py3-none-any.whl", hash = "sha256:41fa54c2a0cc4ba648be4fd43cff00aedf5b9465c9bf18d64325bc225f08f760"}, @@ -1926,8 +1929,8 @@ pyflakes = [ {file = "pyflakes-2.2.0.tar.gz", hash = "sha256:35b2d75ee967ea93b55750aa9edbbf72813e06a66ba54438df2cfac9e3c27fc8"}, ] pygments = [ - {file = "Pygments-2.7.4-py3-none-any.whl", hash = "sha256:bc9591213a8f0e0ca1a5e68a479b4887fdc3e75d0774e5c71c31920c427de435"}, - {file = "Pygments-2.7.4.tar.gz", hash = "sha256:df49d09b498e83c1a73128295860250b0b7edd4c723a32e9bc0d295c7c2ec337"}, + {file = "Pygments-2.8.0-py3-none-any.whl", hash = "sha256:b21b072d0ccdf29297a82a2363359d99623597b8a265b8081760e4d0f7153c88"}, + {file = "Pygments-2.8.0.tar.gz", hash = "sha256:37a13ba168a02ac54cc5891a42b1caec333e59b66addb7fa633ea8a6d73445c0"}, ] pyparsing = [ {file = "pyparsing-2.4.7-py2.py3-none-any.whl", hash = "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b"}, @@ -1973,38 +1976,37 @@ pywinpty = [ {file = "pywinpty-0.5.7.tar.gz", hash = "sha256:2d7e9c881638a72ffdca3f5417dd1563b60f603e1b43e5895674c2a1b01f95a0"}, ] pyzmq = [ - {file = "pyzmq-22.0.2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:c2a8d70fe2a321a83d274970481eb244bff027b58511e943ef564721530ba786"}, - {file = "pyzmq-22.0.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:b68033181dc2e622bb5baa9b16d5933303779a03dc89860f4c44f629426d802c"}, - {file = "pyzmq-22.0.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:9bae89912cac9f03d41adb66981f6e753cfd4e451937b2cd435d732fd4ccb1a3"}, - {file = "pyzmq-22.0.2-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:75b68890219231bd60556a1c6e0d2dc05fa1b179a26c876442c83a0d77958bc9"}, - {file = "pyzmq-22.0.2-cp36-cp36m-win32.whl", hash = "sha256:c6b1d235a08f2c42480cb9a0a5cd2a29c391052d8bc8f43db86aa15387734a33"}, - {file = "pyzmq-22.0.2-cp36-cp36m-win_amd64.whl", hash = "sha256:f3ad3f77ed6a3cf31f61170fc1733afd83a4cf8e02edde0762d4e630bce2a97e"}, - {file = "pyzmq-22.0.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:490a9fe5509b09369722b18b85ef494abdf7c51cb1c9484cf83c3921961c2038"}, - {file = "pyzmq-22.0.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:303b8ebafce9906fc1e8eb35734b9dba4786ca3da7cdc88e04a8997dde2372d3"}, - {file = "pyzmq-22.0.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:1ffb81b08bcaaac30ba913adef686ff41b257252e96fca32497029fdc3962ff0"}, - {file = "pyzmq-22.0.2-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:75fa832c79ce30a23cd44a4e89224c651ef6bf5144b842ad066246e914b92233"}, - {file = "pyzmq-22.0.2-cp37-cp37m-win32.whl", hash = "sha256:d77f6eb839097e4bce96fcac7e05e33b677efe0385bd0ab6c2a9ea818ed7e8f9"}, - {file = "pyzmq-22.0.2-cp37-cp37m-win_amd64.whl", hash = "sha256:5a565af3729b2bf7c2ce1d563084d0cd90a312290ba5e571a0c3ec770ea8a287"}, - {file = "pyzmq-22.0.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ff236d8653f8bb74198223c7af77b9378714f411d6d95255d97c2d69bf991b20"}, - {file = "pyzmq-22.0.2-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:37beae88d6cf102419bb0ec79acb19c062dcea6765b57cf2b265dac5542bcdad"}, - {file = "pyzmq-22.0.2-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:bc9f2c26485dc76520084ee8d76f18171cc89f24f801bed8402302ee99dbbcd9"}, - {file = "pyzmq-22.0.2-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:0b32bd5e7346e534fddb57eab309933ff6b3b177c0106b908b6193dfa75fdabe"}, - {file = "pyzmq-22.0.2-cp38-cp38-win32.whl", hash = "sha256:58a074afa254a53872202e92594b59c0ba8cda62effc6437e34ae7048559dd38"}, - {file = "pyzmq-22.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:66d1190eec0a78bd07d39d1615b7923190ed1ba8aa04742d963b09bc66628681"}, - {file = "pyzmq-22.0.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:013e1343b41aaeb482f40605f3fadcfeb841706039625d7b30d12ae8fa0d3cd0"}, - {file = "pyzmq-22.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d66724bf0d423aa18c9ea43a1bf24ed5c1d143f00bdace7c1b7fc3034f188cc9"}, - {file = "pyzmq-22.0.2-cp39-cp39-manylinux2010_i686.whl", hash = "sha256:86cb0982b02b4fc2fbd4a65155289e0e4e5015982dbe2db14f8856c303cffa08"}, - {file = "pyzmq-22.0.2-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:7b6c855c562d1c1bf7a1ba72c2617c8298e0fa1b1c08dc8d60e225031567ad9e"}, - {file = "pyzmq-22.0.2-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:034f5b9e4ff0bcc67e49fe8f55a1b209ea5761c8fd00c246195c8d0cb6ce096d"}, - {file = "pyzmq-22.0.2-cp39-cp39-win32.whl", hash = "sha256:849444c1699c244d5770d3a684c51f024e95c538f71dd3d1ff423a91745bab7f"}, - {file = "pyzmq-22.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:506d4716ca6e5798345038e75adcb05b4118112a36700941967925285637198b"}, - {file = "pyzmq-22.0.2-pp36-pypy36_pp73-macosx_10_9_x86_64.whl", hash = "sha256:888d850d4b7e1426d210e901bd93075991b36fe0e2ae2547ce5c18b96df95250"}, - {file = "pyzmq-22.0.2-pp36-pypy36_pp73-manylinux2010_x86_64.whl", hash = "sha256:03c001be8c3817d5721137660ed21d90f6175002f0e583306079c791b1d9a855"}, - {file = "pyzmq-22.0.2-pp36-pypy36_pp73-win32.whl", hash = "sha256:3f4e6574d2589e3e22514a3669e86a7bf18a95d3c3ae65733fa6a0a769ec4c9d"}, - {file = "pyzmq-22.0.2-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:35c8c5c8160f0f0fc6d4588037243b668c3f20d981c1b8e7b5d9c33f8eeb7eb6"}, - {file = "pyzmq-22.0.2-pp37-pypy37_pp73-manylinux2010_x86_64.whl", hash = "sha256:841e9563ce9bd33fe9f227ec680ac033e9f1060977d613568c1dcbff09e74cc9"}, - {file = "pyzmq-22.0.2-pp37-pypy37_pp73-win32.whl", hash = "sha256:cc814880ba27f2ea8cea48ff3b480076266d4dd9c3fe29ef6e5a0a807639abe7"}, - {file = "pyzmq-22.0.2.tar.gz", hash = "sha256:d7b82a959e5e22d492f4f5a1e650e909a6c8c76ede178f538313ddb9d1e92963"}, + {file = "pyzmq-22.0.3-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:c0cde362075ee8f3d2b0353b283e203c2200243b5a15d5c5c03b78112a17e7d4"}, + {file = "pyzmq-22.0.3-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:ff1ea14075bbddd6f29bf6beb8a46d0db779bcec6b9820909584081ec119f8fd"}, + {file = "pyzmq-22.0.3-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:26380487eae4034d6c2a3fb8d0f2dff6dd0d9dd711894e8d25aa2d1938950a33"}, + {file = "pyzmq-22.0.3-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:3e29f9cf85a40d521d048b55c63f59d6c772ac1c4bf51cdfc23b62a62e377c33"}, + {file = "pyzmq-22.0.3-cp36-cp36m-win32.whl", hash = "sha256:4f34a173f813b38b83f058e267e30465ed64b22cd0cf6bad21148d3fa718f9bb"}, + {file = "pyzmq-22.0.3-cp36-cp36m-win_amd64.whl", hash = "sha256:30df70f81fe210506aa354d7fd486a39b87d9f7f24c3d3f4f698ec5d96b8c084"}, + {file = "pyzmq-22.0.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:7026f0353977431fc884abd4ac28268894bd1a780ba84bb266d470b0ec26d2ed"}, + {file = "pyzmq-22.0.3-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:6d4163704201fff0f3ab0cd5d7a0ea1514ecfffd3926d62ec7e740a04d2012c7"}, + {file = "pyzmq-22.0.3-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:763c175294d861869f18eb42901d500eda7d3fa4565f160b3b2fd2678ea0ebab"}, + {file = "pyzmq-22.0.3-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:61e4bb6cd60caf1abcd796c3f48395e22c5b486eeca6f3a8797975c57d94b03e"}, + {file = "pyzmq-22.0.3-cp37-cp37m-win32.whl", hash = "sha256:b25e5d339550a850f7e919fe8cb4c8eabe4c917613db48dab3df19bfb9a28969"}, + {file = "pyzmq-22.0.3-cp37-cp37m-win_amd64.whl", hash = "sha256:3ef50d74469b03725d781a2a03c57537d86847ccde587130fe35caafea8f75c6"}, + {file = "pyzmq-22.0.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:60e63577b85055e4cc43892fecd877b86695ee3ef12d5d10a3c5d6e77a7cc1a3"}, + {file = "pyzmq-22.0.3-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:f5831eff6b125992ec65d973f5151c48003b6754030094723ac4c6e80a97c8c4"}, + {file = "pyzmq-22.0.3-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:9221783dacb419604d5345d0e097bddef4459a9a95322de6c306bf1d9896559f"}, + {file = "pyzmq-22.0.3-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:b62ea18c0458a65ccd5be90f276f7a5a3f26a6dea0066d948ce2fa896051420f"}, + {file = "pyzmq-22.0.3-cp38-cp38-win32.whl", hash = "sha256:81e7df0da456206201e226491aa1fc449da85328bf33bbeec2c03bb3a9f18324"}, + {file = "pyzmq-22.0.3-cp38-cp38-win_amd64.whl", hash = "sha256:f52070871a0fd90a99130babf21f8af192304ec1e995bec2a9533efc21ea4452"}, + {file = "pyzmq-22.0.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:d18ddc6741b51f3985978f2fda57ddcdae359662d7a6b395bc8ff2292fca14bd"}, + {file = "pyzmq-22.0.3-cp39-cp39-manylinux2010_i686.whl", hash = "sha256:4231943514812dfb74f44eadcf85e8dd8cf302b4d0bce450ce1357cac88dbfdc"}, + {file = "pyzmq-22.0.3-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:23a74de4b43c05c3044aeba0d1f3970def8f916151a712a3ac1e5cd9c0bc2902"}, + {file = "pyzmq-22.0.3-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:532af3e6dddea62d9c49062ece5add998c9823c2419da943cf95589f56737de0"}, + {file = "pyzmq-22.0.3-cp39-cp39-win32.whl", hash = "sha256:33acd2b9790818b9d00526135acf12790649d8d34b2b04d64558b469c9d86820"}, + {file = "pyzmq-22.0.3-cp39-cp39-win_amd64.whl", hash = "sha256:a558c5bc89d56d7253187dccc4e81b5bb0eac5ae9511eb4951910a1245d04622"}, + {file = "pyzmq-22.0.3-pp36-pypy36_pp73-macosx_10_9_x86_64.whl", hash = "sha256:581787c62eaa0e0db6c5413cedc393ebbadac6ddfd22e1cf9a60da23c4f1a4b2"}, + {file = "pyzmq-22.0.3-pp36-pypy36_pp73-manylinux2010_x86_64.whl", hash = "sha256:38e3dca75d81bec4f2defa14b0a65b74545812bb519a8e89c8df96bbf4639356"}, + {file = "pyzmq-22.0.3-pp36-pypy36_pp73-win32.whl", hash = "sha256:2f971431aaebe0a8b54ac018e041c2f0b949a43745444e4dadcc80d0f0ef8457"}, + {file = "pyzmq-22.0.3-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:da7d4d4c778c86b60949d17531e60c54ed3726878de8a7f8a6d6e7f8cc8c3205"}, + {file = "pyzmq-22.0.3-pp37-pypy37_pp73-manylinux2010_x86_64.whl", hash = "sha256:13465c1ff969cab328bc92f7015ce3843f6e35f8871ad79d236e4fbc85dbe4cb"}, + {file = "pyzmq-22.0.3-pp37-pypy37_pp73-win32.whl", hash = "sha256:279cc9b51db48bec2db146f38e336049ac5a59e5f12fb3a8ad864e238c1c62e3"}, + {file = "pyzmq-22.0.3.tar.gz", hash = "sha256:f7f63ce127980d40f3e6a5fdb87abf17ce1a7c2bd8bf2c7560e1bbce8ab1f92d"}, ] recommonmark = [ {file = "recommonmark-0.7.1-py2.py3-none-any.whl", hash = "sha256:1b1db69af0231efce3fa21b94ff627ea33dee7079a01dd0a7f8482c3da148b3f"}, @@ -2081,8 +2083,8 @@ soupsieve = [ {file = "soupsieve-2.2.tar.gz", hash = "sha256:407fa1e8eb3458d1b5614df51d9651a1180ea5fedf07feb46e45d7e25e6d6cdd"}, ] sphinx = [ - {file = "Sphinx-3.4.3-py3-none-any.whl", hash = "sha256:c314c857e7cd47c856d2c5adff514ac2e6495f8b8e0f886a8a37e9305dfea0d8"}, - {file = "Sphinx-3.4.3.tar.gz", hash = "sha256:41cad293f954f7d37f803d97eb184158cfd90f51195131e94875bc07cd08b93c"}, + {file = "Sphinx-3.5.0-py3-none-any.whl", hash = "sha256:68da66ca3d6b35b22bea5c53d938d5f8988663dca042f0a46429a1eba1010051"}, + {file = "Sphinx-3.5.0.tar.gz", hash = "sha256:deb468efb3abaa70d790add4147d18782d86fdeacf648d6e8afb7a99807f1546"}, ] sphinx-autodoc-typehints = [ {file = "sphinx-autodoc-typehints-1.11.1.tar.gz", hash = "sha256:244ba6d3e2fdb854622f643c7763d6f95b6886eba24bec28e86edf205e4ddb20"}, From 1b675bb5122d4135ef8474ef8518d13e69603465 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 15 Feb 2021 16:11:15 +0100 Subject: [PATCH 11/28] fix: Skip PE section if name is none AND size is 0. Related: #678 --- pymisp/tools/peobject.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pymisp/tools/peobject.py b/pymisp/tools/peobject.py index ea81f75..014fb07 100644 --- a/pymisp/tools/peobject.py +++ b/pymisp/tools/peobject.py @@ -113,6 +113,9 @@ class PEObject(AbstractMISPObjectGenerator): if self.__pe.sections: pos = 0 for section in self.__pe.sections: + if not section.name and not section.size: + # Skip section if name is none AND size is 0. + continue s = PESectionObject(section, standalone=self._standalone, default_attributes_parameters=self._default_attributes_parameters) self.add_reference(s.uuid, 'includes', 'Section {} of PE'.format(pos)) if ((self.__pe.entrypoint >= section.virtual_address) From b9f7bd9dc115f68b1f13160dc48a30afe5f7ac5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 15 Feb 2021 16:12:38 +0100 Subject: [PATCH 12/28] chg: Add deprecation warning for Python < 3.8 --- pymisp/__init__.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pymisp/__init__.py b/pymisp/__init__.py index 25fcda9..f8d8fc7 100644 --- a/pymisp/__init__.py +++ b/pymisp/__init__.py @@ -1,5 +1,7 @@ __version__ = '2.4.138' import logging +import sys +import warnings FORMAT = "%(levelname)s [%(filename)s:%(lineno)s - %(funcName)s() ] %(message)s" formatter = logging.Formatter(FORMAT) @@ -11,6 +13,14 @@ logger.addHandler(default_handler) logger.setLevel(logging.WARNING) +def warning_2022(): + if sys.version_info < (3, 8): + warnings.warn(""" +As our baseline system is the latest Ubuntu LTS, and Ubuntu LTS 20.04 has Python 3.8 available, +we will officially deprecate python versions below 3.8 on January 1st 2022. +**Please update your codebase.**""", DeprecationWarning, stacklevel=3) + + everything_broken = '''Unknown error: the response is not in JSON. Something is broken server-side, please send us everything that follows (careful with the auth key): Request headers: @@ -22,6 +32,7 @@ Response (if any): try: + warning_2022() from .exceptions import PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey, InvalidMISPObject, UnknownMISPObjectTemplate, PyMISPInvalidFormat, MISPServerError, PyMISPNotImplementedYet, PyMISPUnexpectedResponse, PyMISPEmptyResponse # noqa from .abstract import AbstractMISP, MISPEncode, pymisp_json_default, MISPTag, Distribution, ThreatLevel, Analysis # noqa from .mispevent import MISPEvent, MISPAttribute, MISPObjectReference, MISPObjectAttribute, MISPObject, MISPUser, MISPOrganisation, MISPSighting, MISPLog, MISPShadowAttribute, MISPWarninglist, MISPTaxonomy, MISPNoticelist, MISPObjectTemplate, MISPSharingGroup, MISPRole, MISPServer, MISPFeed, MISPEventDelegation, MISPUserSetting, MISPInbox, MISPEventBlocklist, MISPOrganisationBlocklist, MISPEventReport, MISPGalaxyCluster, MISPGalaxyClusterElement, MISPGalaxyClusterRelation # noqa From e183dbc577362097d28f4aa4586449b63e6695e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 16 Feb 2021 18:34:54 +0100 Subject: [PATCH 13/28] fix: Do not add the serial-number twice. Related: #678 --- pymisp/tools/peobject.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pymisp/tools/peobject.py b/pymisp/tools/peobject.py index 014fb07..b94fe59 100644 --- a/pymisp/tools/peobject.py +++ b/pymisp/tools/peobject.py @@ -153,7 +153,6 @@ class PECertificate(AbstractMISPObjectGenerator): self.add_attribute('version', value=self.__certificate.version) self.add_attribute('subject', value=self.__certificate.subject) self.add_attribute('signature_algorithm', value=self.__certificate.signature_algorithm) - self.add_attribute('serial-number', value=self.__certificate.serial_number) self.add_attribute('raw-base64', value=b64encode(self.__certificate.raw)) From 25e44e7d8d691383afc5da6e1fb246c0cd65b074 Mon Sep 17 00:00:00 2001 From: Nick Date: Wed, 17 Feb 2021 12:07:11 -0500 Subject: [PATCH 14/28] Added check for invalid creds Without the added check, the script will error out on line 29 since the key doesn't exist in the dict. This at least gives a reason. --- examples/proofpoint_tap.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/examples/proofpoint_tap.py b/examples/proofpoint_tap.py index 561191e..b5a0fce 100644 --- a/examples/proofpoint_tap.py +++ b/examples/proofpoint_tap.py @@ -22,6 +22,9 @@ headers = { } responseSiem = requests.request("GET", urlSiem, headers=headers, params=queryString) +if 'Credentials authentication failed' in str(responseSiem.text): + print("Credentials invalid, please edit keys.py and try again") + quit() jsonDataSiem = json.loads(responseSiem.text) From ffd4677c993846b1616cd5278b024c06e8e279e2 Mon Sep 17 00:00:00 2001 From: Nick Date: Wed, 17 Feb 2021 12:09:01 -0500 Subject: [PATCH 15/28] removed cast of str to str --- examples/proofpoint_tap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/proofpoint_tap.py b/examples/proofpoint_tap.py index b5a0fce..532a761 100644 --- a/examples/proofpoint_tap.py +++ b/examples/proofpoint_tap.py @@ -22,7 +22,7 @@ headers = { } responseSiem = requests.request("GET", urlSiem, headers=headers, params=queryString) -if 'Credentials authentication failed' in str(responseSiem.text): +if 'Credentials authentication failed' in responseSiem.text: print("Credentials invalid, please edit keys.py and try again") quit() From 1ea59931e08a8cd4301af51f99946455f40fd392 Mon Sep 17 00:00:00 2001 From: Nick Date: Wed, 17 Feb 2021 14:57:59 -0500 Subject: [PATCH 16/28] Multiple updates to proofpoint example - Added additionally necessary keys to keys.py.example - Added error check for unset keys - Used built-in HTTP Basic Auth for requests instead of manually-created header - Removed setting of orgc as that's pulled from the MISP key being used - --- examples/keys.py.sample | 7 +++++-- examples/proofpoint_tap.py | 28 +++++++++++++++------------- 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/examples/keys.py.sample b/examples/keys.py.sample index f1166c8..9a81d75 100644 --- a/examples/keys.py.sample +++ b/examples/keys.py.sample @@ -1,8 +1,11 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -misp_url = 'https:///' +misp_url = 'https:// your MISP URL /' misp_key = 'Your MISP auth key' # The MISP auth key can be found on the MISP web interface under the automation section misp_verifycert = True misp_client_cert = '' -proofpoint_key = 'Your Proofpoint TAP auth key' +misp_orgID = '2' # Org ID to use for ingesting events +misp_orgUUID = '11111111-2222-3333-4444-555555555555' # Org UUID to use for ingesting events +proofpoint_sp = '' # Service Principal from TAP (https://threatinsight.proofpoint.com//settings/connected-applications) +proofpoint_secret = '' \ No newline at end of file diff --git a/examples/proofpoint_tap.py b/examples/proofpoint_tap.py index 532a761..b50824b 100644 --- a/examples/proofpoint_tap.py +++ b/examples/proofpoint_tap.py @@ -1,7 +1,17 @@ import requests +from requests.auth import HTTPBasicAuth import json from pymisp import ExpandedPyMISP, MISPEvent, MISPOrganisation -from keys import misp_url, misp_key, misp_verifycert, proofpoint_key +from keys import misp_url, misp_key, misp_verifycert, proofpoint_sp, proofpoint_secret, misp_orgID, misp_orgUUID + +################# Edit these ################# +orgID = misp_orgID +orgUUID = misp_orgUUID +############################################## + +if orgUUID == '11111111-2222-3333-4444-555555555555': + print('Please edit the orgID and orgUUID variables in keys.py') + quit() # initialize PyMISP and set url for Panorama misp = ExpandedPyMISP(url=misp_url, key=misp_key, ssl=misp_verifycert) @@ -16,27 +26,19 @@ queryString = { "format": "json" } -# auth to api needs to be set as a header, not as part of the query string -headers = { - 'Authorization': "Basic " + proofpoint_key -} -responseSiem = requests.request("GET", urlSiem, headers=headers, params=queryString) + +responseSiem = requests.request("GET", urlSiem, params=queryString, auth=HTTPBasicAuth(proofpoint_sp, proofpoint_secret)) if 'Credentials authentication failed' in responseSiem.text: - print("Credentials invalid, please edit keys.py and try again") + print('Credentials invalid, please edit keys.py and try again') quit() jsonDataSiem = json.loads(responseSiem.text) for alert in alertType: for messages in jsonDataSiem[alert]: - orgc = MISPOrganisation() - orgc.name = 'Proofpoint' - orgc.id = '#{ORGC.ID}' # organisation id - orgc.uuid = '#{ORGC.UUID}' # organisation uuid # initialize and set MISPEvent() event = MISPEvent() - event.Orgc = orgc if alert == "messagesDelivered" or alert == "messagesBlocked": if alert == "messagesDelivered": event.info = alert @@ -115,7 +117,7 @@ for alert in alertType: # get campaignID from each TAP alert and query campaign API if threatInfo["campaignID"] is not None and threatInfo["campaignID"] != "": urlCampaign = "https://tap-api-v2.proofpoint.com/v2/campaign/" + threatInfo["campaignID"] - responseCampaign = requests.request("GET", urlCampaign, headers=headers) + responseCampaign = requests.request("GET", urlCampaign, auth=HTTPBasicAuth(proofpoint_sp, proofpoint_secret)) jsonDataCampaign = json.loads(responseCampaign.text) From 85a67a60e67bbf0a46304ab203726f51f98e3f8f Mon Sep 17 00:00:00 2001 From: Nick Date: Wed, 17 Feb 2021 14:58:54 -0500 Subject: [PATCH 17/28] re-added brackets --- examples/keys.py.sample | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/keys.py.sample b/examples/keys.py.sample index 9a81d75..36fa465 100644 --- a/examples/keys.py.sample +++ b/examples/keys.py.sample @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -misp_url = 'https:// your MISP URL /' +misp_url = 'https:///' misp_key = 'Your MISP auth key' # The MISP auth key can be found on the MISP web interface under the automation section misp_verifycert = True misp_client_cert = '' From 05b50ba2f54171e4c6c0e06803365368dc9380bc Mon Sep 17 00:00:00 2001 From: Nick Date: Wed, 17 Feb 2021 15:01:13 -0500 Subject: [PATCH 18/28] deleted all references to org as it's unneeded --- examples/keys.py.sample | 2 -- examples/proofpoint_tap.py | 11 +---------- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/examples/keys.py.sample b/examples/keys.py.sample index 36fa465..7043aaa 100644 --- a/examples/keys.py.sample +++ b/examples/keys.py.sample @@ -5,7 +5,5 @@ misp_url = 'https:///' misp_key = 'Your MISP auth key' # The MISP auth key can be found on the MISP web interface under the automation section misp_verifycert = True misp_client_cert = '' -misp_orgID = '2' # Org ID to use for ingesting events -misp_orgUUID = '11111111-2222-3333-4444-555555555555' # Org UUID to use for ingesting events proofpoint_sp = '' # Service Principal from TAP (https://threatinsight.proofpoint.com//settings/connected-applications) proofpoint_secret = '' \ No newline at end of file diff --git a/examples/proofpoint_tap.py b/examples/proofpoint_tap.py index b50824b..1cc3bb6 100644 --- a/examples/proofpoint_tap.py +++ b/examples/proofpoint_tap.py @@ -2,16 +2,7 @@ import requests from requests.auth import HTTPBasicAuth import json from pymisp import ExpandedPyMISP, MISPEvent, MISPOrganisation -from keys import misp_url, misp_key, misp_verifycert, proofpoint_sp, proofpoint_secret, misp_orgID, misp_orgUUID - -################# Edit these ################# -orgID = misp_orgID -orgUUID = misp_orgUUID -############################################## - -if orgUUID == '11111111-2222-3333-4444-555555555555': - print('Please edit the orgID and orgUUID variables in keys.py') - quit() +from keys import misp_url, misp_key, misp_verifycert, proofpoint_sp, proofpoint_secret # initialize PyMISP and set url for Panorama misp = ExpandedPyMISP(url=misp_url, key=misp_key, ssl=misp_verifycert) From 4787be1bce257986d0268782f7d6c7f000fad9a6 Mon Sep 17 00:00:00 2001 From: Nick Date: Wed, 17 Feb 2021 15:06:25 -0500 Subject: [PATCH 19/28] re-added error checking for defaults --- examples/keys.py.sample | 4 ++-- examples/proofpoint_tap.py | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/examples/keys.py.sample b/examples/keys.py.sample index 7043aaa..3c59bfe 100644 --- a/examples/keys.py.sample +++ b/examples/keys.py.sample @@ -5,5 +5,5 @@ misp_url = 'https:///' misp_key = 'Your MISP auth key' # The MISP auth key can be found on the MISP web interface under the automation section misp_verifycert = True misp_client_cert = '' -proofpoint_sp = '' # Service Principal from TAP (https://threatinsight.proofpoint.com//settings/connected-applications) -proofpoint_secret = '' \ No newline at end of file +proofpoint_sp = '' # Service Principal from TAP (https://threatinsight.proofpoint.com//settings/connected-applications) +proofpoint_secret = '' \ No newline at end of file diff --git a/examples/proofpoint_tap.py b/examples/proofpoint_tap.py index 1cc3bb6..06e826e 100644 --- a/examples/proofpoint_tap.py +++ b/examples/proofpoint_tap.py @@ -4,6 +4,10 @@ import json from pymisp import ExpandedPyMISP, MISPEvent, MISPOrganisation from keys import misp_url, misp_key, misp_verifycert, proofpoint_sp, proofpoint_secret +if proofpoint_secret == '': + print('Set the proofpoint_secret in keys.py before running. Exiting...') + quit() + # initialize PyMISP and set url for Panorama misp = ExpandedPyMISP(url=misp_url, key=misp_key, ssl=misp_verifycert) From 518e2e6a31cca0fe71842c6958cf6206650ddb40 Mon Sep 17 00:00:00 2001 From: Nick Date: Wed, 17 Feb 2021 15:10:21 -0500 Subject: [PATCH 20/28] supress ssl warnings --- examples/proofpoint_tap.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/examples/proofpoint_tap.py b/examples/proofpoint_tap.py index 06e826e..18e1452 100644 --- a/examples/proofpoint_tap.py +++ b/examples/proofpoint_tap.py @@ -3,6 +3,8 @@ from requests.auth import HTTPBasicAuth import json from pymisp import ExpandedPyMISP, MISPEvent, MISPOrganisation from keys import misp_url, misp_key, misp_verifycert, proofpoint_sp, proofpoint_secret +import urllib3 +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) if proofpoint_secret == '': print('Set the proofpoint_secret in keys.py before running. Exiting...') From b09c8102b711d47d1d04af8751b389b4585d2354 Mon Sep 17 00:00:00 2001 From: Nick Date: Thu, 18 Feb 2021 11:33:34 -0500 Subject: [PATCH 21/28] Removed unused import --- examples/proofpoint_tap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/proofpoint_tap.py b/examples/proofpoint_tap.py index 18e1452..d76aa3f 100644 --- a/examples/proofpoint_tap.py +++ b/examples/proofpoint_tap.py @@ -1,7 +1,7 @@ import requests from requests.auth import HTTPBasicAuth import json -from pymisp import ExpandedPyMISP, MISPEvent, MISPOrganisation +from pymisp import ExpandedPyMISP, MISPEvent from keys import misp_url, misp_key, misp_verifycert, proofpoint_sp, proofpoint_secret import urllib3 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) From 125961a67042aa0769fb7b702801d5099ea7566c Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Sat, 20 Feb 2021 17:28:50 +0100 Subject: [PATCH 22/28] chg: [data] describeTypes updated --- pymisp/data/describeTypes.json | 1030 ++++++++++++++++---------------- 1 file changed, 521 insertions(+), 509 deletions(-) diff --git a/pymisp/data/describeTypes.json b/pymisp/data/describeTypes.json index f3e4bde..645df75 100644 --- a/pymisp/data/describeTypes.json +++ b/pymisp/data/describeTypes.json @@ -1,138 +1,134 @@ { "result": { "categories": [ + "Internal reference", + "Targeting data", "Antivirus detection", + "Payload delivery", "Artifacts dropped", + "Payload installation", + "Persistence mechanism", + "Network activity", + "Payload type", "Attribution", "External analysis", "Financial fraud", - "Internal reference", - "Network activity", - "Other", - "Payload delivery", - "Payload installation", - "Payload type", - "Persistence mechanism", - "Person", - "Social network", "Support Tool", - "Targeting data" + "Social network", + "Person", + "Other" ], "category_type_mappings": { "Antivirus detection": [ - "anonymised", - "attachment", - "comment", - "hex", "link", + "comment", + "text", + "hex", + "attachment", "other", - "text" + "anonymised" ], "Artifacts dropped": [ - "anonymised", - "attachment", - "authentihash", - "cdhash", - "comment", - "cookie", - "filename", - "filename-pattern", - "filename|authentihash", - "filename|impfuzzy", - "filename|imphash", - "filename|md5", - "filename|pehash", - "filename|sha1", - "filename|sha224", - "filename|sha256", - "filename|sha3-224", - "filename|sha3-256", - "filename|sha3-384", - "filename|sha3-512", - "filename|sha384", - "filename|sha512", - "filename|sha512/224", - "filename|sha512/256", - "filename|ssdeep", - "filename|tlsh", - "filename|vhash", - "gene", - "hex", - "impfuzzy", - "imphash", - "kusto-query", - "malware-sample", "md5", - "mime-type", - "mutex", - "named pipe", - "other", - "pattern-in-file", - "pattern-in-memory", - "pdb", - "pgp-private-key", - "pgp-public-key", - "process-state", - "regkey", - "regkey|value", "sha1", "sha224", "sha256", - "sha3-224", - "sha3-256", - "sha3-384", - "sha3-512", "sha384", "sha512", "sha512/224", "sha512/256", - "sigma", + "sha3-224", + "sha3-256", + "sha3-384", + "sha3-512", "ssdeep", - "stix2-pattern", + "imphash", "telfhash", - "text", + "impfuzzy", + "authentihash", "vhash", + "cdhash", + "filename", + "filename|md5", + "filename|sha1", + "filename|sha224", + "filename|sha256", + "filename|sha384", + "filename|sha512", + "filename|sha512/224", + "filename|sha512/256", + "filename|sha3-224", + "filename|sha3-256", + "filename|sha3-384", + "filename|sha3-512", + "filename|authentihash", + "filename|vhash", + "filename|ssdeep", + "filename|tlsh", + "filename|imphash", + "filename|impfuzzy", + "filename|pehash", + "regkey", + "regkey|value", + "pattern-in-file", + "pattern-in-memory", + "filename-pattern", + "pdb", + "stix2-pattern", + "yara", + "sigma", + "attachment", + "malware-sample", + "named pipe", + "mutex", + "process-state", "windows-scheduled-task", - "windows-service-displayname", "windows-service-name", - "x509-fingerprint-md5", + "windows-service-displayname", + "comment", + "text", + "hex", "x509-fingerprint-sha1", + "x509-fingerprint-md5", "x509-fingerprint-sha256", - "yara" + "other", + "cookie", + "gene", + "kusto-query", + "mime-type", + "anonymised", + "pgp-public-key", + "pgp-private-key" ], "Attribution": [ - "anonymised", - "campaign-id", - "campaign-name", - "comment", - "dns-soa-email", - "email", - "other", - "text", "threat-actor", - "whois-creation-date", + "campaign-name", + "campaign-id", + "whois-registrant-phone", "whois-registrant-email", "whois-registrant-name", "whois-registrant-org", - "whois-registrant-phone", "whois-registrar", - "x509-fingerprint-md5", + "whois-creation-date", + "comment", + "text", "x509-fingerprint-sha1", - "x509-fingerprint-sha256" + "x509-fingerprint-md5", + "x509-fingerprint-sha256", + "other", + "dns-soa-email", + "anonymised", + "email" ], "External analysis": [ - "AS", - "anonymised", - "attachment", - "bro", - "comment", - "community-id", - "cortex", - "cpe", - "domain", - "domain|ip", + "md5", + "sha1", + "sha256", + "sha3-224", + "sha3-256", + "sha3-384", + "sha3-512", "filename", - "filename-pattern", "filename|md5", "filename|sha1", "filename|sha256", @@ -140,385 +136,391 @@ "filename|sha3-256", "filename|sha3-384", "filename|sha3-512", - "github-repository", - "hassh-md5", - "hasshserver-md5", - "hostname", + "ip-src", "ip-dst", "ip-dst|port", - "ip-src", "ip-src|port", - "ja3-fingerprint-md5", - "jarm-fingerprint", - "link", "mac-address", "mac-eui-64", - "malware-sample", - "md5", - "other", - "pattern-in-file", - "pattern-in-memory", - "pattern-in-traffic", - "regkey", - "regkey|value", - "sha1", - "sha256", - "sha3-224", - "sha3-256", - "sha3-384", - "sha3-512", - "snort", - "text", + "hostname", + "domain", + "domain|ip", "url", "user-agent", + "regkey", + "regkey|value", + "AS", + "snort", + "bro", + "zeek", + "pattern-in-file", + "pattern-in-traffic", + "pattern-in-memory", + "filename-pattern", "vulnerability", + "cpe", "weakness", - "x509-fingerprint-md5", + "attachment", + "malware-sample", + "link", + "comment", + "text", "x509-fingerprint-sha1", + "x509-fingerprint-md5", "x509-fingerprint-sha256", - "zeek" + "ja3-fingerprint-md5", + "jarm-fingerprint", + "hassh-md5", + "hasshserver-md5", + "github-repository", + "other", + "cortex", + "anonymised", + "community-id" ], "Financial fraud": [ - "aba-rtn", - "anonymised", - "bank-account-nr", - "bic", - "bin", "btc", - "cc-number", - "comment", "dash", - "hex", + "xmr", "iban", - "other", - "phone-number", + "bic", + "bank-account-nr", + "aba-rtn", + "bin", + "cc-number", "prtn", + "phone-number", + "comment", "text", - "xmr" + "other", + "hex", + "anonymised" ], "Internal reference": [ - "anonymised", - "comment", - "git-commit-id", - "hex", + "text", "link", + "comment", "other", - "text" + "hex", + "anonymised", + "git-commit-id" ], "Network activity": [ - "AS", - "anonymised", - "attachment", - "bro", - "comment", - "community-id", - "cookie", + "ip-src", + "ip-dst", + "ip-dst|port", + "ip-src|port", + "port", + "hostname", "domain", "domain|ip", + "mac-address", + "mac-eui-64", "email", "email-dst", "email-src", - "email-subject", "eppn", - "favicon-mmh3", - "filename-pattern", - "hassh-md5", - "hasshserver-md5", - "hex", - "hostname", - "hostname|port", - "http-method", - "ip-dst", - "ip-dst|port", - "ip-src", - "ip-src|port", - "ja3-fingerprint-md5", - "jarm-fingerprint", - "mac-address", - "mac-eui-64", - "other", - "pattern-in-file", - "pattern-in-traffic", - "port", - "snort", - "stix2-pattern", - "text", - "uri", "url", + "uri", "user-agent", + "http-method", + "AS", + "snort", + "pattern-in-file", + "filename-pattern", + "stix2-pattern", + "pattern-in-traffic", + "attachment", + "comment", + "text", "x509-fingerprint-md5", "x509-fingerprint-sha1", "x509-fingerprint-sha256", - "zeek" + "ja3-fingerprint-md5", + "jarm-fingerprint", + "hassh-md5", + "hasshserver-md5", + "other", + "hex", + "cookie", + "hostname|port", + "bro", + "zeek", + "anonymised", + "community-id", + "email-subject", + "favicon-mmh3", + "dkim", + "dkim-signature" ], "Other": [ - "anonymised", - "boolean", "comment", + "text", + "other", + "size-in-bytes", "counter", - "cpe", "datetime", + "cpe", + "port", "float", "hex", - "other", - "pgp-private-key", - "pgp-public-key", "phone-number", - "port", - "size-in-bytes", - "text" + "boolean", + "anonymised", + "pgp-public-key", + "pgp-private-key" ], "Payload delivery": [ - "AS", - "anonymised", - "attachment", + "md5", + "sha1", + "sha224", + "sha256", + "sha384", + "sha512", + "sha512/224", + "sha512/256", + "sha3-224", + "sha3-256", + "sha3-384", + "sha3-512", + "ssdeep", + "imphash", + "telfhash", + "impfuzzy", "authentihash", + "vhash", + "pehash", + "tlsh", "cdhash", - "chrome-extension-id", - "comment", - "cpe", - "domain", - "email", - "email-attachment", - "email-body", - "email-dst", - "email-dst-display-name", - "email-header", - "email-message-id", - "email-mime-boundary", - "email-reply-to", - "email-src", - "email-src-display-name", - "email-subject", - "email-thread-index", - "email-x-mailer", "filename", - "filename-pattern", - "filename|authentihash", - "filename|impfuzzy", - "filename|imphash", "filename|md5", - "filename|pehash", "filename|sha1", "filename|sha224", "filename|sha256", - "filename|sha3-224", - "filename|sha3-256", - "filename|sha3-384", - "filename|sha3-512", "filename|sha384", "filename|sha512", "filename|sha512/224", "filename|sha512/256", + "filename|sha3-224", + "filename|sha3-256", + "filename|sha3-384", + "filename|sha3-512", + "filename|authentihash", + "filename|vhash", "filename|ssdeep", "filename|tlsh", - "filename|vhash", - "hassh-md5", - "hasshserver-md5", - "hex", - "hostname", - "hostname|port", - "impfuzzy", - "imphash", - "ip-dst", - "ip-dst|port", - "ip-src", - "ip-src|port", - "ja3-fingerprint-md5", - "jarm-fingerprint", - "link", + "filename|imphash", + "filename|impfuzzy", + "filename|pehash", "mac-address", "mac-eui-64", - "malware-sample", - "malware-type", - "md5", - "mime-type", - "mobile-application-id", - "other", + "ip-src", + "ip-dst", + "ip-dst|port", + "ip-src|port", + "hostname", + "domain", + "email", + "email-src", + "email-dst", + "email-subject", + "email-attachment", + "email-body", + "url", + "user-agent", + "AS", "pattern-in-file", "pattern-in-traffic", - "pehash", + "filename-pattern", + "stix2-pattern", + "yara", + "sigma", + "mime-type", + "attachment", + "malware-sample", + "link", + "malware-type", + "comment", + "text", + "hex", + "vulnerability", + "cpe", + "weakness", + "x509-fingerprint-sha1", + "x509-fingerprint-md5", + "x509-fingerprint-sha256", + "ja3-fingerprint-md5", + "jarm-fingerprint", + "hassh-md5", + "hasshserver-md5", + "other", + "hostname|port", + "email-dst-display-name", + "email-src-display-name", + "email-header", + "email-reply-to", + "email-x-mailer", + "email-mime-boundary", + "email-thread-index", + "email-message-id", + "mobile-application-id", + "chrome-extension-id", + "whois-registrant-email", + "anonymised" + ], + "Payload installation": [ + "md5", "sha1", "sha224", "sha256", - "sha3-224", - "sha3-256", - "sha3-384", - "sha3-512", "sha384", "sha512", "sha512/224", "sha512/256", - "sigma", + "sha3-224", + "sha3-256", + "sha3-384", + "sha3-512", "ssdeep", - "stix2-pattern", + "imphash", "telfhash", - "text", - "tlsh", - "url", - "user-agent", - "vhash", - "vulnerability", - "weakness", - "whois-registrant-email", - "x509-fingerprint-md5", - "x509-fingerprint-sha1", - "x509-fingerprint-sha256", - "yara" - ], - "Payload installation": [ - "anonymised", - "attachment", + "impfuzzy", "authentihash", + "vhash", + "pehash", + "tlsh", "cdhash", - "chrome-extension-id", - "comment", - "cpe", "filename", - "filename-pattern", - "filename|authentihash", - "filename|impfuzzy", - "filename|imphash", "filename|md5", - "filename|pehash", "filename|sha1", "filename|sha224", "filename|sha256", - "filename|sha3-224", - "filename|sha3-256", - "filename|sha3-384", - "filename|sha3-512", "filename|sha384", "filename|sha512", "filename|sha512/224", "filename|sha512/256", + "filename|sha3-224", + "filename|sha3-256", + "filename|sha3-384", + "filename|sha3-512", + "filename|authentihash", + "filename|vhash", "filename|ssdeep", "filename|tlsh", - "filename|vhash", - "hex", - "impfuzzy", - "imphash", + "filename|imphash", + "filename|impfuzzy", + "filename|pehash", + "pattern-in-file", + "pattern-in-traffic", + "pattern-in-memory", + "filename-pattern", + "stix2-pattern", + "yara", + "sigma", + "vulnerability", + "cpe", + "weakness", + "attachment", "malware-sample", "malware-type", - "md5", - "mime-type", - "mobile-application-id", - "other", - "pattern-in-file", - "pattern-in-memory", - "pattern-in-traffic", - "pehash", - "sha1", - "sha224", - "sha256", - "sha3-224", - "sha3-256", - "sha3-384", - "sha3-512", - "sha384", - "sha512", - "sha512/224", - "sha512/256", - "sigma", - "ssdeep", - "stix2-pattern", - "telfhash", + "comment", "text", - "tlsh", - "vhash", - "vulnerability", - "weakness", - "x509-fingerprint-md5", + "hex", "x509-fingerprint-sha1", + "x509-fingerprint-md5", "x509-fingerprint-sha256", - "yara" + "mobile-application-id", + "chrome-extension-id", + "other", + "mime-type", + "anonymised" ], "Payload type": [ - "anonymised", "comment", + "text", "other", - "text" + "anonymised" ], "Persistence mechanism": [ - "anonymised", - "comment", "filename", - "hex", - "other", "regkey", "regkey|value", - "text" + "comment", + "text", + "other", + "hex", + "anonymised" ], "Person": [ - "anonymised", - "comment", - "country-of-residence", - "date-of-birth", - "email", "first-name", - "frequent-flyer-number", - "gender", - "identity-card-number", - "issue-date-of-the-visa", - "last-name", "middle-name", - "nationality", - "other", - "passenger-name-record-locator-number", + "last-name", + "date-of-birth", + "place-of-birth", + "gender", + "passport-number", "passport-country", "passport-expiration", - "passport-number", + "redress-number", + "nationality", + "visa-number", + "issue-date-of-the-visa", + "primary-residence", + "country-of-residence", + "special-service-request", + "frequent-flyer-number", + "travel-details", "payment-details", - "pgp-private-key", - "pgp-public-key", - "phone-number", - "place-of-birth", + "place-port-of-original-embarkation", "place-port-of-clearance", "place-port-of-onward-foreign-destination", - "place-port-of-original-embarkation", - "primary-residence", - "redress-number", - "special-service-request", + "passenger-name-record-locator-number", + "comment", "text", - "travel-details", - "visa-number" + "other", + "phone-number", + "identity-card-number", + "anonymised", + "email", + "pgp-public-key", + "pgp-private-key" ], "Social network": [ - "anonymised", - "comment", - "email", - "email-dst", - "email-src", - "eppn", - "github-organisation", - "github-repository", "github-username", + "github-repository", + "github-organisation", "jabber-id", - "other", - "pgp-private-key", - "pgp-public-key", - "text", "twitter-id", - "whois-registrant-email" + "email", + "email-src", + "email-dst", + "eppn", + "comment", + "text", + "other", + "whois-registrant-email", + "anonymised", + "pgp-public-key", + "pgp-private-key" ], "Support Tool": [ - "anonymised", + "link", + "text", "attachment", "comment", - "hex", - "link", "other", - "text" + "hex", + "anonymised" ], "Targeting data": [ - "anonymised", - "comment", + "target-user", "target-email", - "target-external", - "target-location", "target-machine", "target-org", - "target-user" + "target-location", + "target-external", + "comment", + "anonymised" ] }, "sane_defaults": { @@ -626,6 +628,14 @@ "default_category": "Other", "to_ids": 0 }, + "dkim": { + "default_category": "Network activity", + "to_ids": 0 + }, + "dkim-signature": { + "default_category": "Network activity", + "to_ids": 0 + }, "dns-soa-email": { "default_category": "Attribution", "to_ids": 0 @@ -1256,189 +1266,191 @@ } }, "types": [ - "AS", - "aba-rtn", - "anonymised", - "attachment", - "authentihash", - "bank-account-nr", - "bic", - "bin", - "boolean", - "bro", - "btc", - "campaign-id", - "campaign-name", - "cc-number", - "cdhash", - "chrome-extension-id", - "comment", - "community-id", - "cookie", - "cortex", - "counter", - "country-of-residence", - "cpe", - "dash", - "date-of-birth", - "datetime", - "dns-soa-email", + "md5", + "sha1", + "sha256", + "filename", + "pdb", + "filename|md5", + "filename|sha1", + "filename|sha256", + "ip-src", + "ip-dst", + "hostname", "domain", "domain|ip", "email", + "email-src", + "eppn", + "email-dst", + "email-subject", "email-attachment", "email-body", - "email-dst", - "email-dst-display-name", - "email-header", - "email-message-id", - "email-mime-boundary", - "email-reply-to", - "email-src", - "email-src-display-name", - "email-subject", - "email-thread-index", - "email-x-mailer", - "eppn", - "favicon-mmh3", - "filename", - "filename|authentihash", - "filename|impfuzzy", - "filename|imphash", - "filename|md5", - "filename|pehash", - "filename|sha1", - "filename|sha224", - "filename|sha256", - "filename|sha3-224", - "filename|sha3-256", - "filename|sha3-384", - "filename|sha3-512", - "filename|sha384", - "filename|sha512", - "filename|sha512/224", - "filename|sha512/256", - "filename|ssdeep", - "filename|tlsh", - "filename|vhash", - "first-name", "float", - "frequent-flyer-number", - "gender", - "gene", "git-commit-id", - "github-organisation", - "github-repository", - "github-username", + "url", + "http-method", + "user-agent", + "ja3-fingerprint-md5", + "jarm-fingerprint", + "favicon-mmh3", "hassh-md5", "hasshserver-md5", - "hex", - "hostname", - "hostname|port", - "http-method", - "iban", - "identity-card-number", - "impfuzzy", - "imphash", - "ip-dst", - "ip-dst|port", - "ip-src", - "ip-src|port", - "issue-date-of-the-visa", - "ja3-fingerprint-md5", - "jabber-id", - "jarm-fingerprint", - "kusto-query", - "last-name", - "link", - "mac-address", - "mac-eui-64", - "malware-sample", - "malware-type", - "md5", - "middle-name", - "mime-type", - "mobile-application-id", - "mutex", - "named pipe", - "nationality", - "other", - "passenger-name-record-locator-number", - "passport-country", - "passport-expiration", - "passport-number", - "pattern-filename", - "pattern-in-file", - "pattern-in-memory", - "pattern-in-traffic", - "payment-details", - "pdb", - "pehash", - "pgp-private-key", - "pgp-public-key", - "phone-number", - "place-of-birth", - "place-port-of-clearance", - "place-port-of-onward-foreign-destination", - "place-port-of-original-embarkation", - "port", - "primary-residence", - "process-state", - "prtn", - "redress-number", "regkey", "regkey|value", - "sha1", + "AS", + "snort", + "bro", + "zeek", + "community-id", + "pattern-in-file", + "pattern-in-traffic", + "pattern-in-memory", + "pattern-filename", + "pgp-public-key", + "pgp-private-key", + "yara", + "stix2-pattern", + "sigma", + "gene", + "kusto-query", + "mime-type", + "identity-card-number", + "cookie", + "vulnerability", + "cpe", + "weakness", + "attachment", + "malware-sample", + "link", + "comment", + "text", + "hex", + "other", + "named pipe", + "mutex", + "process-state", + "target-user", + "target-email", + "target-machine", + "target-org", + "target-location", + "target-external", + "btc", + "dash", + "xmr", + "iban", + "bic", + "bank-account-nr", + "aba-rtn", + "bin", + "cc-number", + "prtn", + "phone-number", + "threat-actor", + "campaign-name", + "campaign-id", + "malware-type", + "uri", + "authentihash", + "vhash", + "ssdeep", + "imphash", + "telfhash", + "pehash", + "impfuzzy", "sha224", - "sha256", - "sha3-224", - "sha3-256", - "sha3-384", - "sha3-512", "sha384", "sha512", "sha512/224", "sha512/256", - "sigma", - "size-in-bytes", - "snort", - "special-service-request", - "ssdeep", - "stix2-pattern", - "target-email", - "target-external", - "target-location", - "target-machine", - "target-org", - "target-user", - "telfhash", - "text", - "threat-actor", + "sha3-224", + "sha3-256", + "sha3-384", + "sha3-512", "tlsh", - "travel-details", - "twitter-id", - "uri", - "url", - "user-agent", - "vhash", - "visa-number", - "vulnerability", - "weakness", - "whois-creation-date", + "cdhash", + "filename|authentihash", + "filename|vhash", + "filename|ssdeep", + "filename|imphash", + "filename|impfuzzy", + "filename|pehash", + "filename|sha224", + "filename|sha384", + "filename|sha512", + "filename|sha512/224", + "filename|sha512/256", + "filename|sha3-224", + "filename|sha3-256", + "filename|sha3-384", + "filename|sha3-512", + "filename|tlsh", + "windows-scheduled-task", + "windows-service-name", + "windows-service-displayname", "whois-registrant-email", + "whois-registrant-phone", "whois-registrant-name", "whois-registrant-org", - "whois-registrant-phone", "whois-registrar", - "windows-scheduled-task", - "windows-service-displayname", - "windows-service-name", - "x509-fingerprint-md5", + "whois-creation-date", "x509-fingerprint-sha1", + "x509-fingerprint-md5", "x509-fingerprint-sha256", - "xmr", - "yara", - "zeek" + "dns-soa-email", + "size-in-bytes", + "counter", + "datetime", + "port", + "ip-dst|port", + "ip-src|port", + "hostname|port", + "mac-address", + "mac-eui-64", + "email-dst-display-name", + "email-src-display-name", + "email-header", + "email-reply-to", + "email-x-mailer", + "email-mime-boundary", + "email-thread-index", + "email-message-id", + "github-username", + "github-repository", + "github-organisation", + "jabber-id", + "twitter-id", + "dkim", + "dkim-signature", + "first-name", + "middle-name", + "last-name", + "date-of-birth", + "place-of-birth", + "gender", + "passport-number", + "passport-country", + "passport-expiration", + "redress-number", + "nationality", + "visa-number", + "issue-date-of-the-visa", + "primary-residence", + "country-of-residence", + "special-service-request", + "frequent-flyer-number", + "travel-details", + "payment-details", + "place-port-of-original-embarkation", + "place-port-of-clearance", + "place-port-of-onward-foreign-destination", + "passenger-name-record-locator-number", + "mobile-application-id", + "chrome-extension-id", + "cortex", + "boolean", + "anonymised" ] } } From 4404876a916c804fd3827595d34493e487ea160d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 26 Feb 2021 13:21:22 +0100 Subject: [PATCH 23/28] chg: Bump deps --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 8287425..c74b124 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,7 +51,7 @@ RTFDE = {version = "^0.0.2", optional = true} oletools = {version = "^0.56", optional = true} python-magic = {version = "^0.4.18", optional = true} pydeep = {version = "^0.4", optional = true} -lief = {version = "^0.11.0", optional = true} +lief = {version = "^0.11.2", optional = true} beautifulsoup4 = {version = "^4.9.3", optional = true} validators = {version = "^0.18.1", optional = true} sphinx-autodoc-typehints = {version = "^1.11.1", optional = true} From 16b2ad9e4b37eb4833bcecb6854139e91273c383 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 26 Feb 2021 16:09:43 +0100 Subject: [PATCH 24/28] chg: Bump poetry file --- poetry.lock | 98 ++++++++++++++++++++++++++--------------------------- 1 file changed, 49 insertions(+), 49 deletions(-) diff --git a/poetry.lock b/poetry.lock index e2209a5..d32837e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -214,7 +214,7 @@ yaml = ["PyYAML (>=3.10)"] [[package]] name = "cryptography" -version = "3.4.5" +version = "3.4.6" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." category = "main" optional = true @@ -303,7 +303,7 @@ python-versions = ">=2.7" [[package]] name = "extract-msg" -version = "0.28.1" +version = "0.28.5" description = "Extracts emails and attachments saved in Microsoft Outlook's .msg files" category = "main" optional = true @@ -363,7 +363,7 @@ test = ["mock (>=1.3.0)"] [[package]] name = "importlib-metadata" -version = "3.4.0" +version = "3.7.0" description = "Read metadata from Python packages" category = "main" optional = false @@ -379,7 +379,7 @@ testing = ["pytest (>=3.5,!=3.7.3)", "pytest-checkdocs (>=1.2.3)", "pytest-flake [[package]] name = "ipykernel" -version = "5.4.3" +version = "5.5.0" description = "IPython Kernel for Jupyter" category = "dev" optional = false @@ -572,7 +572,7 @@ test = ["pytest", "requests"] [[package]] name = "lark-parser" -version = "0.11.1" +version = "0.11.2" description = "a modern parsing library" category = "main" optional = true @@ -584,7 +584,7 @@ regex = ["regex"] [[package]] name = "lief" -version = "0.11.0" +version = "0.11.2" description = "Library to instrument executable formats" category = "main" optional = true @@ -971,7 +971,7 @@ six = ">=1.5" [[package]] name = "python-magic" -version = "0.4.20" +version = "0.4.22" description = "File type identification using libmagic" category = "main" optional = true @@ -1121,7 +1121,7 @@ python-versions = ">=3.6" [[package]] name = "sphinx" -version = "3.5.0" +version = "3.5.1" description = "Python documentation generator" category = "main" optional = true @@ -1146,9 +1146,9 @@ sphinxcontrib-qthelp = "*" sphinxcontrib-serializinghtml = "*" [package.extras] -lint = ["docutils-stubs", "flake8 (>=3.5.0)", "isort", "mypy (>=0.800)"] -test = ["pytest", "pytest-cov", "html5lib", "cython", "typed-ast"] docs = ["sphinxcontrib-websupport"] +lint = ["flake8 (>=3.5.0)", "isort", "mypy (>=0.800)", "docutils-stubs"] +test = ["pytest", "pytest-cov", "html5lib", "cython", "typed-ast"] [[package]] name = "sphinx-autodoc-typehints" @@ -1399,7 +1399,7 @@ virustotal = ["validators"] [metadata] lock-version = "1.1" python-versions = "^3.6" -content-hash = "9f7289c3410f9000dbf655580c0f32f8685b72e799f2d0030b029685b6e1db2a" +content-hash = "c8714d721bedbe775c304c614f6ce8fce15fb44340f9d99364b4843adcbbf60f" [metadata.files] alabaster = [ @@ -1613,13 +1613,13 @@ coveralls = [ {file = "coveralls-3.0.0.tar.gz", hash = "sha256:5399c0565ab822a70a477f7031f6c88a9dd196b3de2877b3facb43b51bd13434"}, ] cryptography = [ - {file = "cryptography-3.4.5-cp36-abi3-macosx_10_10_x86_64.whl", hash = "sha256:18d6f3ac1da14a01c95f4590ee58e96aa6628d231ce738e9eca330b9997127b6"}, - {file = "cryptography-3.4.5-cp36-abi3-manylinux2010_x86_64.whl", hash = "sha256:c8dc9859c5a046e1ca22da360dfd609c09064a4f974881cb5cba977c581088ec"}, - {file = "cryptography-3.4.5-cp36-abi3-manylinux2014_aarch64.whl", hash = "sha256:0bf49d5b38e4f3745a0eab0597fa97720dd49b30d65f534b49a82e303f149deb"}, - {file = "cryptography-3.4.5-cp36-abi3-manylinux2014_x86_64.whl", hash = "sha256:9c6f7552d4f2130542d488b9d9e5b1546204b5d1aa90c823d50cce8eed421363"}, - {file = "cryptography-3.4.5-cp36-abi3-win32.whl", hash = "sha256:b0873ac0c0e6bc6882cd285930cc382ec4e78786be71bdc113c06246eea61294"}, - {file = "cryptography-3.4.5-cp36-abi3-win_amd64.whl", hash = "sha256:8b3b79af57e12aabbc3db81e563eaa07870293a1ffdcc891d107035ce9a0dbff"}, - {file = "cryptography-3.4.5.tar.gz", hash = "sha256:4f6761a82b51fe02cda8f45af1c2f698a10f50003dc9c2572d8a49eda2e6d35b"}, + {file = "cryptography-3.4.6-cp36-abi3-macosx_10_10_x86_64.whl", hash = "sha256:57ad77d32917bc55299b16d3b996ffa42a1c73c6cfa829b14043c561288d2799"}, + {file = "cryptography-3.4.6-cp36-abi3-manylinux2010_x86_64.whl", hash = "sha256:93cfe5b7ff006de13e1e89830810ecbd014791b042cbe5eec253be11ac2b28f3"}, + {file = "cryptography-3.4.6-cp36-abi3-manylinux2014_aarch64.whl", hash = "sha256:5ecf2bcb34d17415e89b546dbb44e73080f747e504273e4d4987630493cded1b"}, + {file = "cryptography-3.4.6-cp36-abi3-manylinux2014_x86_64.whl", hash = "sha256:fec7fb46b10da10d9e1d078d1ff8ed9e05ae14f431fdbd11145edd0550b9a964"}, + {file = "cryptography-3.4.6-cp36-abi3-win32.whl", hash = "sha256:df186fcbf86dc1ce56305becb8434e4b6b7504bc724b71ad7a3239e0c9d14ef2"}, + {file = "cryptography-3.4.6-cp36-abi3-win_amd64.whl", hash = "sha256:66b57a9ca4b3221d51b237094b0303843b914b7d5afd4349970bb26518e350b0"}, + {file = "cryptography-3.4.6.tar.gz", hash = "sha256:2d32223e5b0ee02943f32b19245b61a62db83a882f0e76cc564e1cec60d48f87"}, ] decorator = [ {file = "decorator-4.4.2-py2.py3-none-any.whl", hash = "sha256:41fa54c2a0cc4ba648be4fd43cff00aedf5b9465c9bf18d64325bc225f08f760"}, @@ -1652,8 +1652,8 @@ entrypoints = [ {file = "entrypoints-0.3.tar.gz", hash = "sha256:c70dd71abe5a8c85e55e12c19bd91ccfeec11a6e99044204511f9ed547d48451"}, ] extract-msg = [ - {file = "extract_msg-0.28.1-py2.py3-none-any.whl", hash = "sha256:7ce5761911b2caa9f07042efdecfcc9cf4afe524c3efbfd0aa5fa277faa1cc53"}, - {file = "extract_msg-0.28.1.tar.gz", hash = "sha256:7300a50bfa91405a381826f8e22f39458c7322fea1cd660ef699c4157a58be4b"}, + {file = "extract_msg-0.28.5-py2.py3-none-any.whl", hash = "sha256:4c7d7da5fa50cbeb406226acef572e9c308ffcae6aec35607abf6bfb63ad0448"}, + {file = "extract_msg-0.28.5.tar.gz", hash = "sha256:044d1fbafd706e09aa9e0d8de351791c5517a171bc09e2391728508013793f28"}, ] flake8 = [ {file = "flake8-3.8.4-py2.py3-none-any.whl", hash = "sha256:749dbbd6bfd0cf1318af27bf97a14e28e5ff548ef8e5b1566ccfb25a11e7c839"}, @@ -1672,12 +1672,12 @@ imapclient = [ {file = "IMAPClient-2.1.0.zip", hash = "sha256:60ba79758cc9f13ec910d7a3df9acaaf2bb6c458720d9a02ec33a41352fd1b99"}, ] importlib-metadata = [ - {file = "importlib_metadata-3.4.0-py3-none-any.whl", hash = "sha256:ace61d5fc652dc280e7b6b4ff732a9c2d40db2c0f92bc6cb74e07b73d53a1771"}, - {file = "importlib_metadata-3.4.0.tar.gz", hash = "sha256:fa5daa4477a7414ae34e95942e4dd07f62adf589143c875c133c1e53c4eff38d"}, + {file = "importlib_metadata-3.7.0-py3-none-any.whl", hash = "sha256:c6af5dbf1126cd959c4a8d8efd61d4d3c83bddb0459a17e554284a077574b614"}, + {file = "importlib_metadata-3.7.0.tar.gz", hash = "sha256:24499ffde1b80be08284100393955842be4a59c7c16bbf2738aad0e464a8e0aa"}, ] ipykernel = [ - {file = "ipykernel-5.4.3-py3-none-any.whl", hash = "sha256:4ed205700001a83b5832d4821c46a5733f1bf4b1c55744314ae3c756be6b6095"}, - {file = "ipykernel-5.4.3.tar.gz", hash = "sha256:697103d218e9a8828025af7986e033c89e0b36e2b6eb84a5bda4739b9a27f3cb"}, + {file = "ipykernel-5.5.0-py3-none-any.whl", hash = "sha256:efd07253b54d84d26e0878d268c8c3a41582a18750da633c2febfd2ece0d467d"}, + {file = "ipykernel-5.5.0.tar.gz", hash = "sha256:98321abefdf0505fb3dc7601f60fc4087364d394bd8fad53107eb1adee9ff475"}, ] ipython = [ {file = "ipython-7.16.1-py3-none-any.whl", hash = "sha256:2dbcc8c27ca7d3cfe4fcdff7f45b27f9a8d3edfa70ff8024a71c7a8eb5f09d64"}, @@ -1724,29 +1724,29 @@ jupyterlab-server = [ {file = "jupyterlab_server-1.2.0.tar.gz", hash = "sha256:5431d9dde96659364b7cc877693d5d21e7b80cea7ae3959ecc2b87518e5f5d8c"}, ] lark-parser = [ - {file = "lark-parser-0.11.1.tar.gz", hash = "sha256:20bdefdf1b6e9bcb38165ea5cc4f27921a99c6f4c35264a3a953fd60335f1f8c"}, - {file = "lark_parser-0.11.1-py2.py3-none-any.whl", hash = "sha256:8b747e1f544dcc2789e3feaddd2a50c6a73bed69d62e9c69760c1e1f7d23495f"}, + {file = "lark-parser-0.11.2.tar.gz", hash = "sha256:ef610461ebf2b243502f337d9d49879e39f9add846a4749e88c8dcdc1378bb6b"}, ] lief = [ - {file = "lief-0.11.0-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:8774076dfbcf9b6906be4d9243de4a78fc09d317292251072d460ed1e0eeb96e"}, - {file = "lief-0.11.0-cp36-cp36m-win32.whl", hash = "sha256:348ee294567826cad638b7e6cf2ede4ffe03524cd5b6038896f78e5b006d6295"}, - {file = "lief-0.11.0-cp36-cp36m-win_amd64.whl", hash = "sha256:77ba7dd0d48933c0b26c980bda1ab4a7ad3c7031880181fab0b94caad3bc1a4d"}, - {file = "lief-0.11.0-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:93d79a47db9977e6471b21d5efd4e7af4c29c76261d6583141fcf10f6ccd0eba"}, - {file = "lief-0.11.0-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:d95cf1224c7b311b8d25dbd4de627d28717266e62b9721f1dc4c8427f929a60f"}, - {file = "lief-0.11.0-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:0cd2665ff28937755c8acedc2e3fb9de5ba752353e19b51b89297b8be3f63ccb"}, - {file = "lief-0.11.0-cp37-cp37m-win32.whl", hash = "sha256:b0a55424b3ffa08d16bf8ee6e5e9474b0a4b392ca4d94545c16e47e6366e41d4"}, - {file = "lief-0.11.0-cp37-cp37m-win_amd64.whl", hash = "sha256:96d2a8d2310c7accaeb5c6679833c0cd8f0fb6d8c682a5df59d4d868c8881661"}, - {file = "lief-0.11.0-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:e8c66834a0ee9ed1899db165c09ca04aba8dee574de1ccc866d82fbf0c059bb8"}, - {file = "lief-0.11.0-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:2ee8f9787ea92109f19e5e4d22773042184ac524a3f11399ea5e13d974ac1f05"}, - {file = "lief-0.11.0-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:1fb570712fa17ee153aca263ab1f1ec763772ccb46992e415b3dc1c0248466bc"}, - {file = "lief-0.11.0-cp38-cp38-win32.whl", hash = "sha256:f9b00c396c9f45c5f4cf37c034428ad525d6d7c7d38fc6c49ddc4b558201151b"}, - {file = "lief-0.11.0-cp38-cp38-win_amd64.whl", hash = "sha256:1a110bd5db41b4fde1a61094658b0366352ed4c0a00b55bde821046a59c2f913"}, - {file = "lief-0.11.0-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:afe4d15b519dd7d97732e85b7a2b11154c97a40ce517e1044b5cd5f80074ce36"}, - {file = "lief-0.11.0-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:e6ff05e6ebcb9bea8833fcb558d4db3bc2a78031c4792fcac9f9612fa78258b3"}, - {file = "lief-0.11.0-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:f31fde4e7174b4bc9b67ff22fd93fa15fc3faa1592ac669f3addc95d9e66168e"}, - {file = "lief-0.11.0-cp39-cp39-win32.whl", hash = "sha256:9f2bd417090df21548af3a0216f3a02056291348c0826a5ff78e3f3046283978"}, - {file = "lief-0.11.0-cp39-cp39-win_amd64.whl", hash = "sha256:9837166402248a4ef29018d498c4eccc11cbc92ee4083da046fa747d3863b43d"}, - {file = "lief-0.11.0.zip", hash = "sha256:ccf977099153eaefa351e72e84dfa829334699521ef00434b50647d80de2cc56"}, + {file = "lief-0.11.2-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:6be9b6bf3cbae7b6d68753ed8fcb158153500e5e28a316676bd5d4c54858be98"}, + {file = "lief-0.11.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:463d229f0ca65cfc8f2bc2e412c0197b6db9c46bbe3ca61ca77ce59d1a4cbd10"}, + {file = "lief-0.11.2-cp36-cp36m-win32.whl", hash = "sha256:e940f37ed72c784ef08cfb0e4c3ae83d48cfcf43d56724a0d38805cc1bd65919"}, + {file = "lief-0.11.2-cp36-cp36m-win_amd64.whl", hash = "sha256:7dee7317e1cfc488eda703c58a6271be177fe89d1dde4c8ffa31cfd275bbfe24"}, + {file = "lief-0.11.2-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:370ab0fd40a36564acec4c612ad2a4b4ac39c3f496635fee060f317eea9dbdda"}, + {file = "lief-0.11.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:4f12b7d5901e4822e3b7019dc54e2f33e5c100b0c28c775014978cb5c30a4467"}, + {file = "lief-0.11.2-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:59333b0e0577ba5a231e97c37bfa3259dde935592dbead3bb9af32c878f9a678"}, + {file = "lief-0.11.2-cp37-cp37m-win32.whl", hash = "sha256:27392d5d90ad3539dc065b897eeecd49c2fffa540a117ef0af4ab3c08a15b009"}, + {file = "lief-0.11.2-cp37-cp37m-win_amd64.whl", hash = "sha256:863da532acb72e1c0a264e44d15ab6c809a4e7156f9dc9a843a78d501aa78486"}, + {file = "lief-0.11.2-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:0f87fc6bf31af8595b7b3acfd19f67b431094ccd0518ddea4ad28f6b07684020"}, + {file = "lief-0.11.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:e499693ef29cf2d777db5dae35ef1a46b3dc4a93145a2604e9799aa6e3f58fce"}, + {file = "lief-0.11.2-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:d18fc35111bffb558572d4fa0de02d37d78b0ceebca461d5ce82f59b507d509a"}, + {file = "lief-0.11.2-cp38-cp38-win32.whl", hash = "sha256:e009ccad6cd5485344ee7fdc04e106b1c835cf72c56a35591012b03ebe5114f6"}, + {file = "lief-0.11.2-cp38-cp38-win_amd64.whl", hash = "sha256:b7b8dda460b21d85497dd30f1b1271dd44ac5965843642b810c3b6aeb8da52fa"}, + {file = "lief-0.11.2-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:347acd1b143092858d792d92dd48c56984bffa6fff246f668c0916dfc3e61cb5"}, + {file = "lief-0.11.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:aa797dac08233e675d46db6c680b583caf6d22e3321b2cd312dd8df5fa959cd9"}, + {file = "lief-0.11.2-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:aab5b75afd07b176c5a0979e09bf2fca506524c78d11451f68737150ffa9fec3"}, + {file = "lief-0.11.2-cp39-cp39-win32.whl", hash = "sha256:4fa9803eafbfa6fad31b101b1e98d9bc76f467c0a93cada114b977d6d687a234"}, + {file = "lief-0.11.2-cp39-cp39-win_amd64.whl", hash = "sha256:6f2e67ccc39ce4861168fc37c236b04dfcdd532582bde370095db2581b2c6805"}, + {file = "lief-0.11.2.zip", hash = "sha256:c7a7815ba19a8afa0c27ce584c60fc3c516badb0923d3c04a0bd13f8318b5370"}, ] markupsafe = [ {file = "MarkupSafe-1.1.1-cp27-cp27m-macosx_10_6_intel.whl", hash = "sha256:09027a7803a62ca78792ad89403b1b7a73a01c8cb65909cd876f7fcebd79b161"}, @@ -1944,8 +1944,8 @@ python-dateutil = [ {file = "python_dateutil-2.8.1-py2.py3-none-any.whl", hash = "sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a"}, ] python-magic = [ - {file = "python-magic-0.4.20.tar.gz", hash = "sha256:0cc52ccad086c377b9194014e3dbf98d94b194344630172510a6a3e716b47801"}, - {file = "python_magic-0.4.20-py2.py3-none-any.whl", hash = "sha256:33ce94d9395aa269a9c5fac10ae124a5fb328ebe248f36efc5a43922edee662e"}, + {file = "python-magic-0.4.22.tar.gz", hash = "sha256:ca884349f2c92ce830e3f498c5b7c7051fe2942c3ee4332f65213b8ebff15a62"}, + {file = "python_magic-0.4.22-py2.py3-none-any.whl", hash = "sha256:8551e804c09a3398790bd9e392acb26554ae2609f29c72abb0b9dee9a5571eae"}, ] pytz = [ {file = "pytz-2021.1-py2.py3-none-any.whl", hash = "sha256:eb10ce3e7736052ed3623d49975ce333bcd712c7bb19a58b9e2089d4057d0798"}, @@ -2083,8 +2083,8 @@ soupsieve = [ {file = "soupsieve-2.2.tar.gz", hash = "sha256:407fa1e8eb3458d1b5614df51d9651a1180ea5fedf07feb46e45d7e25e6d6cdd"}, ] sphinx = [ - {file = "Sphinx-3.5.0-py3-none-any.whl", hash = "sha256:68da66ca3d6b35b22bea5c53d938d5f8988663dca042f0a46429a1eba1010051"}, - {file = "Sphinx-3.5.0.tar.gz", hash = "sha256:deb468efb3abaa70d790add4147d18782d86fdeacf648d6e8afb7a99807f1546"}, + {file = "Sphinx-3.5.1-py3-none-any.whl", hash = "sha256:e90161222e4d80ce5fc811ace7c6787a226b4f5951545f7f42acf97277bfc35c"}, + {file = "Sphinx-3.5.1.tar.gz", hash = "sha256:11d521e787d9372c289472513d807277caafb1684b33eb4f08f7574c405893a9"}, ] sphinx-autodoc-typehints = [ {file = "sphinx-autodoc-typehints-1.11.1.tar.gz", hash = "sha256:244ba6d3e2fdb854622f643c7763d6f95b6886eba24bec28e86edf205e4ddb20"}, From 28fed5c7788ffe38e12ba962303db93b0a5d2120 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 26 Feb 2021 17:13:16 +0100 Subject: [PATCH 25/28] fix: support text search again Fix #705 --- pymisp/api.py | 2 +- tests/testlive_comprehensive.py | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/pymisp/api.py b/pymisp/api.py index 7bc3329..fc2952f 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -2394,7 +2394,7 @@ class PyMISP: return self._csv_to_dict(normalized_response_text) # type: ignore else: return normalized_response_text - elif return_format == 'stix-xml': + elif return_format in ['stix-xml', 'text']: return self._check_response(response) normalized_response = self._check_json_response(response) diff --git a/tests/testlive_comprehensive.py b/tests/testlive_comprehensive.py index 92e97b5..366bfca 100644 --- a/tests/testlive_comprehensive.py +++ b/tests/testlive_comprehensive.py @@ -1152,6 +1152,19 @@ class TestComprehensive(unittest.TestCase): self.admin_misp_connector.delete_event(first) self.admin_misp_connector.delete_event(second) + def test_search_text(self): + first = self.create_simple_event() + first.add_attribute('ip-src', '8.8.8.8') + first.publish() + try: + first = self.user_misp_connector.add_event(first) + self.admin_misp_connector.publish(first) + text = self.user_misp_connector.search(return_format='text', eventid=first.id) + self.assertEqual('8.8.8.8', text.strip()) + finally: + # Delete event + self.admin_misp_connector.delete_event(first) + def test_search_stix(self): first = self.create_simple_event() first.add_attribute('ip-src', '8.8.8.8') From 2e05a1b24fa469111427bff85d0f562406589387 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 26 Feb 2021 17:55:08 +0100 Subject: [PATCH 26/28] new: soft delete object in MISPEvent Fix #706 --- pymisp/mispevent.py | 15 ++++++++++++++- tests/testlive_comprehensive.py | 8 ++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index bc0253d..20e9856 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -752,7 +752,7 @@ class MISPObject(AbstractMISP): pass def delete(self): - """Mark the attribute as deleted (soft delete)""" + """Mark the object as deleted (soft delete)""" self.deleted = True @property @@ -1888,6 +1888,19 @@ class MISPEvent(AbstractMISP): self.edited = True return misp_obj + def delete_object(self, object_id: str): + """Delete an object + + :param object_id: ID or UUID + """ + for o in self.objects: + if ((hasattr(o, 'id') and o.id == object_id) + or (hasattr(o, 'uuid') and o.uuid == object_id)): + o.delete() + break + else: + raise PyMISPError('No object with UUID/ID {} found.'.format(object_id)) + def run_expansions(self): for index, attribute in enumerate(self.attributes): if 'expand' not in attribute: diff --git a/tests/testlive_comprehensive.py b/tests/testlive_comprehensive.py index 366bfca..c35e47b 100644 --- a/tests/testlive_comprehensive.py +++ b/tests/testlive_comprehensive.py @@ -1255,6 +1255,14 @@ class TestComprehensive(unittest.TestCase): response = self.admin_misp_connector.delete_tag(t) self.assertEqual(response['message'], 'Tag deleted.') + # Test soft delete object + second.delete_object(ip_dom.uuid) + self.assertTrue(second.objects[-1].deleted) + second = self.user_misp_connector.update_event(second) + self.assertFalse(second.objects) + second = self.user_misp_connector.get_event(second, deleted=True) + self.assertTrue(second.objects[-1].deleted) + # Test delete object r = self.user_misp_connector.delete_object(second.objects[0]) self.assertEqual(r['message'], 'Object deleted', r) From 1533da3558a6245fa0bb3876c5a6bebe40587697 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 26 Feb 2021 17:57:34 +0100 Subject: [PATCH 27/28] chg: Improve Pydoc on search method's timestamp parameter Fix #708 --- pymisp/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymisp/api.py b/pymisp/api.py index fc2952f..2dc3954 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -2265,7 +2265,7 @@ class PyMISP: :param metadata: Only the metadata (event, tags, relations) is returned, attributes and proposals are omitted. :param uuid: Restrict the results by uuid. :param publish_timestamp: Restrict the results by the last publish timestamp (newer than). - :param timestamp: Restrict the results by the timestamp (last edit). Any event with a timestamp newer than the given timestamp will be returned. In case you are dealing with /attributes as scope, the attribute's timestamp will be used for the lookup. + :param timestamp: Restrict the results by the timestamp (last edit). Any event with a timestamp newer than the given timestamp will be returned. In case you are dealing with /attributes as scope, the attribute's timestamp will be used for the lookup. The input can be a timestamp or a short-hand time description (7d or 24h for example). You can also pass a list with two values to set a time range (for example ["14d", "7d"]). :param published: Set whether published or unpublished events should be returned. Do not set the parameter if you want both. :param enforce_warninglist: Remove any attributes from the result that would cause a hit on a warninglist entry. :param to_ids: By default all attributes are returned that match the other filter parameters, regardless of their to_ids setting. To restrict the returned data set to to_ids only attributes set this parameter to 1. 0 for the ones with to_ids set to False. From 81373894521aacb89737b4680a6cce56f208ecc0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 2 Mar 2021 11:49:31 +0100 Subject: [PATCH 28/28] chg: Bump tests for galaxy cluster --- pymisp/mispevent.py | 1 - tests/testlive_comprehensive.py | 5 +++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 20e9856..37d005f 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -1242,7 +1242,6 @@ class MISPGalaxyCluster(AbstractMISP): def from_dict(self, **kwargs): if 'GalaxyCluster' in kwargs: kwargs = kwargs['GalaxyCluster'] - self.default = kwargs.pop('default', False) # If the default field is set, we shouldn't have distribution or sharing group ID set if self.default: diff --git a/tests/testlive_comprehensive.py b/tests/testlive_comprehensive.py index c35e47b..6f0cf7f 100644 --- a/tests/testlive_comprehensive.py +++ b/tests/testlive_comprehensive.py @@ -2739,7 +2739,7 @@ class TestComprehensive(unittest.TestCase): self.assertEqual(element.value, "Test2") # The cluster should have the old meta as a prop - self.assertEqual(new_galaxy_cluster.meta, {'synonyms': ['Test2']}) + self.assertEqual(new_galaxy_cluster.elements_meta, {'synonyms': ['Test2']}) # The cluster element should be updatable element.value = "Test3" @@ -2791,7 +2791,8 @@ class TestComprehensive(unittest.TestCase): resp = self.admin_misp_connector.get_galaxy_cluster(new_galaxy_cluster) self.assertTrue("errors" in resp) finally: - self.admin_misp_connector.delete_galaxy_cluster(new_galaxy_cluster) + self.admin_misp_connector.delete_galaxy_cluster_relation(relation) + self.admin_misp_connector.delete_galaxy_cluster(new_galaxy_cluster, hard=True) self.admin_misp_connector.toggle_global_pythonify() def test_event_galaxy(self):