From 73b56a61da40db9b1d26c3bf42ef5ef15a32ecb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 11 Sep 2020 11:09:14 +0200 Subject: [PATCH 1/4] fix: few outdated calls in the tutorial --- docs/tutorial/FullOverview.ipynb | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/tutorial/FullOverview.ipynb b/docs/tutorial/FullOverview.ipynb index fae177b..ad3323f 100644 --- a/docs/tutorial/FullOverview.ipynb +++ b/docs/tutorial/FullOverview.ipynb @@ -1012,7 +1012,7 @@ "existing_event.add_object(mispObject)\n", "print(existing_event.to_json())\n", "\n", - "res = misp.update(existing_event)\n", + "res = misp.update_event(existing_event)\n", "existing_event = MISPEvent()\n", "existing_event.load(res)\n", "print(existing_event.to_json())" @@ -1071,7 +1071,7 @@ "event_obj.threat_level_id = 1\n", "event_obj.analysis = 1\n", "event_obj.info = \"Event from notebook 2\"\n", - "event = misp.add_event(event_obj)\n", + "event = misp.add_event(event_obj, pythonify=True)\n", "event_id = event.id\n", "print(\"Event id: %s\" % event_id)" ] @@ -1171,7 +1171,7 @@ "attribute.category = category\n", "attribute.to_ids = to_ids\n", "\n", - "attribute_to_change = misp.add_attribute(event_id, attribute)\n", + "attribute_to_change = misp.add_attribute(event_id, attribute, pythonify=True)\n", "print(attribute_to_change.id, attribute_to_change)" ] }, @@ -1386,7 +1386,7 @@ "metadata": {}, "outputs": [], "source": [ - "misp.get_sharing_groups()" + "misp.sharing_groups()" ] }, { @@ -1402,7 +1402,7 @@ "metadata": {}, "outputs": [], "source": [ - "misp.get_users_list()" + "misp.users()" ] }, { @@ -1427,7 +1427,7 @@ "metadata": {}, "outputs": [], "source": [ - "misp.get_organisations_list()" + "misp.organisations()" ] }, { @@ -1443,7 +1443,7 @@ "metadata": {}, "outputs": [], "source": [ - "misp.get_roles_list()" + "misp.roles()" ] }, { @@ -1459,7 +1459,7 @@ "metadata": {}, "outputs": [], "source": [ - "misp.get_feeds_list()" + "misp.feeds(pythonify=True)" ] }, { @@ -1495,7 +1495,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.5" + "version": "3.8.2" } }, "nbformat": 4, From 50e5f156bda88e4f5c37238b8852537f51863b45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 15 Sep 2020 12:31:22 +0200 Subject: [PATCH 2/4] chg: Improve error message, add comments, rename whitelist->allowedlist --- pymisp/api.py | 23 ++++++++++++++++++++++- pymisp/mispevent.py | 6 +++++- tests/testlive_comprehensive.py | 10 +++++----- 3 files changed, 32 insertions(+), 7 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index 11302c4..19e5579 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -35,6 +35,7 @@ logger = logging.getLogger('pymisp') def get_uuid_or_id_from_abstract_misp(obj: Union[AbstractMISP, int, str, UUID]) -> Union[str, int]: + """Extract the relevant ID accordingly to the given type passed as parameter""" if isinstance(obj, UUID): return str(obj) if isinstance(obj, (int, str)): @@ -188,6 +189,7 @@ class PyMISP: @property def pymisp_version_master(self) -> Dict: + """PyMISP version as defined in the main repository""" return self.pymisp_version_main @property @@ -215,36 +217,44 @@ class PyMISP: return {'error': 'Impossible to retrieve the version of the master branch.'} def update_misp(self) -> Dict: + """Trigger a server update""" response = self._prepare_request('POST', 'servers/update') return self._check_json_response(response) def set_server_setting(self, setting: str, value: Union[str, int, bool], force: bool = False) -> Dict: + """Set a setting on the MISP instance""" data = {'value': value, 'force': force} response = self._prepare_request('POST', f'servers/serverSettingsEdit/{setting}', data=data) return self._check_json_response(response) def get_server_setting(self, setting: str) -> Dict: + """Get a setting from the MISP instance""" response = self._prepare_request('GET', f'servers/getSetting/{setting}') return self._check_json_response(response) def server_settings(self) -> Dict: + """Get all the settings from the server""" response = self._prepare_request('GET', 'servers/serverSettings') return self._check_json_response(response) def restart_workers(self) -> Dict: + """Restart all the workers""" response = self._prepare_request('POST', 'servers/restartWorkers') return self._check_json_response(response) def db_schema_diagnostic(self) -> Dict: + """Get the schema diagnostic""" response = self._prepare_request('GET', 'servers/dbSchemaDiagnostic') return self._check_json_response(response) def toggle_global_pythonify(self) -> None: + """Toggle the pythonify variable for the class""" self.global_pythonify = not self.global_pythonify # ## BEGIN Event ## def events(self, pythonify: bool = False) -> Union[Dict, List[MISPEvent]]: + """Get all the events from the MISP instance""" r = self._prepare_request('GET', 'events/index') events_r = self._check_json_response(r) if not (self.global_pythonify or pythonify) or 'errors' in events_r: @@ -424,6 +434,7 @@ class PyMISP: # ## BEGIN Attribute ### def attributes(self, pythonify: bool = False) -> Union[Dict, List[MISPAttribute]]: + """Get all the attributes from the MISP instance""" r = self._prepare_request('GET', 'attributes/index') attributes_r = self._check_json_response(r) if not (self.global_pythonify or pythonify) or 'errors' in attributes_r: @@ -519,6 +530,7 @@ class PyMISP: # ## BEGIN Attribute Proposal ### def attribute_proposals(self, event: Optional[Union[MISPEvent, int, str, UUID]] = None, pythonify: bool = False) -> Union[Dict, List[MISPShadowAttribute]]: + """Get all the attribute proposals""" if event: event_id = get_uuid_or_id_from_abstract_misp(event) r = self._prepare_request('GET', f'shadowAttributes/index/{event_id}') @@ -535,6 +547,7 @@ class PyMISP: return to_return def get_attribute_proposal(self, proposal: Union[MISPShadowAttribute, int, str, UUID], pythonify: bool = False) -> Union[Dict, MISPShadowAttribute]: + """Get an attribute proposal""" proposal_id = get_uuid_or_id_from_abstract_misp(proposal) r = self._prepare_request('GET', f'shadowAttributes/view/{proposal_id}') attribute_proposal = self._check_json_response(r) @@ -1166,6 +1179,7 @@ class PyMISP: return self._check_json_response(response) def test_server(self, server: Union[MISPServer, int, str, UUID]) -> Dict: + """Test if a sync link is working as expected""" server_id = get_uuid_or_id_from_abstract_misp(server) response = self._prepare_request('POST', f'servers/testConnection/{server_id}') return self._check_json_response(response) @@ -1409,6 +1423,7 @@ class PyMISP: role: Optional[Union[MISPRole, int, str]] = None, perm_sync: bool = False, perm_publish: bool = False, perm_admin: bool = False, unsafe_fallback: bool = False): + """Accept a user registration""" registration_id = get_uuid_or_id_from_abstract_misp(registration) if role: role_id = role_id = get_uuid_or_id_from_abstract_misp(role) @@ -1446,6 +1461,7 @@ class PyMISP: return self._check_json_response(r) def discard_user_registration(self, registration: Union[MISPInbox, int, str, UUID]): + """Discard a user registration""" registration_id = get_uuid_or_id_from_abstract_misp(registration) r = self._prepare_request('POST', f'users/discardRegistrations/{registration_id}') return self._check_json_response(r) @@ -1468,6 +1484,7 @@ class PyMISP: return to_return def set_default_role(self, role: Union[MISPRole, int, str, UUID]) -> Dict: + """Set a default role for the new user accounts""" role_id = get_uuid_or_id_from_abstract_misp(role) url = urljoin(self.root_url, f'/admin/roles/set_default/{role_id}') response = self._prepare_request('POST', url) @@ -1964,6 +1981,7 @@ class PyMISP: message: Optional[str] = None, sync: bool = False, anonymise_requestor_server: bool = False, mock: bool = False) -> Dict: + """Request the access to a community""" community_id = get_uuid_or_id_from_abstract_misp(community) to_post = {'org_name': requestor_organisation_name, 'org_uuid': requestor_organisation_uuid, @@ -1992,11 +2010,13 @@ class PyMISP: return to_return def accept_event_delegation(self, delegation: Union[MISPEventDelegation, int, str], pythonify: bool = False) -> Dict: + """Accept the delegation of an event""" delegation_id = get_uuid_or_id_from_abstract_misp(delegation) r = self._prepare_request('POST', f'eventDelegations/acceptDelegation/{delegation_id}') return self._check_json_response(r) def discard_event_delegation(self, delegation: Union[MISPEventDelegation, int, str], pythonify: bool = False) -> Dict: + """Discard the delegation of an event""" delegation_id = get_uuid_or_id_from_abstract_misp(delegation) r = self._prepare_request('POST', f'eventDelegations/deleteDelegation/{delegation_id}') return self._check_json_response(r) @@ -2005,7 +2025,8 @@ class PyMISP: organisation: Optional[Union[MISPOrganisation, int, str, UUID]] = None, event_delegation: Optional[MISPEventDelegation] = None, distribution: int = -1, message: str = '', pythonify: bool = False) -> Union[Dict, MISPEventDelegation]: - '''Note: distribution == -1 means recipient decides''' + '''Delegates an event. + Note: distribution == -1 means recipient decides''' if event and organisation: event_id = get_uuid_or_id_from_abstract_misp(event) organisation_id = get_uuid_or_id_from_abstract_misp(organisation) diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 617f0f6..2de7861 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -1529,7 +1529,11 @@ class MISPFeed(AbstractMISP): kwargs = kwargs['Feed'] super().from_dict(**kwargs) if hasattr(self, 'settings'): - self.settings = json.loads(self.settings) + try: + self.settings = json.loads(self.settings) + except json.decoder.JSONDecodeError as e: + logger.error("Failed to parse feed settings: {}".format(self.settings)) + raise e class MISPWarninglist(AbstractMISP): diff --git a/tests/testlive_comprehensive.py b/tests/testlive_comprehensive.py index 0489742..1961a0e 100644 --- a/tests/testlive_comprehensive.py +++ b/tests/testlive_comprehensive.py @@ -2752,11 +2752,11 @@ class TestComprehensive(unittest.TestCase): "warninglists/enableWarninglist", "warninglists/getToggleField", "warninglists/delete", - "admin/whitelists/add", - "admin/whitelists/index", - "admin/whitelists/edit", - "admin/whitelists/delete", - "whitelists/index" + "admin/allowedlists/add", + "admin/allowedlists/index", + "admin/allowedlists/edit", + "admin/allowedlists/delete", + "allowedlists/index" ] missing = self.admin_misp_connector.get_all_functions(True) with open('all_missing.json', 'w') as f: From 18474a2144d047024b1bb9a789663d8cce73237b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 15 Sep 2020 12:39:59 +0200 Subject: [PATCH 3/4] chg: Add comments to ELF, PE, and MachO object generators. --- pymisp/tools/elfobject.py | 2 ++ pymisp/tools/machoobject.py | 2 ++ pymisp/tools/peobject.py | 2 ++ 3 files changed, 6 insertions(+) diff --git a/pymisp/tools/elfobject.py b/pymisp/tools/elfobject.py index bacf85a..78c7f62 100644 --- a/pymisp/tools/elfobject.py +++ b/pymisp/tools/elfobject.py @@ -33,6 +33,7 @@ def make_elf_objects(lief_parsed: lief.Binary, misp_file: FileObject, standalone class ELFObject(AbstractMISPObjectGenerator): def __init__(self, parsed: lief.ELF.Binary = None, filepath: Union[Path, str] = None, pseudofile: Union[BytesIO, bytes] = None, **kwargs): + """Creates an ELF object, with lief""" super(ELFObject, self).__init__('elf', **kwargs) if not HAS_PYDEEP: logger.warning("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git") @@ -74,6 +75,7 @@ class ELFObject(AbstractMISPObjectGenerator): class ELFSectionObject(AbstractMISPObjectGenerator): def __init__(self, section: lief.ELF.Section, **kwargs): + """Creates an ELF Section object. Object generated by ELFObject.""" # Python3 way # super().__init__('pe-section') super(ELFSectionObject, self).__init__('elf-section', **kwargs) diff --git a/pymisp/tools/machoobject.py b/pymisp/tools/machoobject.py index 503ef13..c08ad7d 100644 --- a/pymisp/tools/machoobject.py +++ b/pymisp/tools/machoobject.py @@ -33,6 +33,7 @@ def make_macho_objects(lief_parsed: lief.Binary, misp_file: FileObject, standalo class MachOObject(AbstractMISPObjectGenerator): def __init__(self, parsed: Optional[lief.MachO.Binary] = None, filepath: Optional[Union[Path, str]] = None, pseudofile: Optional[BytesIO] = None, **kwargs): + """Creates an MachO object, with lief""" # Python3 way # super().__init__('elf') super(MachOObject, self).__init__('macho', **kwargs) @@ -76,6 +77,7 @@ class MachOObject(AbstractMISPObjectGenerator): class MachOSectionObject(AbstractMISPObjectGenerator): def __init__(self, section: lief.MachO.Section, **kwargs): + """Creates an MachO Section object. Object generated by MachOObject.""" # Python3 way # super().__init__('pe-section') super(MachOSectionObject, self).__init__('macho-section', **kwargs) diff --git a/pymisp/tools/peobject.py b/pymisp/tools/peobject.py index 7d5bcc9..47f0899 100644 --- a/pymisp/tools/peobject.py +++ b/pymisp/tools/peobject.py @@ -35,6 +35,7 @@ def make_pe_objects(lief_parsed: lief.Binary, misp_file: FileObject, standalone: class PEObject(AbstractMISPObjectGenerator): def __init__(self, parsed: Optional[lief.PE.Binary] = None, filepath: Optional[Union[Path, str]] = None, pseudofile: Optional[BytesIO] = None, **kwargs): + """Creates an PE object, with lief""" # Python3 way # super().__init__('pe') super(PEObject, self).__init__('pe', **kwargs) @@ -125,6 +126,7 @@ class PEObject(AbstractMISPObjectGenerator): class PESectionObject(AbstractMISPObjectGenerator): def __init__(self, section: lief.PE.Section, **kwargs): + """Creates an PE Section object. Object generated by PEObject.""" # Python3 way # super().__init__('pe-section') super(PESectionObject, self).__init__('pe-section', **kwargs) From d3db7fe52a99329e028e80c1ed732ea9019b218a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 15 Sep 2020 12:41:49 +0200 Subject: [PATCH 4/4] chg: Remove PyMISPExpanded from the docs --- docs/source/modules.rst | 6 ------ 1 file changed, 6 deletions(-) diff --git a/docs/source/modules.rst b/docs/source/modules.rst index 39843e4..7db7414 100644 --- a/docs/source/modules.rst +++ b/docs/source/modules.rst @@ -14,12 +14,6 @@ PyMISP .. autoclass:: PyMISP :members: -PyMISPExpanded (Python 3.6+ only) ---------------------------------- - -.. autoclass:: ExpandedPyMISP - :members: - MISPAbstract ------------