From 4e586d0de5c5322d65e558d703de0560404ab127 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 30 Jan 2020 11:44:13 +0100 Subject: [PATCH] chg: Bump deps, add pep8 test --- Pipfile | 1 + Pipfile.lock | 68 ++++++++++++++------------ pymisp/__init__.py | 2 - pymisp/abstract.py | 4 +- pymisp/exceptions.py | 2 +- pymisp/tools/abstractgenerator.py | 1 + pymisp/tools/peobject.py | 12 ++--- pymisp/tools/reportlab_generator.py | 75 +++++++++++++++-------------- pymisp/tools/vehicleobject.py | 3 -- travis/test_travis.sh | 1 + 10 files changed, 88 insertions(+), 81 deletions(-) diff --git a/Pipfile b/Pipfile index e72a3b7..1efd961 100644 --- a/Pipfile +++ b/Pipfile @@ -12,6 +12,7 @@ pymisp = {editable = true,extras = ["fileobjects", "neo", "openioc", "virustotal docutils = "==0.15" memory-profiler = "*" mypy = "*" +flake8 = "*" [packages] pymisp = {editable = true,extras = ["fileobjects", "openioc", "virustotal", "pdfexport"],path = "."} diff --git a/Pipfile.lock b/Pipfile.lock index 93611b5..a8f5ef2 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "77accb43d4bbba1ff86f29a66cae21eb56bc19ce82e39b096763dfaf65a9d5d8" + "sha256": "980c848909285e25224dc957df15e733666b06107dfbd97e6edfcd51c8da9206" }, "pipfile-spec": 6, "requires": { @@ -66,14 +66,6 @@ ], "version": "==2.8" }, - "importlib-metadata": { - "hashes": [ - "sha256:bdd9b7c397c273bcc9a11d6629a38487cd07154fa255a467bf704cd2c258e359", - "sha256:f17c015735e1a88296994c0697ecea7e11db24290941983b08c9feb30921e6d8" - ], - "markers": "python_version < '3.8'", - "version": "==1.4.0" - }, "jsonschema": { "hashes": [ "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163", @@ -240,13 +232,6 @@ "sha256:565a021fd19419476b9362b05eeaa094178de64f8361e44468f9e9d7843901e1" ], "version": "==1.11.2" - }, - "zipp": { - "hashes": [ - "sha256:ccc94ed0909b58ffe34430ea5451f07bc0c76467d7081619a454bf5c98b89e28", - "sha256:feae2f18633c32fc71f2de629bfb3bd3c9325cd4419642b1f1da42ee488d9b98" - ], - "version": "==2.1.0" } }, "develop": { @@ -394,6 +379,21 @@ "index": "pypi", "version": "==0.15" }, + "entrypoints": { + "hashes": [ + "sha256:589f874b313739ad35be6e0cd7efde2a4e9b6fea91edcc34e58ecbb8dbe56d19", + "sha256:c70dd71abe5a8c85e55e12c19bd91ccfeec11a6e99044204511f9ed547d48451" + ], + "version": "==0.3" + }, + "flake8": { + "hashes": [ + "sha256:45681a117ecc81e870cbf1262835ae4af5e7a8b08e40b944a8a6e6b895914cfb", + "sha256:49356e766643ad15072a789a20915d3c91dc89fd313ccd71802303fd67e4deca" + ], + "index": "pypi", + "version": "==3.7.9" + }, "idna": { "hashes": [ "sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407", @@ -408,14 +408,6 @@ ], "version": "==1.2.0" }, - "importlib-metadata": { - "hashes": [ - "sha256:bdd9b7c397c273bcc9a11d6629a38487cd07154fa255a467bf704cd2c258e359", - "sha256:f17c015735e1a88296994c0697ecea7e11db24290941983b08c9feb30921e6d8" - ], - "markers": "python_version < '3.8'", - "version": "==1.4.0" - }, "jinja2": { "hashes": [ "sha256:6e7a3c2934694d59ad334c93dd1b6c96699cf24c53fdb8ec848ac6b23e685734", @@ -487,6 +479,13 @@ ], "version": "==1.1.1" }, + "mccabe": { + "hashes": [ + "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42", + "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f" + ], + "version": "==0.6.1" + }, "memory-profiler": { "hashes": [ "sha256:23b196f91ea9ac9996e30bfab1e82fecc30a4a1d24870e81d1e81625f786a2c3" @@ -606,12 +605,26 @@ ], "version": "==4.3.0" }, + "pycodestyle": { + "hashes": [ + "sha256:95a2219d12372f05704562a14ec30bc76b05a5b297b21a5dfe3f6fac3491ae56", + "sha256:e40a936c9a450ad81df37f549d676d127b1b66000a6c500caa2b085bc0ca976c" + ], + "version": "==2.5.0" + }, "pydeep": { "hashes": [ "sha256:22866eb422d1d5907f8076ee792da65caecb172425d27576274e2a8eacf6afc1" ], "version": "==0.4" }, + "pyflakes": { + "hashes": [ + "sha256:17dbeb2e3f4d772725c777fabc446d5634d1038f234e77343108ce445ea69ce0", + "sha256:d976835886f8c5b31d47970ed689944a0262b5f3afa00a5a7b4dc81e5449f8a2" + ], + "version": "==2.1.1" + }, "pygments": { "hashes": [ "sha256:5ffada19f6203563680669ee7f53b64dabbeb100eb51b61996085e99c03b284a", @@ -854,13 +867,6 @@ "sha256:565a021fd19419476b9362b05eeaa094178de64f8361e44468f9e9d7843901e1" ], "version": "==1.11.2" - }, - "zipp": { - "hashes": [ - "sha256:ccc94ed0909b58ffe34430ea5451f07bc0c76467d7081619a454bf5c98b89e28", - "sha256:feae2f18633c32fc71f2de629bfb3bd3c9325cd4419642b1f1da42ee488d9b98" - ], - "version": "==2.1.0" } } } diff --git a/pymisp/__init__.py b/pymisp/__init__.py index 0a9f67c..3ac78e2 100644 --- a/pymisp/__init__.py +++ b/pymisp/__init__.py @@ -1,7 +1,5 @@ __version__ = '2.4.120' import logging -import warnings -import sys FORMAT = "%(levelname)s [%(filename)s:%(lineno)s - %(funcName)s() ] %(message)s" formatter = logging.Formatter(FORMAT) diff --git a/pymisp/abstract.py b/pymisp/abstract.py index f9eee1c..74d0a3d 100644 --- a/pymisp/abstract.py +++ b/pymisp/abstract.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -from datetime import date, datetime, timezone +from datetime import date, datetime from deprecated import deprecated # type: ignore from json import JSONEncoder @@ -21,7 +21,7 @@ except ImportError: import logging from enum import Enum -from typing import Union, Optional, List +from typing import Union, Optional from .exceptions import PyMISPInvalidFormat, PyMISPError diff --git a/pymisp/exceptions.py b/pymisp/exceptions.py index 1d7663f..8a809cc 100644 --- a/pymisp/exceptions.py +++ b/pymisp/exceptions.py @@ -66,7 +66,7 @@ class PyMISPNotImplementedYet(PyMISPError): class PyMISPUnexpectedResponse(PyMISPError): - pass + pass class PyMISPEmptyResponse(PyMISPError): diff --git a/pymisp/tools/abstractgenerator.py b/pymisp/tools/abstractgenerator.py index 9985ccd..12c1e35 100644 --- a/pymisp/tools/abstractgenerator.py +++ b/pymisp/tools/abstractgenerator.py @@ -7,6 +7,7 @@ from datetime import datetime, date from dateutil.parser import parse from typing import Union, Optional + class AbstractMISPObjectGenerator(MISPObject): def _detect_epoch(self, timestamp: Union[str, int, float]) -> bool: diff --git a/pymisp/tools/peobject.py b/pymisp/tools/peobject.py index f93ecea..aa22a19 100644 --- a/pymisp/tools/peobject.py +++ b/pymisp/tools/peobject.py @@ -84,10 +84,10 @@ class PEObject(AbstractMISPObjectGenerator): self.add_attribute('compilation-timestamp', value=datetime.utcfromtimestamp(self.__pe.header.time_date_stamps).isoformat()) # self.imphash = self.__pe.get_imphash() try: - if (self.__pe.has_resources and - self.__pe.resources_manager.has_version and - self.__pe.resources_manager.version.has_string_file_info and - self.__pe.resources_manager.version.string_file_info.langcode_items): + if (self.__pe.has_resources + and self.__pe.resources_manager.has_version + and self.__pe.resources_manager.version.has_string_file_info + and self.__pe.resources_manager.version.string_file_info.langcode_items): fileinfo = dict(self.__pe.resources_manager.version.string_file_info.langcode_items[0].items.items()) self.add_attribute('original-filename', value=fileinfo.get('OriginalFilename')) self.add_attribute('internal-filename', value=fileinfo.get('InternalName')) @@ -108,8 +108,8 @@ class PEObject(AbstractMISPObjectGenerator): for section in self.__pe.sections: s = PESectionObject(section, self._standalone, default_attributes_parameters=self._default_attributes_parameters) self.add_reference(s.uuid, 'includes', 'Section {} of PE'.format(pos)) - if ((self.__pe.entrypoint >= section.virtual_address) and - (self.__pe.entrypoint < (section.virtual_address + section.virtual_size))): + if ((self.__pe.entrypoint >= section.virtual_address) + and (self.__pe.entrypoint < (section.virtual_address + section.virtual_size))): self.add_attribute('entrypoint-section-at-position', value='{}|{}'.format(section.name, pos)) pos += 1 self.sections.append(s) diff --git a/pymisp/tools/reportlab_generator.py b/pymisp/tools/reportlab_generator.py index ed6c52c..b8f3369 100644 --- a/pymisp/tools/reportlab_generator.py +++ b/pymisp/tools/reportlab_generator.py @@ -481,14 +481,17 @@ def get_clusters_table_styles(): def safe_string(bad_str): return escape(str(bad_str)) + def is_safe_value(value): return (value is not None and value != "") + def is_safe_table(value): return (value is not None and value != []) + def is_safe_attribute(curr_object, attribute_name): return (hasattr(curr_object, attribute_name) and getattr(curr_object, attribute_name) is not None @@ -660,7 +663,7 @@ class Value_Formatter(): return self.get_unoverflowable_paragraph(answer) - def get_threat_value(self, threat_level = None): + def get_threat_value(self, threat_level=None): ''' Returns a flowable paragraph to add to the pdf given the misp_event threat :param threat_level: MISP_EVENT threat level (int) to be formatted @@ -671,9 +674,9 @@ class Value_Formatter(): if is_safe_value(threat_level) and str(threat_level) in threat_map: answer = threat_map[safe_string(threat_level)] - return self.get_unoverflowable_paragraph(answer,do_escape_string=False) + return self.get_unoverflowable_paragraph(answer, do_escape_string=False) - def get_analysis_value(self, analysis_level = None): + def get_analysis_value(self, analysis_level=None): ''' Returns a flowable paragraph to add to the pdf given the misp_event analysis :param analysis_level: MISP_EVENT analysis level (int) to be formatted @@ -684,7 +687,7 @@ class Value_Formatter(): if is_safe_value(analysis_level) and str(analysis_level) in analysis_map: answer = analysis_map[safe_string(analysis_level)] - return self.get_unoverflowable_paragraph(answer,do_escape_string=False) + return self.get_unoverflowable_paragraph(answer, do_escape_string=False) def get_timestamp_value(self, timestamp=None): ''' @@ -764,7 +767,7 @@ class Value_Formatter(): try: # Get the image - buf = image_buffer # TODO : Do verification on the buffer ? + buf = image_buffer # TODO : Do verification on the buffer ? # Create image within a bounded box (to allow pdf creation) img = Image(buf, width=FRAME_PICTURE_MAX_WIDTH, height=FRAME_PICTURE_MAX_HEIGHT, kind='bound') @@ -821,8 +824,8 @@ class Value_Formatter(): if is_safe_dict_attribute(misp_galaxy, 'name'): answer = '{} from {}:{}'.format(safe_string(misp_galaxy['name']), - safe_string(misp_galaxy["namespace"]), - safe_string(misp_galaxy["type"])) + safe_string(misp_galaxy["namespace"]), + safe_string(misp_galaxy["type"])) return self.get_unoverflowable_paragraph(answer, do_small=True) @@ -866,7 +869,7 @@ class Event_Metadata(): ######################################################################## # General Event's Attributes formater - def create_flowable_table_from_event(self, misp_event ): + def create_flowable_table_from_event(self, misp_event): ''' Returns Table presenting a MISP event :param misp_event: A misp event (complete or not) @@ -879,8 +882,8 @@ class Event_Metadata(): # Manual addition # UUID data.append([self.value_formatter.get_col1_paragraph("UUID"), - self.value_formatter.get_value_link_to_event(uuid=misp_event.get('uuid',None), - text=misp_event.get('uuid',None))]) + self.value_formatter.get_value_link_to_event(uuid=misp_event.get('uuid', None), + text=misp_event.get('uuid', None))]) # Date data.append({self.value_formatter.get_col1_paragraph("Date"), @@ -888,48 +891,48 @@ class Event_Metadata(): # Owner data.append([self.value_formatter.get_col1_paragraph("Owner org"), - self.value_formatter.get_owner_value(owner=misp_event.get('owner',None))]) + self.value_formatter.get_owner_value(owner=misp_event.get('owner', None))]) # Threat data.append([self.value_formatter.get_col1_paragraph("Threat level"), - self.value_formatter.get_threat_value(threat_level=misp_event.get('threat_level_id',None))]) + self.value_formatter.get_threat_value(threat_level=misp_event.get('threat_level_id', None))]) # Analysis data.append([self.value_formatter.get_col1_paragraph("Analysis"), - self.value_formatter.get_analysis_value(analysis_level=misp_event.get('analysis',None))]) + self.value_formatter.get_analysis_value(analysis_level=misp_event.get('analysis', None))]) # Info data.append([self.value_formatter.get_col1_paragraph("Info"), - self.value_formatter.get_value_link_to_event(uuid=misp_event.get('uuid',None), - text=misp_event.get('info',None))]) + self.value_formatter.get_value_link_to_event(uuid=misp_event.get('uuid', None), + text=misp_event.get('info', None))]) # Timestamp data.append([self.value_formatter.get_col1_paragraph("Event date"), - self.value_formatter.get_timestamp_value(timestamp=misp_event.get('timestamp',None))]) + self.value_formatter.get_timestamp_value(timestamp=misp_event.get('timestamp', None))]) # Published data.append([self.value_formatter.get_col1_paragraph("Published"), - self.value_formatter.get_published_value(published_bool=misp_event.get('published',None), - published_timestamp=misp_event.get('publish_timestamp',None))]) + self.value_formatter.get_published_value(published_bool=misp_event.get('published', None), + published_timestamp=misp_event.get('publish_timestamp', None))]) # Creator organisation data.append([self.value_formatter.get_col1_paragraph("Creator Org"), - self.value_formatter.get_creator_organisation_value(creator=misp_event.get('Orgc',None))]) + self.value_formatter.get_creator_organisation_value(creator=misp_event.get('Orgc', None))]) # Number of Attributes data.append([self.value_formatter.get_col1_paragraph("# Attributes"), - self.value_formatter.get_attributes_number_value(attributes=misp_event.get('Attribute',None))]) + self.value_formatter.get_attributes_number_value(attributes=misp_event.get('Attribute', None))]) # Tags curr_Tags = Tags(self.config, self.value_formatter) data.append([self.value_formatter.get_col1_paragraph("Tags"), - curr_Tags.get_tag_value(tags=misp_event.get('Tag',None))]) + curr_Tags.get_tag_value(tags=misp_event.get('Tag', None))]) flowable_table.append(create_flowable_table_from_data(data)) # Correlation - if is_safe_table(misp_event.get('RelatedEvent',None)) and is_in_config(self.config, 4): - flowable_table += self.get_correlation_values(related_events=misp_event.get('RelatedEvent',None)) + if is_safe_table(misp_event.get('RelatedEvent', None)) and is_in_config(self.config, 4): + flowable_table += self.get_correlation_values(related_events=misp_event.get('RelatedEvent', None)) # Galaxies if is_safe_attribute_table(misp_event, "Related Galaxies") and is_in_config(self.config, 3): @@ -952,17 +955,17 @@ class Event_Metadata(): # Manual addition # UUID data.append([self.value_formatter.get_col1_paragraph("UUID"), - self.value_formatter.get_value_link_to_event(uuid=misp_event.get('uuid',None), - text=misp_event.get('uuid',None))]) + self.value_formatter.get_value_link_to_event(uuid=misp_event.get('uuid', None), + text=misp_event.get('uuid', None))]) # Info data.append([self.value_formatter.get_col1_paragraph("Info"), - self.value_formatter.get_value_link_to_event(uuid=misp_event.get('uuid',None), - text=misp_event.get('info',None))]) + self.value_formatter.get_value_link_to_event(uuid=misp_event.get('uuid', None), + text=misp_event.get('info', None))]) # Timestamp data.append([self.value_formatter.get_col1_paragraph("Event date"), - self.value_formatter.get_timestamp_value(timestamp=misp_event.get('timestamp',None))]) + self.value_formatter.get_timestamp_value(timestamp=misp_event.get('timestamp', None))]) flowable_table.append(create_flowable_table_from_data(data)) @@ -1167,10 +1170,10 @@ class Attributes(): # data.append([Paragraph(item[0], col1_style), Paragraph(item[2], col2_style)]) # Handle Special case for links (Value) - There were not written in the previous loop - if not STANDARD_TYPE and is_safe_value(misp_attribute.get('value',None)): + if not STANDARD_TYPE and is_safe_value(misp_attribute.get('value', None)): data.append([self.value_formatter.get_col1_paragraph("Value"), - self.value_formatter.get_good_or_bad_link(value=misp_attribute.get('value',None), - type=misp_attribute.get('type',None))]) + self.value_formatter.get_good_or_bad_link(value=misp_attribute.get('value', None), + type=misp_attribute.get('type', None))]) # Handle pictures if is_safe_value(misp_attribute.get('data', None)) and misp_attribute.type == IMAGE_TYPE: @@ -1190,7 +1193,7 @@ class Attributes(): if is_safe_table(misp_attribute.get('Sighting', None)): data.append([self.value_formatter.get_col1_paragraph("Sighting"), - curr_Sighting.create_flowable_paragraph_from_sightings(sightings=misp_attribute.get('Sighting',None))]) + curr_Sighting.create_flowable_paragraph_from_sightings(sightings=misp_attribute.get('Sighting', None))]) flowable_table.append(create_flowable_table_from_data(data)) @@ -1399,7 +1402,7 @@ class Object(): data = [create_flowable_table_from_data(data)] # Handle all the attributes - if is_safe_value(misp_object.get("Attribute",None)): + if is_safe_value(misp_object.get("Attribute", None)): curr_attributes = Attributes(self.config, self.value_formatter) data.append(Indenter(left=INDENT_SIZE)) data += curr_attributes.create_flowable_table_from_attributes(misp_object) @@ -1674,8 +1677,8 @@ def collect_parts(misp_event, config=None): # Create stuff title_style = ParagraphStyle(name='Column_1', parent=sample_style_sheet['Heading1'], fontName=FIRST_COL_FONT, alignment=TA_CENTER) - title = curr_val_f.get_value_link_to_event(uuid=misp_event.get('uuid',None), - text=misp_event.get('info',None), + title = curr_val_f.get_value_link_to_event(uuid=misp_event.get('uuid', None), + text=misp_event.get('info', None), curr_style=title_style, color=False) # Add all parts to final PDF flowables.append(title) @@ -1708,7 +1711,7 @@ def collect_parts(misp_event, config=None): flowables.append(PageBreak()) event_objects_title = Paragraph("Objects", sample_style_sheet['Heading2']) - table_objects = curr_object.create_flowable_table_from_objects(objects=misp_event.get("Object",None)) + table_objects = curr_object.create_flowable_table_from_objects(objects=misp_event.get("Object", None)) flowables.append(event_objects_title) flowables += table_objects diff --git a/pymisp/tools/vehicleobject.py b/pymisp/tools/vehicleobject.py index ea347bc..1c2bf6a 100644 --- a/pymisp/tools/vehicleobject.py +++ b/pymisp/tools/vehicleobject.py @@ -1,10 +1,7 @@ #!/usr/bin/python3 -import sys -import getopt import requests import json -from pymisp import MISPObject from .abstractgenerator import AbstractMISPObjectGenerator diff --git a/travis/test_travis.sh b/travis/test_travis.sh index 8fcbc60..26674ee 100644 --- a/travis/test_travis.sh +++ b/travis/test_travis.sh @@ -5,3 +5,4 @@ set -x pipenv run nosetests-3.4 --with-coverage --cover-package=pymisp,tests --cover-tests tests/test_*.py pipenv run mypy tests/testlive_comprehensive.py tests/test_mispevent.py tests/testlive_sync.py pymisp +pipenv run flake8 --ignore=E501,W503,E226,E252 pymisp