MISP
/
PyMISP
mirror of https://github.com/MISP/PyMISP
2
0
Fork 0
Code Issues Releases Wiki Activity

chg: Bump deps, add pep8 test

pull/534/head
Raphaël Vinot 2020-01-30 11:44:13 +01:00
parent cbce2cfbfe
commit 4e586d0de5
10 changed files with 88 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Pipfile
View File

@ -12,6 +12,7 @@ pymisp = {editable = true,extras = ["fileobjects", "neo", "openioc", "virustotal
docutils = "==0.15"
memory-profiler = "*"
mypy = "*"
flake8 = "*"
[packages]
pymisp = {editable = true,extras = ["fileobjects", "openioc", "virustotal", "pdfexport"],path = "."}

68
Pipfile.lock generated
View File

@ -1,7 +1,7 @@
{
"_meta": {
"hash": {
"sha256": "77accb43d4bbba1ff86f29a66cae21eb56bc19ce82e39b096763dfaf65a9d5d8"
"sha256": "980c848909285e25224dc957df15e733666b06107dfbd97e6edfcd51c8da9206"
},
"pipfile-spec": 6,
"requires": {
@ -66,14 +66,6 @@
],
"version": "==2.8"
},
"importlib-metadata": {
"hashes": [
"sha256:bdd9b7c397c273bcc9a11d6629a38487cd07154fa255a467bf704cd2c258e359",
"sha256:f17c015735e1a88296994c0697ecea7e11db24290941983b08c9feb30921e6d8"
],
"markers": "python_version < '3.8'",
"version": "==1.4.0"
},
"jsonschema": {
"hashes": [
"sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163",
@ -240,13 +232,6 @@
"sha256:565a021fd19419476b9362b05eeaa094178de64f8361e44468f9e9d7843901e1"
],
"version": "==1.11.2"
},
"zipp": {
"hashes": [
"sha256:ccc94ed0909b58ffe34430ea5451f07bc0c76467d7081619a454bf5c98b89e28",
"sha256:feae2f18633c32fc71f2de629bfb3bd3c9325cd4419642b1f1da42ee488d9b98"
],
"version": "==2.1.0"
}
},
"develop": {
@ -394,6 +379,21 @@
"index": "pypi",
"version": "==0.15"
},
"entrypoints": {
"hashes": [
"sha256:589f874b313739ad35be6e0cd7efde2a4e9b6fea91edcc34e58ecbb8dbe56d19",
"sha256:c70dd71abe5a8c85e55e12c19bd91ccfeec11a6e99044204511f9ed547d48451"
],
"version": "==0.3"
},
"flake8": {
"hashes": [
"sha256:45681a117ecc81e870cbf1262835ae4af5e7a8b08e40b944a8a6e6b895914cfb",
"sha256:49356e766643ad15072a789a20915d3c91dc89fd313ccd71802303fd67e4deca"
],
"index": "pypi",
"version": "==3.7.9"
},
"idna": {
"hashes": [
"sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407",
@ -408,14 +408,6 @@
],
"version": "==1.2.0"
},
"importlib-metadata": {
"hashes": [
"sha256:bdd9b7c397c273bcc9a11d6629a38487cd07154fa255a467bf704cd2c258e359",
"sha256:f17c015735e1a88296994c0697ecea7e11db24290941983b08c9feb30921e6d8"
],
"markers": "python_version < '3.8'",
"version": "==1.4.0"
},
"jinja2": {
"hashes": [
"sha256:6e7a3c2934694d59ad334c93dd1b6c96699cf24c53fdb8ec848ac6b23e685734",
@ -487,6 +479,13 @@
],
"version": "==1.1.1"
},
"mccabe": {
"hashes": [
"sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42",
"sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"
],
"version": "==0.6.1"
},
"memory-profiler": {
"hashes": [
"sha256:23b196f91ea9ac9996e30bfab1e82fecc30a4a1d24870e81d1e81625f786a2c3"
@ -606,12 +605,26 @@
],
"version": "==4.3.0"
},
"pycodestyle": {
"hashes": [
"sha256:95a2219d12372f05704562a14ec30bc76b05a5b297b21a5dfe3f6fac3491ae56",
"sha256:e40a936c9a450ad81df37f549d676d127b1b66000a6c500caa2b085bc0ca976c"
],
"version": "==2.5.0"
},
"pydeep": {
"hashes": [
"sha256:22866eb422d1d5907f8076ee792da65caecb172425d27576274e2a8eacf6afc1"
],
"version": "==0.4"
},
"pyflakes": {
"hashes": [
"sha256:17dbeb2e3f4d772725c777fabc446d5634d1038f234e77343108ce445ea69ce0",
"sha256:d976835886f8c5b31d47970ed689944a0262b5f3afa00a5a7b4dc81e5449f8a2"
],
"version": "==2.1.1"
},
"pygments": {
"hashes": [
"sha256:5ffada19f6203563680669ee7f53b64dabbeb100eb51b61996085e99c03b284a",
@ -854,13 +867,6 @@
"sha256:565a021fd19419476b9362b05eeaa094178de64f8361e44468f9e9d7843901e1"
],
"version": "==1.11.2"
},
"zipp": {
"hashes": [
"sha256:ccc94ed0909b58ffe34430ea5451f07bc0c76467d7081619a454bf5c98b89e28",
"sha256:feae2f18633c32fc71f2de629bfb3bd3c9325cd4419642b1f1da42ee488d9b98"
],
"version": "==2.1.0"
}
}
}

2
pymisp/__init__.py
View File

@ -1,7 +1,5 @@
__version__ = '2.4.120'
import logging
import warnings
import sys
FORMAT = "%(levelname)s [%(filename)s:%(lineno)s - %(funcName)s() ] %(message)s"
formatter = logging.Formatter(FORMAT)

4
pymisp/abstract.py
View File

@ -1,7 +1,7 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from datetime import date, datetime, timezone
from datetime import date, datetime
from deprecated import deprecated # type: ignore
from json import JSONEncoder
@ -21,7 +21,7 @@ except ImportError:
import logging
from enum import Enum
from typing import Union, Optional, List
from typing import Union, Optional
from .exceptions import PyMISPInvalidFormat, PyMISPError

2
pymisp/exceptions.py
View File

@ -66,7 +66,7 @@ class PyMISPNotImplementedYet(PyMISPError):
class PyMISPUnexpectedResponse(PyMISPError):
pass
pass
class PyMISPEmptyResponse(PyMISPError):

1
pymisp/tools/abstractgenerator.py
View File

@ -7,6 +7,7 @@ from datetime import datetime, date
from dateutil.parser import parse
from typing import Union, Optional
class AbstractMISPObjectGenerator(MISPObject):
def _detect_epoch(self, timestamp: Union[str, int, float]) -> bool:

12
pymisp/tools/peobject.py
View File

@ -84,10 +84,10 @@ class PEObject(AbstractMISPObjectGenerator):
self.add_attribute('compilation-timestamp', value=datetime.utcfromtimestamp(self.__pe.header.time_date_stamps).isoformat())
# self.imphash = self.__pe.get_imphash()
try:
if (self.__pe.has_resources and
self.__pe.resources_manager.has_version and
self.__pe.resources_manager.version.has_string_file_info and
self.__pe.resources_manager.version.string_file_info.langcode_items):
if (self.__pe.has_resources
and self.__pe.resources_manager.has_version
and self.__pe.resources_manager.version.has_string_file_info
and self.__pe.resources_manager.version.string_file_info.langcode_items):
fileinfo = dict(self.__pe.resources_manager.version.string_file_info.langcode_items[0].items.items())
self.add_attribute('original-filename', value=fileinfo.get('OriginalFilename'))
self.add_attribute('internal-filename', value=fileinfo.get('InternalName'))
@ -108,8 +108,8 @@ class PEObject(AbstractMISPObjectGenerator):
for section in self.__pe.sections:
s = PESectionObject(section, self._standalone, default_attributes_parameters=self._default_attributes_parameters)
self.add_reference(s.uuid, 'includes', 'Section {} of PE'.format(pos))
if ((self.__pe.entrypoint >= section.virtual_address) and
(self.__pe.entrypoint < (section.virtual_address + section.virtual_size))):
if ((self.__pe.entrypoint >= section.virtual_address)
and (self.__pe.entrypoint < (section.virtual_address + section.virtual_size))):
self.add_attribute('entrypoint-section-at-position', value='{}|{}'.format(section.name, pos))
pos += 1
self.sections.append(s)

75
pymisp/tools/reportlab_generator.py
View File

@ -481,14 +481,17 @@ def get_clusters_table_styles():
def safe_string(bad_str):
return escape(str(bad_str))
def is_safe_value(value):
return (value is not None
and value != "")
def is_safe_table(value):
return (value is not None
and value != [])
def is_safe_attribute(curr_object, attribute_name):
return (hasattr(curr_object, attribute_name)
and getattr(curr_object, attribute_name) is not None
@ -660,7 +663,7 @@ class Value_Formatter():
return self.get_unoverflowable_paragraph(answer)
def get_threat_value(self, threat_level = None):
def get_threat_value(self, threat_level=None):
'''
Returns a flowable paragraph to add to the pdf given the misp_event threat
:param threat_level: MISP_EVENT threat level (int) to be formatted
@ -671,9 +674,9 @@ class Value_Formatter():
if is_safe_value(threat_level) and str(threat_level) in threat_map:
answer = threat_map[safe_string(threat_level)]
return self.get_unoverflowable_paragraph(answer,do_escape_string=False)
return self.get_unoverflowable_paragraph(answer, do_escape_string=False)
def get_analysis_value(self, analysis_level = None):
def get_analysis_value(self, analysis_level=None):
'''
Returns a flowable paragraph to add to the pdf given the misp_event analysis
:param analysis_level: MISP_EVENT analysis level (int) to be formatted
@ -684,7 +687,7 @@ class Value_Formatter():
if is_safe_value(analysis_level) and str(analysis_level) in analysis_map:
answer = analysis_map[safe_string(analysis_level)]
return self.get_unoverflowable_paragraph(answer,do_escape_string=False)
return self.get_unoverflowable_paragraph(answer, do_escape_string=False)
def get_timestamp_value(self, timestamp=None):
'''
@ -764,7 +767,7 @@ class Value_Formatter():
try:
# Get the image
buf = image_buffer # TODO : Do verification on the buffer ?
buf = image_buffer # TODO : Do verification on the buffer ?
# Create image within a bounded box (to allow pdf creation)
img = Image(buf, width=FRAME_PICTURE_MAX_WIDTH, height=FRAME_PICTURE_MAX_HEIGHT, kind='bound')
@ -821,8 +824,8 @@ class Value_Formatter():
if is_safe_dict_attribute(misp_galaxy, 'name'):
answer = '{} <i>from</i> {}:{}'.format(safe_string(misp_galaxy['name']),
safe_string(misp_galaxy["namespace"]),
safe_string(misp_galaxy["type"]))
safe_string(misp_galaxy["namespace"]),
safe_string(misp_galaxy["type"]))
return self.get_unoverflowable_paragraph(answer, do_small=True)
@ -866,7 +869,7 @@ class Event_Metadata():
########################################################################
# General Event's Attributes formater
def create_flowable_table_from_event(self, misp_event ):
def create_flowable_table_from_event(self, misp_event):
'''
Returns Table presenting a MISP event
:param misp_event: A misp event (complete or not)
@ -879,8 +882,8 @@ class Event_Metadata():
# Manual addition
# UUID
data.append([self.value_formatter.get_col1_paragraph("UUID"),
self.value_formatter.get_value_link_to_event(uuid=misp_event.get('uuid',None),
text=misp_event.get('uuid',None))])
self.value_formatter.get_value_link_to_event(uuid=misp_event.get('uuid', None),
text=misp_event.get('uuid', None))])
# Date
data.append({self.value_formatter.get_col1_paragraph("Date"),
@ -888,48 +891,48 @@ class Event_Metadata():
# Owner
data.append([self.value_formatter.get_col1_paragraph("Owner org"),
self.value_formatter.get_owner_value(owner=misp_event.get('owner',None))])
self.value_formatter.get_owner_value(owner=misp_event.get('owner', None))])
# Threat
data.append([self.value_formatter.get_col1_paragraph("Threat level"),
self.value_formatter.get_threat_value(threat_level=misp_event.get('threat_level_id',None))])
self.value_formatter.get_threat_value(threat_level=misp_event.get('threat_level_id', None))])
# Analysis
data.append([self.value_formatter.get_col1_paragraph("Analysis"),
self.value_formatter.get_analysis_value(analysis_level=misp_event.get('analysis',None))])
self.value_formatter.get_analysis_value(analysis_level=misp_event.get('analysis', None))])
# Info
data.append([self.value_formatter.get_col1_paragraph("Info"),
self.value_formatter.get_value_link_to_event(uuid=misp_event.get('uuid',None),
text=misp_event.get('info',None))])
self.value_formatter.get_value_link_to_event(uuid=misp_event.get('uuid', None),
text=misp_event.get('info', None))])
# Timestamp
data.append([self.value_formatter.get_col1_paragraph("Event date"),
self.value_formatter.get_timestamp_value(timestamp=misp_event.get('timestamp',None))])
self.value_formatter.get_timestamp_value(timestamp=misp_event.get('timestamp', None))])
# Published
data.append([self.value_formatter.get_col1_paragraph("Published"),
self.value_formatter.get_published_value(published_bool=misp_event.get('published',None),
published_timestamp=misp_event.get('publish_timestamp',None))])
self.value_formatter.get_published_value(published_bool=misp_event.get('published', None),
published_timestamp=misp_event.get('publish_timestamp', None))])
# Creator organisation
data.append([self.value_formatter.get_col1_paragraph("Creator Org"),
self.value_formatter.get_creator_organisation_value(creator=misp_event.get('Orgc',None))])
self.value_formatter.get_creator_organisation_value(creator=misp_event.get('Orgc', None))])
# Number of Attributes
data.append([self.value_formatter.get_col1_paragraph("# Attributes"),
self.value_formatter.get_attributes_number_value(attributes=misp_event.get('Attribute',None))])
self.value_formatter.get_attributes_number_value(attributes=misp_event.get('Attribute', None))])
# Tags
curr_Tags = Tags(self.config, self.value_formatter)
data.append([self.value_formatter.get_col1_paragraph("Tags"),
curr_Tags.get_tag_value(tags=misp_event.get('Tag',None))])
curr_Tags.get_tag_value(tags=misp_event.get('Tag', None))])
flowable_table.append(create_flowable_table_from_data(data))
# Correlation
if is_safe_table(misp_event.get('RelatedEvent',None)) and is_in_config(self.config, 4):
flowable_table += self.get_correlation_values(related_events=misp_event.get('RelatedEvent',None))
if is_safe_table(misp_event.get('RelatedEvent', None)) and is_in_config(self.config, 4):
flowable_table += self.get_correlation_values(related_events=misp_event.get('RelatedEvent', None))
# Galaxies
if is_safe_attribute_table(misp_event, "Related Galaxies") and is_in_config(self.config, 3):
@ -952,17 +955,17 @@ class Event_Metadata():
# Manual addition
# UUID
data.append([self.value_formatter.get_col1_paragraph("UUID"),
self.value_formatter.get_value_link_to_event(uuid=misp_event.get('uuid',None),
text=misp_event.get('uuid',None))])
self.value_formatter.get_value_link_to_event(uuid=misp_event.get('uuid', None),
text=misp_event.get('uuid', None))])
# Info
data.append([self.value_formatter.get_col1_paragraph("Info"),
self.value_formatter.get_value_link_to_event(uuid=misp_event.get('uuid',None),
text=misp_event.get('info',None))])
self.value_formatter.get_value_link_to_event(uuid=misp_event.get('uuid', None),
text=misp_event.get('info', None))])
# Timestamp
data.append([self.value_formatter.get_col1_paragraph("Event date"),
self.value_formatter.get_timestamp_value(timestamp=misp_event.get('timestamp',None))])
self.value_formatter.get_timestamp_value(timestamp=misp_event.get('timestamp', None))])
flowable_table.append(create_flowable_table_from_data(data))
@ -1167,10 +1170,10 @@ class Attributes():
# data.append([Paragraph(item[0], col1_style), Paragraph(item[2], col2_style)])
# Handle Special case for links (Value) - There were not written in the previous loop
if not STANDARD_TYPE and is_safe_value(misp_attribute.get('value',None)):
if not STANDARD_TYPE and is_safe_value(misp_attribute.get('value', None)):
data.append([self.value_formatter.get_col1_paragraph("Value"),
self.value_formatter.get_good_or_bad_link(value=misp_attribute.get('value',None),
type=misp_attribute.get('type',None))])
self.value_formatter.get_good_or_bad_link(value=misp_attribute.get('value', None),
type=misp_attribute.get('type', None))])
# Handle pictures
if is_safe_value(misp_attribute.get('data', None)) and misp_attribute.type == IMAGE_TYPE:
@ -1190,7 +1193,7 @@ class Attributes():
if is_safe_table(misp_attribute.get('Sighting', None)):
data.append([self.value_formatter.get_col1_paragraph("Sighting"),
curr_Sighting.create_flowable_paragraph_from_sightings(sightings=misp_attribute.get('Sighting',None))])
curr_Sighting.create_flowable_paragraph_from_sightings(sightings=misp_attribute.get('Sighting', None))])
flowable_table.append(create_flowable_table_from_data(data))
@ -1399,7 +1402,7 @@ class Object():
data = [create_flowable_table_from_data(data)]
# Handle all the attributes
if is_safe_value(misp_object.get("Attribute",None)):
if is_safe_value(misp_object.get("Attribute", None)):
curr_attributes = Attributes(self.config, self.value_formatter)
data.append(Indenter(left=INDENT_SIZE))
data += curr_attributes.create_flowable_table_from_attributes(misp_object)
@ -1674,8 +1677,8 @@ def collect_parts(misp_event, config=None):
# Create stuff
title_style = ParagraphStyle(name='Column_1', parent=sample_style_sheet['Heading1'],
fontName=FIRST_COL_FONT, alignment=TA_CENTER)
title = curr_val_f.get_value_link_to_event(uuid=misp_event.get('uuid',None),
text=misp_event.get('info',None),
title = curr_val_f.get_value_link_to_event(uuid=misp_event.get('uuid', None),
text=misp_event.get('info', None),
curr_style=title_style, color=False)
# Add all parts to final PDF
flowables.append(title)
@ -1708,7 +1711,7 @@ def collect_parts(misp_event, config=None):
flowables.append(PageBreak())
event_objects_title = Paragraph("Objects", sample_style_sheet['Heading2'])
table_objects = curr_object.create_flowable_table_from_objects(objects=misp_event.get("Object",None))
table_objects = curr_object.create_flowable_table_from_objects(objects=misp_event.get("Object", None))
flowables.append(event_objects_title)
flowables += table_objects

3
pymisp/tools/vehicleobject.py
View File

@ -1,10 +1,7 @@
#!/usr/bin/python3
import sys
import getopt
import requests
import json
from pymisp import MISPObject
from .abstractgenerator import AbstractMISPObjectGenerator

1
travis/test_travis.sh
View File

@ -5,3 +5,4 @@ set -x
pipenv run nosetests-3.4 --with-coverage --cover-package=pymisp,tests --cover-tests tests/test_*.py
pipenv run mypy tests/testlive_comprehensive.py tests/test_mispevent.py tests/testlive_sync.py pymisp
pipenv run flake8 --ignore=E501,W503,E226,E252 pymisp