MISP
/
PyMISP
mirror of https://github.com/MISP/PyMISP
2
0
Fork 0
Code Issues Releases Wiki Activity

Merge remote-tracking branch 'upstream/master'

pull/142/head
c-goes 2017-11-09 16:00:05 +01:00
parent 4bc3ec91bf e6ab90012d
commit dee33507e7
27 changed files with 925 additions and 304 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
.travis.yml
View File

@ -17,14 +17,10 @@ python:
- "3.6-dev"
install:
- pip install -U nose
- pip install coveralls
- pip install codecov
- pip install requests-mock
- pip install https://github.com/lief-project/packages/raw/lief-master-latest/pylief-0.7.0.dev.zip
- pip install -U nose pip setuptools
- pip install coveralls codecov requests-mock
- pip install git+https://github.com/kbandla/pydeep.git
- pip install python-magic
- pip install .
- pip install .[fileobjects,neo,openioc,virustotal]
- pushd tests
- git clone https://github.com/viper-framework/viper-test-files.git
- popd

55
CHANGELOG.txt
View File

@ -2,6 +2,61 @@ Changelog
=========
v2.4.81.2 (2017-10-24)
----------------------
Changes
~~~~~~~
- Version bump. [Raphaël Vinot]
- Update changelog. [Raphaël Vinot]
Fix
~~~
- Properly bundle object templates. [Raphaël Vinot]
v2.4.81.1 (2017-10-24)
----------------------
Changes
~~~~~~~
- Bump version. [Raphaël Vinot]
- Do not raise an exception when the object template is unknown.
[Raphaël Vinot]
+ bump misp-object
- Bump misp-objects. [Raphaël Vinot]
- Allow to hard delete an attribute by ID. [Raphaël Vinot]
- Update comments. [Raphaël Vinot]
- Bump misp-objects and describeTypes. [Raphaël Vinot]
Fix
~~~
- Properly bundle object templates. [Raphaël Vinot]
- Fix typos and logic mistakes in mispevent. [Raphaël Vinot]
- Fix travis build. [Raphaël Vinot]
- Min required version of setuptools. [Raphaël Vinot]
- Improve dependencies listing. [Raphaël Vinot]
Partial fix for #110
- Missing default category. [Raphaël Vinot]
Fix #119
Other
~~~~~
- Merge branch 'master' of github.com:MISP/PyMISP. [Raphaël Vinot]
- Update openioc.py. [Andras Iklody]
- Merge branch 'master' of github.com:MISP/PyMISP. [Raphaël Vinot]
- Merge pull request #121 from kx499/master. [Raphaël Vinot]
Added **kwargs to add_named_attribute call in add_attachment
- Added **kwargs to add_named_attribute call in add_attachment.
[tssbo82]
- Update README. [Raphaël Vinot]
- Update changelog. [Raphaël Vinot]
v2.4.81 (2017-10-09)
--------------------

5
MANIFEST.in
View File

@ -1 +1,4 @@
include pymisp/data/*
include pymisp/data/*.json
include pymisp/data/misp-objects/*.json
include pymisp/data/misp-objects/objects/*/definition.json
include pymisp/data/misp-objects/relationships/definition.json

17
README.md
View File

@ -50,6 +50,23 @@ cd examples
python3 last.py -l 10
```
## Debugging
You have two options there:
1. Pass `debug=True` to `PyMISP` and it will enable logging.DEBUG to stderr on the whole module
2. Use the python logging module directly:
```python
import logging
logger = logging.getLogger('pymisp')
# Configure it as you whish, for example, enable DEBUG mode:
logger.setLevel(logging.DEBUG)
```
## Documentation
[PyMISP API documentation is available](https://media.readthedocs.org/pdf/pymisp/master/pymisp.pdf).

33
examples/add_generic_object.py Normal file
View File

@ -0,0 +1,33 @@
import json
from pymisp import PyMISP
from pymisp.tools.abstractgenerator import AbstractMISPObjectGenerator
from keys import misp_url, misp_key, misp_verifycert
import argparse
class GenericObject(AbstractMISPObjectGenerator):
def __init__(self, type, data_dict):
super(GenericObject, self).__init__(type)
self.__data = data_dict
self.generate_attributes()
def generate_attributes(self):
for key, value in self.__data.items():
self.add_attribute(key, value=value)
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Create a MISP Object selectable by type starting from a dictionary')
parser.add_argument("-e", "--event", required=True, help="Event ID to update")
parser.add_argument("-t", "--type", required=True, help="Type of the generic object")
parser.add_argument("-d", "--dict", required=True, help="Dict ")
args = parser.parse_args()
pymisp = PyMISP(misp_url, misp_key, misp_verifycert)
try:
template_id = [x['ObjectTemplate']['id'] for x in pymisp.get_object_templates_list() if x['ObjectTemplate']['name'] == args.type][0]
except IndexError:
valid_types = ", ".join([x['ObjectTemplate']['name'] for x in pymisp.get_object_templates_list()])
print ("Template for type %s not found! Valid types are: %s" % (args.type, valid_types))
exit()
misp_object = GenericObject(args.type.replace("|", "-"), json.loads(args.dict))
r = pymisp.add_object(args.event, template_id, misp_object)

114
examples/asciidoc_generator.py Executable file
View File

@ -0,0 +1,114 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import argparse
from datetime import date
import importlib
from pymisp import MISPEvent
from defang import defang
from pytaxonomies import Taxonomies
class ReportGenerator():
def __init__(self, profile="daily_report"):
self.taxonomies = Taxonomies()
self.report = ''
profile_name = "profiles.{}".format(profile)
self.template = importlib.import_module(name=profile_name)
def from_remote(self, event_id):
from pymisp import PyMISP
from keys import misp_url, misp_key, misp_verifycert
misp = PyMISP(misp_url, misp_key, misp_verifycert)
result = misp.get(event_id)
self.misp_event = MISPEvent()
self.misp_event.load(result)
def from_file(self, path):
self.misp_event = MISPEvent()
self.misp_event.load_file(path)
def attributes(self):
if not self.misp_event.attributes:
return ''
list_attributes = []
for attribute in self.misp_event.attributes:
if attribute.type in self.template.types_to_attach:
list_attributes.append("* {}".format(defang(attribute.value)))
for obj in self.misp_event.Object:
if obj.name in self.template.objects_to_attach:
for attribute in obj.Attribute:
if attribute.type in self.template.types_to_attach:
list_attributes.append("* {}".format(defang(attribute.value)))
return self.template.attributes.format(list_attributes="\n".join(list_attributes))
def _get_tag_info(self, machinetag):
return self.taxonomies.revert_machinetag(machinetag)
def report_headers(self):
content = {'org_name': 'name',
'date': date.today().isoformat()}
self.report += self.template.headers.format(**content)
def event_level_tags(self):
if not self.misp_event.Tag:
return ''
for tag in self.misp_event.Tag:
# Only look for TLP for now
if tag['name'].startswith('tlp'):
tax, predicate = self._get_tag_info(tag['name'])
return self.template.event_level_tags.format(value=predicate.predicate.upper(), expanded=predicate.expanded)
def title(self):
internal_id = ''
summary = ''
# Get internal refs for report
for obj in self.misp_event.Object:
if obj.name != 'report':
continue
for a in obj.Attribute:
if a.object_relation == 'case-number':
internal_id = a.value
if a.object_relation == 'summary':
summary = a.value
return self.template.title.format(internal_id=internal_id, title=self.misp_event.info,
summary=summary)
def asciidoc(self, lang='en'):
self.report += self.title()
self.report += self.event_level_tags()
self.report += self.attributes()
if __name__ == '__main__':
try:
parser = argparse.ArgumentParser(description='Create a human-readable report out of a MISP event')
parser.add_argument("--profile", default="daily_report", help="Profile template to use")
parser.add_argument("-o", "--output", help="Output file to write to (generally ends in .adoc)")
group = parser.add_mutually_exclusive_group(required=True)
group.add_argument("-e", "--event", default=[], nargs='+', help="Event ID to get.")
group.add_argument("-p", "--path", default=[], nargs='+', help="Path to the JSON dump.")
args = parser.parse_args()
report = ReportGenerator(args.profile)
report.report_headers()
if args.event:
for eid in args.event:
report.from_remote(eid)
report.asciidoc()
else:
for f in args.path:
report.from_file(f)
report.asciidoc()
if args.output:
with open(args.output, "w") as ofile:
ofile.write(report.report)
else:
print(report.report)
except ModuleNotFoundError as err:
print(err)

0
examples/profiles/__init__.py Normal file
View File

37
examples/profiles/daily_report.py Normal file
View File

@ -0,0 +1,37 @@
types_to_attach = ['ip-dst', 'url', 'domain']
objects_to_attach = ['domain-ip']
headers = """
:toc: right
:toclevels: 1
:toc-title: Daily Report
:icons: font
:sectanchors:
:sectlinks:
= Daily report by {org_name}
{date}
:icons: font
"""
event_level_tags = """
IMPORTANT: This event is classified TLP:{value}.
{expanded}
"""
attributes = """
=== Indicator(s) of compromise
{list_attributes}
"""
title = """
== ({internal_id}) {title}
{summary}
"""

33
examples/profiles/weekly_report.py Normal file
View File

@ -0,0 +1,33 @@
types_to_attach = ['ip-dst', 'url', 'domain', 'md5']
objects_to_attach = ['domain-ip', 'file']
headers = """
:toc: right
:toclevels: 1
:toc-title: Weekly Report
:icons: font
:sectanchors:
:sectlinks:
= Weekly report by {org_name}
{date}
:icons: font
"""
event_level_tags = """
"""
attributes = """
=== Indicator(s) of compromise
{list_attributes}
"""
title = """
== ({internal_id}) {title}
{summary}
"""

182
examples/vt_to_misp.py Normal file
View File

@ -0,0 +1,182 @@
''' Convert a VirusTotal report into MISP objects '''
import argparse
import json
import logging
from datetime import datetime
from urllib.parse import urlsplit
import pymisp
from pymisp.tools import VTReportObject
logging.basicConfig(level=logging.INFO, format="%(asctime)s | %(levelname)s | %(module)s.%(funcName)s.%(lineno)d | %(message)s")
def build_cli():
'''
Build the command-line arguments
'''
desc = "Take an indicator or list of indicators to search VT for and import the results into MISP"
post_desc = """
config.json: Should be a JSON file containing MISP and VirusTotal credentials with the following format:
{"misp": {"url": "<url_to_misp>", "key": "<misp_api_key>"}, "virustotal": {"key": "<vt_api_key>"}}
Please note: Only public API features work in the VTReportObject for now. I don't have a quarter million to spare ;)
Example:
python vt_to_misp.py -i 719c97a8cd8db282586c1416894dcaf8 -c ./config.json
"""
parser = argparse.ArgumentParser(description=desc, epilog=post_desc, formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument("-e", "--event", help="MISP event id to add to")
parser.add_argument("-c", "--config", default="config.json", help="Path to JSON configuration file to read")
indicators = parser.add_mutually_exclusive_group(required=True)
indicators.add_argument("-i", "--indicator", help="Single indicator to look up")
indicators.add_argument("-f", "--file", help="File of indicators to look up - one on each line")
indicators.add_argument("-l", "--link", help="Link to a VirusTotal report")
return parser.parse_args()
def build_config(path=None):
'''
Read a configuration file path. File is expected to be
:path: Path to a configuration file
'''
try:
with open(path, "r") as ifile:
return json.load(ifile)
except OSError:
raise OSError("Couldn't find path to configuration file: %s", path)
except json.JSONDecodeError:
raise IOError("Couldn't parse configuration file. Please make sure it is a proper JSON document")
def generate_report(indicator, apikey):
'''
Build our VirusTotal report object, File object, and AV signature objects
and link them appropriately
:indicator: Indicator hash to search in VT for
'''
report_objects = []
vt_report = VTReportObject(apikey, indicator)
report_objects.append(vt_report)
raw_report = vt_report._report
if vt_report._resource_type == "file":
file_object = pymisp.MISPObject(name="file")
file_object.add_attribute("md5", value=raw_report["md5"])
file_object.add_attribute("sha1", value=raw_report["sha1"])
file_object.add_attribute("sha256", value=raw_report["sha256"])
vt_report.add_reference(referenced_uuid=file_object.uuid, relationship_type="report of")
report_objects.append(file_object)
elif vt_report._resource_type == "url":
parsed = urlsplit(indicator)
url_object = pymisp.MISPObject(name="url")
url_object.add_attribute("url", value=parsed.geturl())
url_object.add_attribute("host", value=parsed.hostname)
url_object.add_attribute("scheme", value=parsed.scheme)
url_object.add_attribute("port", value=parsed.port)
vt_report.add_reference(referenced_uuid=url_object.uuid, relationship_type="report of")
report_objects.append(url_object)
for antivirus in raw_report["scans"]:
if raw_report["scans"][antivirus]["detected"]:
av_object = pymisp.MISPObject(name="av-signature")
av_object.add_attribute("software", value=antivirus)
signature_name = raw_report["scans"][antivirus]["result"]
av_object.add_attribute("signature", value=signature_name, disable_correlation=True)
vt_report.add_reference(referenced_uuid=av_object.uuid, relationship_type="included-in")
report_objects.append(av_object)
return report_objects
def get_misp_event(event_id=None, info=None):
'''
Smaller helper function for generating a new MISP event or using a preexisting one
:event_id: The event id of the MISP event to upload objects to
:info: The event's title/info
'''
if event_id:
event = misp.get_event(event_id)
elif info:
event = misp.new_event(info=info)
else:
event = misp.new_event(info="VirusTotal Report")
misp_event = pymisp.MISPEvent()
misp_event.load(event)
return misp_event
def main(misp, config, args):
'''
Main program logic
:misp: PyMISP API object for interfacing with MISP
:config: Configuration dictionary
:args: Argparse CLI object
'''
if args.indicator:
misp_objects = generate_report(args.indicator, config["virustotal"]["key"])
if misp_objects:
misp_event = get_misp_event(args.event, "VirusTotal Report for {}".format(args.indicator))
submit_to_misp(misp, misp_event, misp_objects)
elif args.file:
try:
reports = []
with open(args.file, "r") as ifile:
for indicator in ifile:
try:
misp_objects = generate_report(indicator, config["virustotal"]["key"])
if misp_objects:
reports.append(misp_objects)
except pymisp.exceptions.InvalidMISPObject as err:
logging.error(err)
if reports:
current_time = datetime.now().strftime("%x %X")
misp_event = get_misp_event(args.event, "VirusTotal Reports: {}".format(current_time))
for report in reports:
submit_to_misp(misp, misp_event, report)
except OSError:
logging.error("Couldn't open indicators file at '%s'. Check path", args.file)
elif args.link:
# https://www.virustotal.com/#/file/<ioc>/detection
indicator = args.link.split("/")[5]
misp_objects = generate_report(indicator, config["virustotal"]["key"])
if misp_objects:
misp_event = get_misp_event(args.event, "VirusTotal Report for {}".format(indicator))
submit_to_misp(misp, misp_event, misp_objects)
def submit_to_misp(misp, misp_event, misp_objects):
'''
Submit a list of MISP objects to a MISP event
:misp: PyMISP API object for interfacing with MISP
:misp_event: MISPEvent object
:misp_objects: List of MISPObject objects. Must be a list
'''
# go through round one and only add MISP objects
for misp_object in misp_objects:
template_id = misp.get_object_template_id(misp_object.template_uuid)
misp.add_object(misp_event.id, template_id, misp_object)
# go through round two and add all the object references for each object
for misp_object in misp_objects:
for reference in misp_object.ObjectReference:
misp.add_object_reference(reference)
if __name__ == "__main__":
try:
args = build_cli()
config = build_config(args.config)
# change the 'ssl' value if you want to verify your MISP's SSL instance
misp = pymisp.PyMISP(url=config["misp"]["url"], key=config["misp"]["key"], ssl=False)
# finally, let's start checking VT and converting the reports
main(misp, config, args)
except KeyboardInterrupt:
print("Bye Felicia")
except pymisp.exceptions.InvalidMISPObject as err:
logging.error(err)

12
pymisp/__init__.py
View File

@ -1,4 +1,9 @@
__version__ = '2.4.81'
__version__ = '2.4.81.2'
import sys
import logging
logger = logging.getLogger(__name__)
FORMAT = "%(levelname)s [%(filename)s:%(lineno)s - %(funcName)s() ] %(message)s"
logging.basicConfig(stream=sys.stderr, level=logging.WARNING, format=FORMAT)
try:
from .exceptions import PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey, InvalidMISPObject, UnknownMISPObjectTemplate # noqa
@ -9,5 +14,6 @@ try:
from .tools import Neo4j # noqa
from .tools import stix # noqa
from .tools import openioc # noqa
except ImportError:
pass
logger.debug('pymisp loaded properly')
except ImportError as e:
logger.warning('Unable to load pymisp properly: {}'.format(e))

8
pymisp/abstract.py
View File

@ -6,10 +6,12 @@ import json
from json import JSONEncoder
import collections
import six # Remove that import when discarding python2 support.
import logging
logger = logging.getLogger('pymisp')
if six.PY2:
import warnings
warnings.warn("You're using python 2, it is strongly recommended to use python >=3.5")
logger.warning("You're using python 2, it is strongly recommended to use python >=3.5")
class MISPEncode(JSONEncoder):
@ -62,7 +64,7 @@ class AbstractMISP(collections.MutableMapping):
return self.to_dict()
def to_json(self):
return json.dumps(self.to_dict(), cls=MISPEncode)
return json.dumps(self, cls=MISPEncode)
def __getitem__(self, key):
return getattr(self, key)

512
pymisp/api.py
View File

File diff suppressed because it is too large Load Diff

3
pymisp/data/describeTypes.json
View File

@ -805,8 +805,6 @@
"vulnerability",
"x509-fingerprint-sha1",
"other",
"ip-dst|port",
"ip-src|port",
"hostname|port",
"email-dst-display-name",
"email-src-display-name",
@ -1023,7 +1021,6 @@
"text",
"attachment",
"comment",
"text",
"other",
"hex"
],

2
pymisp/data/misp-objects

@ -1 +1 @@
Subproject commit bc5795dc189bcf9cd001527656fe10e6ed10dc5f
Subproject commit 6b43b68651a350a26891080ef0feda364b74727a

1
pymisp/exceptions.py
View File

@ -40,6 +40,7 @@ class InvalidMISPObject(MISPObjectException):
"""Exception raised when an object doesn't respect the contrains in the definition"""
pass
class UnknownMISPObjectTemplate(MISPObjectException):
"""Exception raised when the template is unknown"""
pass

70
pymisp/mispevent.py
View File

@ -16,12 +16,14 @@ from collections import Counter
from .abstract import AbstractMISP
from .exceptions import UnknownMISPObjectTemplate, InvalidMISPObject, PyMISPError, NewEventError, NewAttributeError
import six # Remove that import when discarding python2 support.
import logging
logger = logging.getLogger('pymisp')
if six.PY2:
import warnings
warnings.warn("You're using python 2, it is strongly recommended to use python >=3.5")
logger.warning("You're using python 2, it is strongly recommended to use python >=3.5")
try:
from dateutil.parser import parse
@ -138,7 +140,7 @@ class MISPAttribute(AbstractMISP):
try:
c.verify(signed_data, signature=base64.b64decode(self.sig), verify=keys[:1])
return {self.uuid: True}
except:
except Exception:
return {self.uuid: False}
def set_all_values(self, **kwargs):
@ -165,15 +167,21 @@ class MISPAttribute(AbstractMISP):
# Default values
self.category = kwargs.pop('category', type_defaults['default_category'])
if self.category is None:
# In case the category key is passed, but None
self.category = type_defaults['default_category']
if self.category not in self.__categories:
raise NewAttributeError('{} is invalid, category has to be in {}'.format(self.category, (', '.join(self.__categories))))
self.to_ids = kwargs.pop('to_ids', bool(int(type_defaults['to_ids'])))
if self.to_ids is None:
self.to_ids = bool(int(type_defaults['to_ids']))
if not isinstance(self.to_ids, bool):
raise NewAttributeError('{} is invalid, to_ids has to be True or False'.format(self.to_ids))
if kwargs.get('distribution') is not None:
self.distribution = int(kwargs.pop('distribution'))
self.distribution = kwargs.pop('distribution', None)
if self.distribution is not None:
self.distribution = int(self.distribution)
if self.distribution not in [0, 1, 2, 3, 4, 5]:
raise NewAttributeError('{} is invalid, the distribution has to be in 0, 1, 2, 3, 4, 5'.format(self.distribution))
@ -215,20 +223,37 @@ class MISPAttribute(AbstractMISP):
self._malware_binary = self.data
self.encrypt = True
def __is_misp_encrypted_file(self, f):
files_list = f.namelist()
if len(files_list) != 2:
return False
md5_from_filename = ''
md5_from_file = ''
for name in files_list:
if name.endswith('.filename.txt'):
md5_from_filename = name.replace('.filename.txt', '')
else:
md5_from_file = name
if not md5_from_filename or not md5_from_file or md5_from_filename != md5_from_file:
return False
return True
def _load_data(self):
if not isinstance(self.data, BytesIO):
self.data = BytesIO(base64.b64decode(self.data))
if self.type == 'malware-sample':
try:
with ZipFile(self.data) as f:
if not self.__is_misp_encrypted_file(f):
raise Exception('Not an existing malware sample')
for name in f.namelist():
if name.endswith('.txt'):
if name.endswith('.filename.txt'):
with f.open(name, pwd=b'infected') as unpacked:
self.malware_filename = unpacked.read().decode()
self.malware_filename = unpacked.read().decode().strip()
else:
with f.open(name, pwd=b'infected') as unpacked:
self._malware_binary = BytesIO(unpacked.read())
except:
except Exception:
# not a encrypted zip file, assuming it is a new malware sample
self._prepare_new_malware_sample()
@ -359,7 +384,7 @@ class MISPEvent(AbstractMISP):
try:
c.verify(signed_data, signature=base64.b64decode(self.sig), verify=keys[:1])
to_return[self.uuid] = True
except:
except Exception:
to_return[self.uuid] = False
for a in self.attributes:
to_return.update(a.verify(gpg_uid))
@ -369,7 +394,7 @@ class MISPEvent(AbstractMISP):
try:
c.verify(to_verify_global, signature=base64.b64decode(self.global_sig), verify=keys[:1])
to_return['global'] = True
except:
except Exception:
to_return['global'] = False
return to_return
@ -422,8 +447,9 @@ class MISPEvent(AbstractMISP):
raise NewAttributeError('The info field of the new event is required.')
# Default values for a valid event to send to a MISP instance
if kwargs.get('distribution') is not None:
self.distribution = int(kwargs.pop('distribution'))
self.distribution = kwargs.pop('distribution', None)
if self.distribution is not None:
self.distribution = int(self.distribution)
if self.distribution not in [0, 1, 2, 3, 4]:
raise NewAttributeError('{} is invalid, the distribution has to be in 0, 1, 2, 3, 4'.format(self.distribution))
@ -565,7 +591,7 @@ class MISPObjectReference(AbstractMISP):
self.referenced_uuid = referenced_uuid
self.relationship_type = relationship_type
self.comment = comment
for k, v in kwargs:
for k, v in kwargs.items():
setattr(self, k, v)
@ -598,7 +624,7 @@ class MISPObjectAttribute(MISPAttribute):
class MISPObject(AbstractMISP):
def __init__(self, name, strict=True):
def __init__(self, name, strict=False):
super(MISPObject, self).__init__()
self.__strict = strict
self.name = name
@ -634,7 +660,7 @@ class MISPObject(AbstractMISP):
else:
self.__known_template = False
if kwargs.get('template_version') and int(kwargs['template_version']) != self.template_version:
if self.strict:
if self.__strict:
raise UnknownMISPObjectTemplate('Version of the object ({}) is different from the one of the template ({}).'.format(kwargs['template_version'], self.template_version))
else:
self.__known_template = False
@ -649,12 +675,12 @@ class MISPObject(AbstractMISP):
else:
setattr(self, key, value)
def to_dict(self, strict=True):
def to_dict(self, strict=False):
if strict or self.__strict and self.__known_template:
self._validate()
return super(MISPObject, self).to_dict()
def to_json(self, strict=True):
def to_json(self, strict=False):
if strict or self.__strict and self.__known_template:
self._validate()
return super(MISPObject, self).to_json()
@ -684,7 +710,7 @@ class MISPObject(AbstractMISP):
"""Add a link (uuid) to an other object"""
if kwargs.get('object_uuid'):
# Load existing object
object_uuid = kwargs.get('object_uuid')
object_uuid = kwargs.pop('object_uuid')
else:
# New reference
object_uuid = self.uuid
@ -697,7 +723,11 @@ class MISPObject(AbstractMISP):
if value.get('value') is None:
return None
if self.__known_template:
attribute = MISPObjectAttribute(self.__definition['attributes'][object_relation])
if self.__definition['attributes'].get(object_relation):
attribute = MISPObjectAttribute(self.__definition['attributes'][object_relation])
else:
# Woopsie, this object_relation is unknown, no sane defaults for you.
attribute = MISPObjectAttribute({})
else:
attribute = MISPObjectAttribute({})
attribute.from_dict(object_relation, **value)

1
pymisp/tools/__init__.py
View File

@ -1,3 +1,4 @@
from .vtreportobject import VTReportObject # noqa
from .neo4j import Neo4j # noqa
from .fileobject import FileObject # noqa
from .peobject import PEObject, PESectionObject # noqa

16
pymisp/tools/create_misp_object.py
View File

@ -3,7 +3,9 @@
from . import FileObject, PEObject, ELFObject, MachOObject
from ..exceptions import MISPObjectException
import warnings
import logging
logger = logging.getLogger('pymisp')
try:
import lief
@ -57,15 +59,15 @@ def make_binary_objects(filepath=None, pseudofile=None, filename=None):
elif isinstance(lief_parsed, lief.MachO.Binary):
return make_macho_objects(lief_parsed, misp_file)
except lief.bad_format as e:
warnings.warn('\tBad format: {}'.format(e))
logger.warning('Bad format: {}'.format(e))
except lief.bad_file as e:
warnings.warn('\tBad file: {}'.format(e))
logger.warning('Bad file: {}'.format(e))
except lief.parser_error as e:
warnings.warn('\tParser error: {}'.format(e))
logger.warning('Parser error: {}'.format(e))
except FileTypeNotImplemented as e: # noqa
warnings.warn(e)
logger.warning(e)
if not HAS_LIEF:
warnings.warn('Please install lief, documentation here: https://github.com/lief-project/LIEF')
logger.warning('Please install lief, documentation here: https://github.com/lief-project/LIEF')
if not filepath:
warnings.warn('LIEF currently requires a filepath and not a pseudo file')
logger.warning('LIEF currently requires a filepath and not a pseudo file')
return misp_file, None, None

5
pymisp/tools/elfobject.py
View File

@ -5,8 +5,9 @@ from .abstractgenerator import AbstractMISPObjectGenerator
from ..exceptions import InvalidMISPObject
from io import BytesIO
from hashlib import md5, sha1, sha256, sha512
import warnings
import logging
logger = logging.getLogger('pymisp')
try:
import lief
@ -25,7 +26,7 @@ class ELFObject(AbstractMISPObjectGenerator):
def __init__(self, parsed=None, filepath=None, pseudofile=None):
if not HAS_PYDEEP:
warnings.warn("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git")
logger.warning("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git")
if not HAS_LIEF:
raise ImportError('Please install lief, documentation here: https://github.com/lief-project/LIEF')
if pseudofile:

9
pymisp/tools/fileobject.py
View File

@ -8,7 +8,10 @@ from io import BytesIO
from hashlib import md5, sha1, sha256, sha512
import math
from collections import Counter
import warnings
import logging
logger = logging.getLogger('pymisp')
try:
import pydeep
@ -27,9 +30,9 @@ class FileObject(AbstractMISPObjectGenerator):
def __init__(self, filepath=None, pseudofile=None, filename=None):
if not HAS_PYDEEP:
warnings.warn("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git")
logger.warning("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git")
if not HAS_MAGIC:
warnings.warn("Please install python-magic: pip install python-magic.")
logger.warning("Please install python-magic: pip install python-magic.")
if filename:
# Useful in case the file is copied with a pre-defined name by a script but we want to keep the original name
self.__filename = filename

6
pymisp/tools/machoobject.py
View File

@ -5,7 +5,9 @@ from ..exceptions import InvalidMISPObject
from .abstractgenerator import AbstractMISPObjectGenerator
from io import BytesIO
from hashlib import md5, sha1, sha256, sha512
import warnings
import logging
logger = logging.getLogger('pymisp')
try:
@ -25,7 +27,7 @@ class MachOObject(AbstractMISPObjectGenerator):
def __init__(self, parsed=None, filepath=None, pseudofile=None):
if not HAS_PYDEEP:
warnings.warn("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git")
logger.warning("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git")
if not HAS_LIEF:
raise ImportError('Please install lief, documentation here: https://github.com/lief-project/LIEF')
if pseudofile:

1
pymisp/tools/openioc.py
View File

@ -11,7 +11,6 @@ except ImportError:
has_bs4 = False
iocMispMapping = {
# ~ @Link https://wiki.ops.fr/doku.php/manuels:misp:event-guidelines
'CookieHistoryItem/HostName': {'type': 'hostname', 'comment': 'CookieHistory.'},
'DriverItem/DriverName': {'category': 'Artifacts dropped', 'type': 'other', 'comment': 'DriverName.'},

5
pymisp/tools/peobject.py
View File

@ -6,8 +6,9 @@ from .abstractgenerator import AbstractMISPObjectGenerator
from io import BytesIO
from hashlib import md5, sha1, sha256, sha512
from datetime import datetime
import warnings
import logging
logger = logging.getLogger('pymisp')
try:
import lief
@ -26,7 +27,7 @@ class PEObject(AbstractMISPObjectGenerator):
def __init__(self, parsed=None, filepath=None, pseudofile=None):
if not HAS_PYDEEP:
warnings.warn("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git")
logger.warning("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git")
if not HAS_LIEF:
raise ImportError('Please install lief, documentation here: https://github.com/lief-project/LIEF')
if pseudofile:

81
pymisp/tools/vtreportobject.py Normal file
View File

@ -0,0 +1,81 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import re
import requests
try:
import validators
has_validators = True
except ImportError:
has_validators = False
from .abstractgenerator import AbstractMISPObjectGenerator
from .. import InvalidMISPObject
class VTReportObject(AbstractMISPObjectGenerator):
'''
VirusTotal Report
:apikey: VirusTotal API key (private works, but only public features are supported right now)
:indicator: IOC to search VirusTotal for
'''
def __init__(self, apikey, indicator):
# PY3 way:
# super().__init__("virustotal-report")
super(VTReportObject, self).__init__("virustotal-report")
indicator = indicator.strip()
self._resource_type = self.__validate_resource(indicator)
if self._resource_type:
self._report = self.__query_virustotal(apikey, indicator)
self.generate_attributes()
else:
error_msg = "A valid indicator is required. (One of type url, md5, sha1, sha256). Received '{}' instead".format(indicator)
raise InvalidMISPObject(error_msg)
# Mark as non_jsonable because we need to add the references manually after the object(s) have been created
self.update_not_jsonable('ObjectReference')
def generate_attributes(self):
''' Parse the VirusTotal report for relevant attributes '''
self.add_attribute("last-submission", value=self._report["scan_date"])
self.add_attribute("permalink", value=self._report["permalink"])
ratio = "{}/{}".format(self._report["positives"], self._report["total"])
self.add_attribute("detection-ratio", value=ratio)
def __validate_resource(self, ioc):
'''
Validate the data type of an indicator.
Domains and IP addresses aren't supported because
they don't return the same type of data as the URLs/files do
:ioc: Indicator to search VirusTotal for
'''
if not has_validators:
raise Exception('You need to install validators: pip install validators')
if validators.url(ioc):
return "url"
elif re.match(r"\b([a-fA-F0-9]{32}|[a-fA-F0-9]{40}|[a-fA-F0-9]{64})\b", ioc):
return "file"
return False
def __query_virustotal(self, apikey, resource):
'''
Query VirusTotal for information about an indicator
:apikey: VirusTotal API key
:resource: Indicator to search in VirusTotal
'''
url = "https://www.virustotal.com/vtapi/v2/{}/report".format(self._resource_type)
params = {"apikey": apikey, "resource": resource}
# for now assume we're using a public API key - we'll figure out private keys later
report = requests.get(url, params=params)
report = report.json()
if report["response_code"] == 1:
return report
else:
error_msg = "{}: {}".format(resource, report["verbose_msg"])
raise InvalidMISPObject(error_msg)

9
setup.py
View File

@ -27,7 +27,11 @@ setup(
'Topic :: Internet',
],
test_suite="tests.test_offline",
install_requires=['six', 'requests', 'python-dateutil', 'jsonschema'],
install_requires=['six', 'requests', 'python-dateutil', 'jsonschema', 'setuptools>=36.4'],
extras_require={'fileobjects': ['lief>=0.8', 'python-magic'],
'neo': ['py2neo'],
'openioc': ['beautifulsoup4'],
'virustotal': ['validators']},
tests_require=[
'jsonschema',
'python-dateutil',
@ -36,7 +40,8 @@ setup(
'six'
],
include_package_data=True,
package_data={'pymisp': ['data/*.json', 'data/misp-objects/schema_objects.json',
package_data={'pymisp': ['data/*.json',
'data/misp-objects/schema_objects.json',
'data/misp-objects/schema_relationships.json',
'data/misp-objects/objects/*/definition.json',
'data/misp-objects/relationships/definition.json']},

2
tests/test_offline.py
View File

@ -47,7 +47,7 @@ class TestOffline(unittest.TestCase):
m.register_uri('POST', self.domain + 'events/5758ebf5-c898-48e6-9fe9-5665c0a83866', json=self.event)
m.register_uri('DELETE', self.domain + 'events/2', json={'message': 'Event deleted.'})
m.register_uri('DELETE', self.domain + 'events/3', json={'errors': ['Invalid event'], 'message': 'Invalid event', 'name': 'Invalid event', 'url': '/events/3'})
m.register_uri('DELETE', self.domain + 'attributes/2', json={'message': 'Attribute deleted.'})
m.register_uri('GET', self.domain + 'attributes/delete/2', json={'message': 'Attribute deleted.'})
m.register_uri('POST', self.domain + 'events/index', json=self.search_index_result)
def test_getEvent(self, m):