Merge remote-tracking branch 'upstream/master'

.travis.yml

@@ -17,14 +17,10 @@ python:
     - "3.6-dev"
 
 install:
-    - pip install -U nose
+    - pip install -U nose pip setuptools
-    - pip install coveralls
+    - pip install coveralls codecov requests-mock
-    - pip install codecov
-    - pip install requests-mock
-    - pip install https://github.com/lief-project/packages/raw/lief-master-latest/pylief-0.7.0.dev.zip
     - pip install git+https://github.com/kbandla/pydeep.git
-    - pip install python-magic
+    - pip install .[fileobjects,neo,openioc,virustotal]
-    - pip install .
     - pushd tests
     - git clone https://github.com/viper-framework/viper-test-files.git
     - popd

CHANGELOG.txt

@@ -2,6 +2,61 @@ Changelog
 =========
 
 
+v2.4.81.2 (2017-10-24)
+----------------------
+
+Changes
+~~~~~~~
+- Version bump. [Raphaël Vinot]
+- Update changelog. [Raphaël Vinot]
+
+Fix
+~~~
+- Properly bundle object templates. [Raphaël Vinot]
+
+
+v2.4.81.1 (2017-10-24)
+----------------------
+
+Changes
+~~~~~~~
+- Bump version. [Raphaël Vinot]
+- Do not raise an exception when the object template is unknown.
+  [Raphaël Vinot]
+
+  + bump misp-object
+- Bump misp-objects. [Raphaël Vinot]
+- Allow to hard delete an attribute by ID. [Raphaël Vinot]
+- Update comments. [Raphaël Vinot]
+- Bump misp-objects and describeTypes. [Raphaël Vinot]
+
+Fix
+~~~
+- Properly bundle object templates. [Raphaël Vinot]
+- Fix typos and logic mistakes in mispevent. [Raphaël Vinot]
+- Fix travis build. [Raphaël Vinot]
+- Min required version of setuptools. [Raphaël Vinot]
+- Improve dependencies listing. [Raphaël Vinot]
+
+  Partial fix for #110
+- Missing default category. [Raphaël Vinot]
+
+  Fix #119
+
+Other
+~~~~~
+- Merge branch 'master' of github.com:MISP/PyMISP. [Raphaël Vinot]
+- Update openioc.py. [Andras Iklody]
+- Merge branch 'master' of github.com:MISP/PyMISP. [Raphaël Vinot]
+- Merge pull request #121 from kx499/master. [Raphaël Vinot]
+
+  Added **kwargs to add_named_attribute call in add_attachment
+- Added **kwargs to add_named_attribute call in add_attachment.
+  [tssbo82]
+- Update README. [Raphaël Vinot]
+- Update changelog. [Raphaël Vinot]
+
+
 v2.4.81 (2017-10-09)
 --------------------
 

MANIFEST.in

@@ -1 +1,4 @@
-include pymisp/data/*
+include pymisp/data/*.json
+include pymisp/data/misp-objects/*.json
+include pymisp/data/misp-objects/objects/*/definition.json
+include pymisp/data/misp-objects/relationships/definition.json

README.md

@@ -50,6 +50,23 @@ cd examples
 python3 last.py -l 10
 ```
 
+## Debugging
+
+You have two options there:
+
+1. Pass `debug=True` to `PyMISP` and it will enable logging.DEBUG to stderr on the whole module
+
+2. Use the python logging module directly:
+
+```python
+
+import logging
+logger = logging.getLogger('pymisp')
+
+# Configure it as you whish, for example, enable DEBUG mode:
+logger.setLevel(logging.DEBUG)
+```
+
 ## Documentation
 
 [PyMISP API documentation is available](https://media.readthedocs.org/pdf/pymisp/master/pymisp.pdf).

examples/add_generic_object.py

@@ -0,0 +1,33 @@
+import json
+from pymisp import PyMISP
+from pymisp.tools.abstractgenerator import AbstractMISPObjectGenerator
+from keys import misp_url, misp_key, misp_verifycert
+import argparse
+
+class GenericObject(AbstractMISPObjectGenerator):
+    def __init__(self, type, data_dict):
+        super(GenericObject, self).__init__(type)
+        self.__data = data_dict
+        self.generate_attributes()
+
+    def generate_attributes(self):
+        for key, value in self.__data.items():
+            self.add_attribute(key, value=value)
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Create a MISP Object selectable by type starting from a dictionary')
+    parser.add_argument("-e", "--event", required=True, help="Event ID to update")
+    parser.add_argument("-t", "--type", required=True, help="Type of the generic object")
+    parser.add_argument("-d", "--dict", required=True, help="Dict ")
+    args = parser.parse_args()
+
+    pymisp = PyMISP(misp_url, misp_key, misp_verifycert)
+    try:
+        template_id = [x['ObjectTemplate']['id'] for x in pymisp.get_object_templates_list() if x['ObjectTemplate']['name'] == args.type][0]
+    except IndexError:
+        valid_types = ", ".join([x['ObjectTemplate']['name'] for x in pymisp.get_object_templates_list()])
+        print ("Template for type %s not found! Valid types are: %s" % (args.type, valid_types))
+        exit()
+
+    misp_object = GenericObject(args.type.replace("|", "-"), json.loads(args.dict))
+    r = pymisp.add_object(args.event, template_id, misp_object)

examples/asciidoc_generator.py

@@ -0,0 +1,114 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import argparse
+from datetime import date
+import importlib
+
+from pymisp import MISPEvent
+from defang import defang
+from pytaxonomies import Taxonomies
+
+
+class ReportGenerator():
+    def __init__(self, profile="daily_report"):
+        self.taxonomies = Taxonomies()
+        self.report = ''
+        profile_name = "profiles.{}".format(profile)
+        self.template = importlib.import_module(name=profile_name)
+
+    def from_remote(self, event_id):
+        from pymisp import PyMISP
+        from keys import misp_url, misp_key, misp_verifycert
+        misp = PyMISP(misp_url, misp_key, misp_verifycert)
+        result = misp.get(event_id)
+        self.misp_event = MISPEvent()
+        self.misp_event.load(result)
+
+    def from_file(self, path):
+        self.misp_event = MISPEvent()
+        self.misp_event.load_file(path)
+
+    def attributes(self):
+        if not self.misp_event.attributes:
+            return ''
+        list_attributes = []
+        for attribute in self.misp_event.attributes:
+            if attribute.type in self.template.types_to_attach:
+                list_attributes.append("* {}".format(defang(attribute.value)))
+        for obj in self.misp_event.Object:
+            if obj.name in self.template.objects_to_attach:
+                for attribute in obj.Attribute:
+                    if attribute.type in self.template.types_to_attach:
+                        list_attributes.append("* {}".format(defang(attribute.value)))
+        return self.template.attributes.format(list_attributes="\n".join(list_attributes))
+
+    def _get_tag_info(self, machinetag):
+        return self.taxonomies.revert_machinetag(machinetag)
+
+    def report_headers(self):
+        content = {'org_name': 'name',
+                   'date': date.today().isoformat()}
+        self.report += self.template.headers.format(**content)
+
+    def event_level_tags(self):
+        if not self.misp_event.Tag:
+            return ''
+        for tag in self.misp_event.Tag:
+            # Only look for TLP for now
+            if tag['name'].startswith('tlp'):
+                tax, predicate = self._get_tag_info(tag['name'])
+                return self.template.event_level_tags.format(value=predicate.predicate.upper(), expanded=predicate.expanded)
+
+    def title(self):
+        internal_id = ''
+        summary = ''
+        # Get internal refs for report
+        for obj in self.misp_event.Object:
+            if obj.name != 'report':
+                continue
+            for a in obj.Attribute:
+                if a.object_relation == 'case-number':
+                    internal_id = a.value
+                if a.object_relation == 'summary':
+                    summary = a.value
+
+        return self.template.title.format(internal_id=internal_id, title=self.misp_event.info,
+                                          summary=summary)
+
+    def asciidoc(self, lang='en'):
+        self.report += self.title()
+        self.report += self.event_level_tags()
+        self.report += self.attributes()
+
+
+if __name__ == '__main__':
+    try:
+        parser = argparse.ArgumentParser(description='Create a human-readable report out of a MISP event')
+        parser.add_argument("--profile", default="daily_report", help="Profile template to use")
+        parser.add_argument("-o", "--output", help="Output file to write to (generally ends in .adoc)")
+        group = parser.add_mutually_exclusive_group(required=True)
+        group.add_argument("-e", "--event", default=[], nargs='+', help="Event ID to get.")
+        group.add_argument("-p", "--path", default=[], nargs='+', help="Path to the JSON dump.")
+
+        args = parser.parse_args()
+
+        report = ReportGenerator(args.profile)
+        report.report_headers()
+
+        if args.event:
+            for eid in args.event:
+                report.from_remote(eid)
+                report.asciidoc()
+        else:
+            for f in args.path:
+                report.from_file(f)
+                report.asciidoc()
+
+        if args.output:
+            with open(args.output, "w") as ofile:
+                ofile.write(report.report)
+        else:
+            print(report.report)
+    except ModuleNotFoundError as err:
+        print(err)

examples/profiles/daily_report.py

@@ -0,0 +1,37 @@
+types_to_attach = ['ip-dst', 'url', 'domain']
+objects_to_attach = ['domain-ip']
+
+headers = """
+:toc: right
+:toclevels: 1
+:toc-title: Daily Report
+:icons: font
+:sectanchors:
+:sectlinks:
+= Daily report by {org_name}
+{date}
+
+:icons: font
+
+"""
+
+event_level_tags = """
+IMPORTANT: This event is classified TLP:{value}.
+
+{expanded}
+
+"""
+
+attributes = """
+=== Indicator(s) of compromise
+
+{list_attributes}
+
+"""
+
+title = """
+== ({internal_id}) {title}
+
+{summary}
+
+"""

examples/profiles/weekly_report.py

@@ -0,0 +1,33 @@
+types_to_attach = ['ip-dst', 'url', 'domain', 'md5']
+objects_to_attach = ['domain-ip', 'file']
+
+headers = """
+:toc: right
+:toclevels: 1
+:toc-title: Weekly Report
+:icons: font
+:sectanchors:
+:sectlinks:
+= Weekly report by {org_name}
+{date}
+
+:icons: font
+
+"""
+
+event_level_tags = """
+"""
+
+attributes = """
+=== Indicator(s) of compromise
+
+{list_attributes}
+
+"""
+
+title = """
+== ({internal_id}) {title}
+
+{summary}
+
+"""

examples/vt_to_misp.py

@@ -0,0 +1,182 @@
+''' Convert a VirusTotal report into MISP objects '''
+import argparse
+import json
+import logging
+from datetime import datetime
+from urllib.parse import urlsplit
+
+import pymisp
+from pymisp.tools import VTReportObject
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s | %(levelname)s | %(module)s.%(funcName)s.%(lineno)d | %(message)s")
+
+
+def build_cli():
+    '''
+    Build the command-line arguments
+    '''
+    desc = "Take an indicator or list of indicators to search VT for and import the results into MISP"
+    post_desc = """
+config.json: Should be a JSON file containing MISP and VirusTotal credentials with the following format:
+{"misp": {"url": "<url_to_misp>", "key": "<misp_api_key>"}, "virustotal": {"key": "<vt_api_key>"}}
+Please note: Only public API features work in the VTReportObject for now. I don't have a quarter million to spare ;)
+
+Example:
+    python vt_to_misp.py -i 719c97a8cd8db282586c1416894dcaf8 -c ./config.json
+    """
+    parser = argparse.ArgumentParser(description=desc, epilog=post_desc, formatter_class=argparse.RawTextHelpFormatter)
+    parser.add_argument("-e", "--event", help="MISP event id to add to")
+    parser.add_argument("-c", "--config", default="config.json", help="Path to JSON configuration file to read")
+    indicators = parser.add_mutually_exclusive_group(required=True)
+    indicators.add_argument("-i", "--indicator", help="Single indicator to look up")
+    indicators.add_argument("-f", "--file", help="File of indicators to look up - one on each line")
+    indicators.add_argument("-l", "--link", help="Link to a VirusTotal report")
+    return parser.parse_args()
+
+
+def build_config(path=None):
+    '''
+    Read a configuration file path. File is expected to be
+
+    :path: Path to a configuration file
+    '''
+    try:
+        with open(path, "r") as ifile:
+            return json.load(ifile)
+    except OSError:
+        raise OSError("Couldn't find path to configuration file: %s", path)
+    except json.JSONDecodeError:
+        raise IOError("Couldn't parse configuration file. Please make sure it is a proper JSON document")
+
+
+def generate_report(indicator, apikey):
+    '''
+    Build our VirusTotal report object, File object, and AV signature objects
+    and link them appropriately
+
+    :indicator: Indicator hash to search in VT for
+    '''
+    report_objects = []
+    vt_report = VTReportObject(apikey, indicator)
+    report_objects.append(vt_report)
+    raw_report = vt_report._report
+    if vt_report._resource_type == "file":
+        file_object = pymisp.MISPObject(name="file")
+        file_object.add_attribute("md5", value=raw_report["md5"])
+        file_object.add_attribute("sha1", value=raw_report["sha1"])
+        file_object.add_attribute("sha256", value=raw_report["sha256"])
+        vt_report.add_reference(referenced_uuid=file_object.uuid, relationship_type="report of")
+        report_objects.append(file_object)
+    elif vt_report._resource_type == "url":
+        parsed = urlsplit(indicator)
+        url_object = pymisp.MISPObject(name="url")
+        url_object.add_attribute("url", value=parsed.geturl())
+        url_object.add_attribute("host", value=parsed.hostname)
+        url_object.add_attribute("scheme", value=parsed.scheme)
+        url_object.add_attribute("port", value=parsed.port)
+        vt_report.add_reference(referenced_uuid=url_object.uuid, relationship_type="report of")
+        report_objects.append(url_object)
+    for antivirus in raw_report["scans"]:
+        if raw_report["scans"][antivirus]["detected"]:
+            av_object = pymisp.MISPObject(name="av-signature")
+            av_object.add_attribute("software", value=antivirus)
+            signature_name = raw_report["scans"][antivirus]["result"]
+            av_object.add_attribute("signature", value=signature_name, disable_correlation=True)
+            vt_report.add_reference(referenced_uuid=av_object.uuid, relationship_type="included-in")
+            report_objects.append(av_object)
+    return report_objects
+
+
+def get_misp_event(event_id=None, info=None):
+    '''
+    Smaller helper function for generating a new MISP event or using a preexisting one
+
+    :event_id: The event id of the MISP event to upload objects to
+
+    :info: The event's title/info
+    '''
+    if event_id:
+        event = misp.get_event(event_id)
+    elif info:
+        event = misp.new_event(info=info)
+    else:
+        event = misp.new_event(info="VirusTotal Report")
+    misp_event = pymisp.MISPEvent()
+    misp_event.load(event)
+    return misp_event
+
+
+def main(misp, config, args):
+    '''
+    Main program logic
+
+    :misp: PyMISP API object for interfacing with MISP
+
+    :config: Configuration dictionary
+
+    :args: Argparse CLI object
+    '''
+    if args.indicator:
+        misp_objects = generate_report(args.indicator, config["virustotal"]["key"])
+        if misp_objects:
+            misp_event = get_misp_event(args.event, "VirusTotal Report for {}".format(args.indicator))
+            submit_to_misp(misp, misp_event, misp_objects)
+    elif args.file:
+        try:
+            reports = []
+            with open(args.file, "r") as ifile:
+                for indicator in ifile:
+                    try:
+                        misp_objects = generate_report(indicator, config["virustotal"]["key"])
+                        if misp_objects:
+                            reports.append(misp_objects)
+                    except pymisp.exceptions.InvalidMISPObject as err:
+                        logging.error(err)
+            if reports:
+                current_time = datetime.now().strftime("%x %X")
+                misp_event = get_misp_event(args.event, "VirusTotal Reports: {}".format(current_time))
+                for report in reports:
+                    submit_to_misp(misp, misp_event, report)
+        except OSError:
+            logging.error("Couldn't open indicators file at '%s'. Check path", args.file)
+    elif args.link:
+        # https://www.virustotal.com/#/file/<ioc>/detection
+        indicator = args.link.split("/")[5]
+        misp_objects = generate_report(indicator, config["virustotal"]["key"])
+        if misp_objects:
+            misp_event = get_misp_event(args.event, "VirusTotal Report for {}".format(indicator))
+            submit_to_misp(misp, misp_event, misp_objects)
+
+
+def submit_to_misp(misp, misp_event, misp_objects):
+    '''
+    Submit a list of MISP objects to a MISP event
+
+    :misp: PyMISP API object for interfacing with MISP
+
+    :misp_event: MISPEvent object
+
+    :misp_objects: List of MISPObject objects. Must be a list
+    '''
+# go through round one and only add MISP objects
+    for misp_object in misp_objects:
+        template_id = misp.get_object_template_id(misp_object.template_uuid)
+        misp.add_object(misp_event.id, template_id, misp_object)
+    # go through round two and add all the object references for each object
+    for misp_object in misp_objects:
+        for reference in misp_object.ObjectReference:
+            misp.add_object_reference(reference)
+
+
+if __name__ == "__main__":
+    try:
+        args = build_cli()
+        config = build_config(args.config)
+        # change the 'ssl' value if you want to verify your MISP's SSL instance
+        misp = pymisp.PyMISP(url=config["misp"]["url"], key=config["misp"]["key"], ssl=False)
+        # finally, let's start checking VT and converting the reports
+        main(misp, config, args)
+    except KeyboardInterrupt:
+        print("Bye Felicia")
+    except pymisp.exceptions.InvalidMISPObject as err:
+        logging.error(err)

pymisp/__init__.py

@@ -1,4 +1,9 @@
-__version__ = '2.4.81'
+__version__ = '2.4.81.2'
+import sys
+import logging
+logger = logging.getLogger(__name__)
+FORMAT = "%(levelname)s [%(filename)s:%(lineno)s - %(funcName)s() ] %(message)s"
+logging.basicConfig(stream=sys.stderr, level=logging.WARNING, format=FORMAT)
 
 try:
     from .exceptions import PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey, InvalidMISPObject, UnknownMISPObjectTemplate  # noqa
@@ -9,5 +14,6 @@ try:
     from .tools import Neo4j  # noqa
     from .tools import stix  # noqa
     from .tools import openioc  # noqa
-except ImportError:
+    logger.debug('pymisp loaded properly')
-    pass
+except ImportError as e:
+    logger.warning('Unable to load pymisp properly: {}'.format(e))

pymisp/abstract.py

@@ -6,10 +6,12 @@ import json
 from json import JSONEncoder
 import collections
 import six  # Remove that import when discarding python2 support.
+import logging
+
+logger = logging.getLogger('pymisp')
 
 if six.PY2:
-    import warnings
+    logger.warning("You're using python 2, it is strongly recommended to use python >=3.5")
-    warnings.warn("You're using python 2, it is strongly recommended to use python >=3.5")
 
 
 class MISPEncode(JSONEncoder):
@@ -62,7 +64,7 @@ class AbstractMISP(collections.MutableMapping):
         return self.to_dict()
 
     def to_json(self):
-        return json.dumps(self.to_dict(), cls=MISPEncode)
+        return json.dumps(self, cls=MISPEncode)
 
     def __getitem__(self, key):
         return getattr(self, key)

pymisp/data/describeTypes.json

@@ -805,8 +805,6 @@
         "vulnerability",
         "x509-fingerprint-sha1",
         "other",
-        "ip-dst|port",
-        "ip-src|port",
         "hostname|port",
         "email-dst-display-name",
         "email-src-display-name",
@@ -1023,7 +1021,6 @@
         "text",
         "attachment",
         "comment",
-        "text",
         "other",
         "hex"
       ],

pymisp/data/misp-objects

@@ -1 +1 @@
-Subproject commit bc5795dc189bcf9cd001527656fe10e6ed10dc5f
+Subproject commit 6b43b68651a350a26891080ef0feda364b74727a

pymisp/exceptions.py

@@ -40,6 +40,7 @@ class InvalidMISPObject(MISPObjectException):
     """Exception raised when an object doesn't respect the contrains in the definition"""
     pass
 
+
 class UnknownMISPObjectTemplate(MISPObjectException):
     """Exception raised when the template is unknown"""
     pass

pymisp/mispevent.py

@@ -16,12 +16,14 @@ from collections import Counter
 from .abstract import AbstractMISP
 from .exceptions import UnknownMISPObjectTemplate, InvalidMISPObject, PyMISPError, NewEventError, NewAttributeError
 
-
 import six  # Remove that import when discarding python2 support.
 
+import logging
+logger = logging.getLogger('pymisp')
+
+
 if six.PY2:
-    import warnings
+    logger.warning("You're using python 2, it is strongly recommended to use python >=3.5")
-    warnings.warn("You're using python 2, it is strongly recommended to use python >=3.5")
 
 try:
     from dateutil.parser import parse
@@ -138,7 +140,7 @@ class MISPAttribute(AbstractMISP):
             try:
                 c.verify(signed_data, signature=base64.b64decode(self.sig), verify=keys[:1])
                 return {self.uuid: True}
-            except:
+            except Exception:
                 return {self.uuid: False}
 
     def set_all_values(self, **kwargs):
@@ -165,15 +167,21 @@ class MISPAttribute(AbstractMISP):
 
         # Default values
         self.category = kwargs.pop('category', type_defaults['default_category'])
+        if self.category is None:
+            # In case the category key is passed, but None
+            self.category = type_defaults['default_category']
         if self.category not in self.__categories:
             raise NewAttributeError('{} is invalid, category has to be in {}'.format(self.category, (', '.join(self.__categories))))
 
         self.to_ids = kwargs.pop('to_ids', bool(int(type_defaults['to_ids'])))
+        if self.to_ids is None:
+            self.to_ids = bool(int(type_defaults['to_ids']))
         if not isinstance(self.to_ids, bool):
             raise NewAttributeError('{} is invalid, to_ids has to be True or False'.format(self.to_ids))
 
-        if kwargs.get('distribution') is not None:
+        self.distribution = kwargs.pop('distribution', None)
-            self.distribution = int(kwargs.pop('distribution'))
+        if self.distribution is not None:
+            self.distribution = int(self.distribution)
             if self.distribution not in [0, 1, 2, 3, 4, 5]:
                 raise NewAttributeError('{} is invalid, the distribution has to be in 0, 1, 2, 3, 4, 5'.format(self.distribution))
 
@@ -215,20 +223,37 @@ class MISPAttribute(AbstractMISP):
         self._malware_binary = self.data
         self.encrypt = True
 
+    def __is_misp_encrypted_file(self, f):
+        files_list = f.namelist()
+        if len(files_list) != 2:
+            return False
+        md5_from_filename = ''
+        md5_from_file = ''
+        for name in files_list:
+            if name.endswith('.filename.txt'):
+                md5_from_filename = name.replace('.filename.txt', '')
+            else:
+                md5_from_file = name
+        if not md5_from_filename or not md5_from_file or md5_from_filename != md5_from_file:
+            return False
+        return True
+
     def _load_data(self):
         if not isinstance(self.data, BytesIO):
             self.data = BytesIO(base64.b64decode(self.data))
         if self.type == 'malware-sample':
             try:
                 with ZipFile(self.data) as f:
+                    if not self.__is_misp_encrypted_file(f):
+                        raise Exception('Not an existing malware sample')
                     for name in f.namelist():
-                        if name.endswith('.txt'):
+                        if name.endswith('.filename.txt'):
                             with f.open(name, pwd=b'infected') as unpacked:
-                                self.malware_filename = unpacked.read().decode()
+                                self.malware_filename = unpacked.read().decode().strip()
                         else:
                             with f.open(name, pwd=b'infected') as unpacked:
                                 self._malware_binary = BytesIO(unpacked.read())
-            except:
+            except Exception:
                 # not a encrypted zip file, assuming it is a new malware sample
                 self._prepare_new_malware_sample()
 
@@ -359,7 +384,7 @@ class MISPEvent(AbstractMISP):
             try:
                 c.verify(signed_data, signature=base64.b64decode(self.sig), verify=keys[:1])
                 to_return[self.uuid] = True
-            except:
+            except Exception:
                 to_return[self.uuid] = False
         for a in self.attributes:
             to_return.update(a.verify(gpg_uid))
@@ -369,7 +394,7 @@ class MISPEvent(AbstractMISP):
             try:
                 c.verify(to_verify_global, signature=base64.b64decode(self.global_sig), verify=keys[:1])
                 to_return['global'] = True
-            except:
+            except Exception:
                 to_return['global'] = False
         return to_return
 
@@ -422,8 +447,9 @@ class MISPEvent(AbstractMISP):
             raise NewAttributeError('The info field of the new event is required.')
 
         # Default values for a valid event to send to a MISP instance
-        if kwargs.get('distribution') is not None:
+        self.distribution = kwargs.pop('distribution', None)
-            self.distribution = int(kwargs.pop('distribution'))
+        if self.distribution is not None:
+            self.distribution = int(self.distribution)
             if self.distribution not in [0, 1, 2, 3, 4]:
                 raise NewAttributeError('{} is invalid, the distribution has to be in 0, 1, 2, 3, 4'.format(self.distribution))
 
@@ -565,7 +591,7 @@ class MISPObjectReference(AbstractMISP):
         self.referenced_uuid = referenced_uuid
         self.relationship_type = relationship_type
         self.comment = comment
-        for k, v in kwargs:
+        for k, v in kwargs.items():
             setattr(self, k, v)
 
 
@@ -598,7 +624,7 @@ class MISPObjectAttribute(MISPAttribute):
 
 class MISPObject(AbstractMISP):
 
-    def __init__(self, name, strict=True):
+    def __init__(self, name, strict=False):
         super(MISPObject, self).__init__()
         self.__strict = strict
         self.name = name
@@ -634,7 +660,7 @@ class MISPObject(AbstractMISP):
                 else:
                     self.__known_template = False
             if kwargs.get('template_version') and int(kwargs['template_version']) != self.template_version:
-                if self.strict:
+                if self.__strict:
                     raise UnknownMISPObjectTemplate('Version of the object ({}) is different from the one of the template ({}).'.format(kwargs['template_version'], self.template_version))
                 else:
                     self.__known_template = False
@@ -649,12 +675,12 @@ class MISPObject(AbstractMISP):
             else:
                 setattr(self, key, value)
 
-    def to_dict(self, strict=True):
+    def to_dict(self, strict=False):
         if strict or self.__strict and self.__known_template:
             self._validate()
         return super(MISPObject, self).to_dict()
 
-    def to_json(self, strict=True):
+    def to_json(self, strict=False):
         if strict or self.__strict and self.__known_template:
             self._validate()
         return super(MISPObject, self).to_json()
@@ -684,7 +710,7 @@ class MISPObject(AbstractMISP):
         """Add a link (uuid) to an other object"""
         if kwargs.get('object_uuid'):
             # Load existing object
-            object_uuid = kwargs.get('object_uuid')
+            object_uuid = kwargs.pop('object_uuid')
         else:
             # New reference
             object_uuid = self.uuid
@@ -697,7 +723,11 @@ class MISPObject(AbstractMISP):
         if value.get('value') is None:
             return None
         if self.__known_template:
-            attribute = MISPObjectAttribute(self.__definition['attributes'][object_relation])
+            if self.__definition['attributes'].get(object_relation):
+                attribute = MISPObjectAttribute(self.__definition['attributes'][object_relation])
+            else:
+                # Woopsie, this object_relation is unknown, no sane defaults for you.
+                attribute = MISPObjectAttribute({})
         else:
             attribute = MISPObjectAttribute({})
         attribute.from_dict(object_relation, **value)

pymisp/tools/__init__.py

@@ -1,3 +1,4 @@
+from .vtreportobject import VTReportObject  # noqa
 from .neo4j import Neo4j  # noqa
 from .fileobject import FileObject  # noqa
 from .peobject import PEObject, PESectionObject  # noqa

pymisp/tools/create_misp_object.py

@@ -3,7 +3,9 @@
 
 from . import FileObject, PEObject, ELFObject, MachOObject
 from ..exceptions import MISPObjectException
-import warnings
+import logging
+
+logger = logging.getLogger('pymisp')
 
 try:
     import lief
@@ -57,15 +59,15 @@ def make_binary_objects(filepath=None, pseudofile=None, filename=None):
             elif isinstance(lief_parsed, lief.MachO.Binary):
                 return make_macho_objects(lief_parsed, misp_file)
         except lief.bad_format as e:
-            warnings.warn('\tBad format: {}'.format(e))
+            logger.warning('Bad format: {}'.format(e))
         except lief.bad_file as e:
-            warnings.warn('\tBad file: {}'.format(e))
+            logger.warning('Bad file: {}'.format(e))
         except lief.parser_error as e:
-            warnings.warn('\tParser error: {}'.format(e))
+            logger.warning('Parser error: {}'.format(e))
         except FileTypeNotImplemented as e:  # noqa
-            warnings.warn(e)
+            logger.warning(e)
     if not HAS_LIEF:
-        warnings.warn('Please install lief, documentation here: https://github.com/lief-project/LIEF')
+        logger.warning('Please install lief, documentation here: https://github.com/lief-project/LIEF')
     if not filepath:
-        warnings.warn('LIEF currently requires a filepath and not a pseudo file')
+        logger.warning('LIEF currently requires a filepath and not a pseudo file')
     return misp_file, None, None

pymisp/tools/elfobject.py

@@ -5,8 +5,9 @@ from .abstractgenerator import AbstractMISPObjectGenerator
 from ..exceptions import InvalidMISPObject
 from io import BytesIO
 from hashlib import md5, sha1, sha256, sha512
-import warnings
+import logging
 
+logger = logging.getLogger('pymisp')
 
 try:
     import lief
@@ -25,7 +26,7 @@ class ELFObject(AbstractMISPObjectGenerator):
 
     def __init__(self, parsed=None, filepath=None, pseudofile=None):
         if not HAS_PYDEEP:
-            warnings.warn("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git")
+            logger.warning("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git")
         if not HAS_LIEF:
             raise ImportError('Please install lief, documentation here: https://github.com/lief-project/LIEF')
         if pseudofile:

pymisp/tools/fileobject.py

@@ -8,7 +8,10 @@ from io import BytesIO
 from hashlib import md5, sha1, sha256, sha512
 import math
 from collections import Counter
-import warnings
+import logging
+
+logger = logging.getLogger('pymisp')
+
 
 try:
     import pydeep
@@ -27,9 +30,9 @@ class FileObject(AbstractMISPObjectGenerator):
 
     def __init__(self, filepath=None, pseudofile=None, filename=None):
         if not HAS_PYDEEP:
-            warnings.warn("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git")
+            logger.warning("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git")
         if not HAS_MAGIC:
-            warnings.warn("Please install python-magic: pip install python-magic.")
+            logger.warning("Please install python-magic: pip install python-magic.")
         if filename:
             # Useful in case the file is copied with a pre-defined name by a script but we want to keep the original name
             self.__filename = filename

pymisp/tools/machoobject.py

@@ -5,7 +5,9 @@ from ..exceptions import InvalidMISPObject
 from .abstractgenerator import AbstractMISPObjectGenerator
 from io import BytesIO
 from hashlib import md5, sha1, sha256, sha512
-import warnings
+import logging
+
+logger = logging.getLogger('pymisp')
 
 
 try:
@@ -25,7 +27,7 @@ class MachOObject(AbstractMISPObjectGenerator):
 
     def __init__(self, parsed=None, filepath=None, pseudofile=None):
         if not HAS_PYDEEP:
-            warnings.warn("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git")
+            logger.warning("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git")
         if not HAS_LIEF:
             raise ImportError('Please install lief, documentation here: https://github.com/lief-project/LIEF')
         if pseudofile:

pymisp/tools/openioc.py

@@ -11,7 +11,6 @@ except ImportError:
     has_bs4 = False
 
 iocMispMapping = {
-    # ~ @Link https://wiki.ops.fr/doku.php/manuels:misp:event-guidelines
     'CookieHistoryItem/HostName': {'type': 'hostname', 'comment': 'CookieHistory.'},
 
     'DriverItem/DriverName': {'category': 'Artifacts dropped', 'type': 'other', 'comment': 'DriverName.'},

pymisp/tools/peobject.py

@@ -6,8 +6,9 @@ from .abstractgenerator import AbstractMISPObjectGenerator
 from io import BytesIO
 from hashlib import md5, sha1, sha256, sha512
 from datetime import datetime
-import warnings
+import logging
 
+logger = logging.getLogger('pymisp')
 
 try:
     import lief
@@ -26,7 +27,7 @@ class PEObject(AbstractMISPObjectGenerator):
 
     def __init__(self, parsed=None, filepath=None, pseudofile=None):
         if not HAS_PYDEEP:
-            warnings.warn("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git")
+            logger.warning("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git")
         if not HAS_LIEF:
             raise ImportError('Please install lief, documentation here: https://github.com/lief-project/LIEF')
         if pseudofile:

pymisp/tools/vtreportobject.py

@@ -0,0 +1,81 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import re
+
+import requests
+try:
+    import validators
+    has_validators = True
+except ImportError:
+    has_validators = False
+
+
+from .abstractgenerator import AbstractMISPObjectGenerator
+from .. import InvalidMISPObject
+
+
+class VTReportObject(AbstractMISPObjectGenerator):
+    '''
+    VirusTotal Report
+
+    :apikey: VirusTotal API key (private works, but only public features are supported right now)
+
+    :indicator: IOC to search VirusTotal for
+    '''
+    def __init__(self, apikey, indicator):
+        # PY3 way:
+        # super().__init__("virustotal-report")
+        super(VTReportObject, self).__init__("virustotal-report")
+        indicator = indicator.strip()
+        self._resource_type = self.__validate_resource(indicator)
+        if self._resource_type:
+            self._report = self.__query_virustotal(apikey, indicator)
+            self.generate_attributes()
+        else:
+            error_msg = "A valid indicator is required. (One of type url, md5, sha1, sha256). Received '{}' instead".format(indicator)
+            raise InvalidMISPObject(error_msg)
+        # Mark as non_jsonable because we need to add the references manually after the object(s) have been created
+        self.update_not_jsonable('ObjectReference')
+
+    def generate_attributes(self):
+        ''' Parse the VirusTotal report for relevant attributes '''
+        self.add_attribute("last-submission", value=self._report["scan_date"])
+        self.add_attribute("permalink", value=self._report["permalink"])
+        ratio = "{}/{}".format(self._report["positives"], self._report["total"])
+        self.add_attribute("detection-ratio", value=ratio)
+
+    def __validate_resource(self, ioc):
+        '''
+        Validate the data type of an indicator.
+        Domains and IP addresses aren't supported because
+        they don't return the same type of data as the URLs/files do
+
+        :ioc: Indicator to search VirusTotal for
+        '''
+        if not has_validators:
+            raise Exception('You need to install validators: pip install validators')
+        if validators.url(ioc):
+            return "url"
+        elif re.match(r"\b([a-fA-F0-9]{32}|[a-fA-F0-9]{40}|[a-fA-F0-9]{64})\b", ioc):
+            return "file"
+        return False
+
+    def __query_virustotal(self, apikey, resource):
+        '''
+        Query VirusTotal for information about an indicator
+
+        :apikey: VirusTotal API key
+
+        :resource: Indicator to search in VirusTotal
+        '''
+        url = "https://www.virustotal.com/vtapi/v2/{}/report".format(self._resource_type)
+        params = {"apikey": apikey, "resource": resource}
+        # for now assume we're using a public API key - we'll figure out private keys later
+        report = requests.get(url, params=params)
+        report = report.json()
+        if report["response_code"] == 1:
+            return report
+        else:
+            error_msg = "{}: {}".format(resource, report["verbose_msg"])
+            raise InvalidMISPObject(error_msg)

setup.py

@@ -27,7 +27,11 @@ setup(
         'Topic :: Internet',
     ],
     test_suite="tests.test_offline",
-    install_requires=['six', 'requests', 'python-dateutil', 'jsonschema'],
+    install_requires=['six', 'requests', 'python-dateutil', 'jsonschema', 'setuptools>=36.4'],
+    extras_require={'fileobjects': ['lief>=0.8', 'python-magic'],
+                    'neo': ['py2neo'],
+                    'openioc': ['beautifulsoup4'],
+                    'virustotal': ['validators']},
     tests_require=[
         'jsonschema',
         'python-dateutil',
@@ -36,7 +40,8 @@ setup(
         'six'
     ],
     include_package_data=True,
-    package_data={'pymisp': ['data/*.json', 'data/misp-objects/schema_objects.json',
+    package_data={'pymisp': ['data/*.json',
+                             'data/misp-objects/schema_objects.json',
                              'data/misp-objects/schema_relationships.json',
                              'data/misp-objects/objects/*/definition.json',
                              'data/misp-objects/relationships/definition.json']},

tests/test_offline.py

@@ -47,7 +47,7 @@ class TestOffline(unittest.TestCase):
         m.register_uri('POST', self.domain + 'events/5758ebf5-c898-48e6-9fe9-5665c0a83866', json=self.event)
         m.register_uri('DELETE', self.domain + 'events/2', json={'message': 'Event deleted.'})
         m.register_uri('DELETE', self.domain + 'events/3', json={'errors': ['Invalid event'], 'message': 'Invalid event', 'name': 'Invalid event', 'url': '/events/3'})
-        m.register_uri('DELETE', self.domain + 'attributes/2', json={'message': 'Attribute deleted.'})
+        m.register_uri('GET', self.domain + 'attributes/delete/2', json={'message': 'Attribute deleted.'})
         m.register_uri('POST', self.domain + 'events/index', json=self.search_index_result)
 
     def test_getEvent(self, m):