Improvements in the user api

examples/graphdb/make_neo4j.py

@@ -0,0 +1,45 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from pymisp import PyMISP
+from pymisp import Neo4j
+from pymisp import MISPEvent
+from keys import misp_url, misp_key
+import argparse
+
+"""
+Sample Neo4J query:
+
+
+MATCH ()-[r:has]->(n)
+WITH n, count(r) as rel_cnt
+WHERE rel_cnt > 5
+MATCH (m)-[r:has]->(n)
+RETURN m, n LIMIT 200;
+"""
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Get all the events matching a value.')
+    parser.add_argument("-s", "--search", required=True, help="String to search.")
+    parser.add_argument("--host", default='localhost:7474', help="Host where neo4j is running.")
+    parser.add_argument("-u", "--user", default='neo4j', help="User on neo4j.")
+    parser.add_argument("-p", "--password", default='neo4j', help="Password on neo4j.")
+    parser.add_argument("-d", "--deleteall", action="store_true", default=False, help="Delete all nodes from the database")
+    args = parser.parse_args()
+
+    neo4j = Neo4j(args.host, args.user, args.password)
+    if args.deleteall:
+        neo4j.del_all()
+    misp = PyMISP(misp_url, misp_key)
+    result = misp.search_all(args.search)
+    for json_event in result['response']:
+        if not json_event['Event']:
+            print(json_event)
+            continue
+        print('Importing', json_event['Event']['info'], json_event['Event']['id'])
+        try:
+            misp_event = MISPEvent()
+            misp_event.load(json_event)
+            neo4j.import_event(misp_event)
+        except:
+            print('broken')

pymisp/__init__.py

@@ -3,3 +3,5 @@ __version__ = '2.4.53'
 from .exceptions import PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey
 from .api import PyMISP
 from .mispevent import MISPEvent, MISPAttribute, EncodeUpdate, EncodeFull
+from .tools.neo4j import Neo4j
+from .tools import stix

pymisp/api.py

@@ -3,6 +3,7 @@
 
 """Python API using the REST interface of MISP"""
 
+import sys
 import json
 import datetime
 import os
@@ -136,7 +137,7 @@ class PyMISP(object):
             {'Authorization': self.key,
              'Accept': 'application/{}'.format(output),
              'content-type': 'application/{}'.format(output),
-             'User-Agent': 'PyMISP {}'.format(__version__)})
+             'User-Agent': 'PyMISP {} - Python {}.{}.{}'.format(__version__, *sys.version_info)})
         return session
 
     def flatten_error_messages(self, response):

pymisp/mispevent.py

@@ -6,6 +6,7 @@ import time
 import json
 from json import JSONEncoder
 import os
+import warnings
 try:
     from dateutil.parser import parse
 except ImportError:
@@ -20,10 +21,12 @@ from .exceptions import PyMISPError, NewEventError, NewAttributeError
 
 # Least dirty way to support python 2 and 3
 try:
-    warnings.warn("You're using python 2, it is strongly recommended to use python >=3.3")
     basestring
+    unicode
+    warnings.warn("You're using python 2, it is strongly recommended to use python >=3.3")
 except NameError:
     basestring = str
+    unicode = str
 
 
 class MISPAttribute(object):
@@ -231,6 +234,19 @@ class MISPEvent(object):
         self._reinitialize_event()
         self.set_all_values(**e)
 
+    def set_date(self, date, ignore_invalid=False):
+        if isinstance(date, basestring) or isinstance(date, unicode):
+            self.date = parse(date).date()
+        elif isinstance(date, datetime.datetime):
+            self.date = date.date()
+        elif isinstance(date, datetime.date):
+            self.date = date
+        else:
+            if ignore_invalid:
+                self.date = datetime.date.today()
+            else:
+                raise NewEventError('Invalid format for the date: {} - {}'.format(date, type(date)))
+
     def set_all_values(self, **kwargs):
         # Required value
         if kwargs.get('info'):
@@ -254,14 +270,7 @@ class MISPEvent(object):
         if kwargs.get('published') is not None:
             self.publish()
         if kwargs.get('date'):
-            if isinstance(kwargs['date'], basestring) or isinstance(kwargs['date'], unicode):
+            self.set_date(kwargs['date'])
-                self.date = parse(kwargs['date']).date()
-            elif isinstance(kwargs['date'], datetime.datetime):
-                self.date = kwargs['date'].date()
-            elif isinstance(kwargs['date'], datetime.date):
-                self.date = kwargs['date']
-            else:
-                raise NewEventError('Invalid format for the date: {} - {}'.format(kwargs['date'], type(kwargs['date'])))
         if kwargs.get('Attribute'):
             for a in kwargs['Attribute']:
                 attribute = MISPAttribute(self.describe_types)

pymisp/tools/neo4j.py

@@ -0,0 +1,58 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import glob
+import os
+from pymisp import MISPEvent
+
+try:
+    from py2neo import authenticate, Graph, Node, Relationship
+    has_py2neo = True
+except ImportError:
+    has_py2neo = False
+
+
+class Neo4j():
+
+    def __init__(self, host='localhost:7474', username='neo4j', password='neo4j'):
+        if not has_py2neo:
+            raise Exception('py2neo is required, please install: pip install py2neo')
+        authenticate(host, username, password)
+        self.graph = Graph()
+
+    def load_events_directory(self, directory):
+        self.events = []
+        for path in glob.glob(os.path.join(directory, '*.json')):
+            e = MISPEvent()
+            e.load(path)
+            self.import_event(e)
+
+    def del_all(self):
+        self.graph.delete_all()
+
+    def import_event(self, event):
+        tx = self.graph.begin()
+        event_node = Node('Event', uuid=event.uuid, name=event.info)
+        # event_node['distribution'] = event.distribution
+        # event_node['threat_level_id'] = event.threat_level_id
+        # event_node['analysis'] = event.analysis
+        # event_node['published'] = event.published
+        # event_node['date'] = event.date.isoformat()
+        tx.create(event_node)
+        for a in event.attributes:
+            attr_node = Node('Attribute', a.type, uuid=a.uuid)
+            attr_node['category'] = a.category
+            attr_node['name'] = a.value
+            # attr_node['to_ids'] = a.to_ids
+            # attr_node['comment'] = a.comment
+            # attr_node['distribution'] = a.distribution
+            tx.create(attr_node)
+            member_rel = Relationship(event_node, "is member", attr_node)
+            tx.create(member_rel)
+            val = Node('Value', name=a.value)
+            ev = Relationship(event_node, "has", val)
+            av = Relationship(attr_node, "is", val)
+            s = val | ev | av
+            tx.merge(s)
+            tx.graph.push(s)
+        tx.commit()

pymisp/tools/openioc.py

@@ -0,0 +1,128 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from pymisp import MISPEvent
+try:
+    from bs4 import BeautifulSoup
+    has_bs4 = True
+except ImportError:
+    has_bs4 = False
+
+
+iocMispMapping = {
+    'DriverItem/DriverName': {'category': 'Artifacts dropped', 'type': 'other', 'comment': 'DriverName.'},
+
+    'DnsEntryItem/Host': {'type': 'domain'},
+
+    'Email/To': {'type': 'target-email'},
+    'Email/Date': {'type': 'comment', 'comment': 'EmailDate.'},
+    'Email/Body': {'type': 'email-subject'},
+    'Email/From': {'type': 'email-dst'},
+    'Email/Subject': {'type': 'email-subject'},
+    'Email/Attachment/Name': {'type': 'email-attachment'},
+
+    'FileItem/Md5sum': {'type': 'md5'},
+    'FileItem/Sha1sum': {'type': 'sha1'},
+    'TaskItem/Sha1sum': {'type': 'sha1'},
+    'FileItem/Sha256sum': {'type': 'sha256'},
+    'FileItem/FileName': {'type': 'filename'},
+    'FileItem/FullPath': {'type': 'filename'},
+    'FileItem/FilePath': {'type': 'filename'},
+    'DriverItem/DriverName': {'type': 'filename'},
+
+    'Network/URI': {'type': 'uri'},
+    'Network/DNS': {'type': 'domain'},
+    'Network/String': {'type': 'ip-dst'},
+    'RouteEntryItem/Destination': {'type': 'ip-dst'},
+    'Network/UserAgent': {'type': 'user-agent'},
+
+    'PortItem/localIP': {'type': 'ip-dst'},
+
+    'ProcessItem/name': {'type': 'pattern-in-memory', 'comment': 'ProcessName.'},
+    'ProcessItem/path': {'type': 'pattern-in-memory', 'comment': 'ProcessPath.'},
+    'ProcessItem/Mutex': {'type': 'mutex'},
+    'ProcessItem/Pipe/Name': {'type': 'named pipe'},
+    'ProcessItem/Mutex/Name': {'type': 'mutex', 'comment': 'MutexName.'},
+
+    'CookieHistoryItem/HostName': {'type': 'hostname'},
+    'FormHistoryItem/HostName': {'type': 'hostname'},
+    'SystemInfoItem/HostName': {'type': 'hostname'},
+    'UrlHistoryItem/HostName': {'type': 'hostname'},
+    'DnsEntryItem/RecordName': {'type': 'hostname'},
+    'DnsEntryItem/Host': {'type': 'hostname'},
+
+    # Is it the regkey value?
+    # 'RegistryItem/Text': {'type': 'regkey', 'RegistryText. '},
+    'RegistryItem/KeyPath': {'type': 'regkey'},
+    'RegistryItem/Path': {'type': 'regkey'},
+
+    'ServiceItem/name': {'type': 'windows-service-name'},
+    'ServiceItem/type': {'type': 'pattern-in-memory', 'comment': 'ServiceType. '},
+
+    'Snort/Snort': {'type': 'snort'},
+}
+
+
+def extract_field(report, field_name):
+    data = report.find(field_name.lower())
+    if data and hasattr(data, 'text'):
+        return data.text
+    return None
+
+
+def load_openioc(openioc):
+    if not has_bs4:
+        raise Exception('You need to install BeautifulSoup: pip install bs4')
+    misp_event = MISPEvent()
+    with open(openioc, "r") as ioc_file:
+        iocreport = BeautifulSoup(ioc_file, "lxml")
+        # Set event fields
+        info = extract_field(iocreport, 'short_description')
+        if info:
+            misp_event.info = info
+        date = extract_field(iocreport, 'authored_date')
+        if date:
+            misp_event.set_date(date)
+        # Set special attributes
+        description = extract_field(iocreport, 'description')
+        if description:
+            misp_event.add_attribute('comment', description)
+        author = extract_field(iocreport, 'authored_by')
+        if author:
+            misp_event.add_attribute('comment', author)
+        misp_event = set_all_attributes(iocreport, misp_event)
+    return misp_event
+
+
+def get_mapping(openioc_type):
+    t = openioc_type.lower()
+    for k, v in iocMispMapping.items():
+        if k.lower() == t:
+            return v
+    return False
+
+
+def set_all_attributes(openioc, misp_event):
+    for item in openioc.find_all("indicatoritem"):
+        attribute_values = {'comment': ''}
+        if item.find('context'):
+            mapping = get_mapping(item.find('context')['search'])
+            if mapping:
+                attribute_values.update(mapping)
+            else:
+                # Unknown mapping, ignoring
+                # print(item.find('context'))
+                continue
+        else:
+            continue
+        value = extract_field(openioc, 'Content')
+        if value:
+            attribute_values['value'] = value
+        else:
+            # No value, ignoring
+            continue
+        comment = extract_field(openioc, 'Comment')
+        if comment:
+            attribute_values["comment"] = '{} {}'.format(attribute_values["comment"], comment)
+        misp_event.add_attribute(**attribute_values)
+    return misp_event

pymisp/tools/stix.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+try:
+    from misp_stix_converter.converters.buildMISPAttribute import buildEvent, open_stix
+    from misp_stix_converter.converters.convert import MISPtoSTIX
+    has_misp_stix_converter = True
+except ImportError:
+    has_misp_stix_converter = False
+
+
+def load_stix(stix, distribution=3, threat_level_id=2, analysis=0):
+    '''Returns a MISPEvent object from a STIX package'''
+    if not has_misp_stix_converter:
+        raise Exception('You need to install misp_stix_converter from https://github.com/MISP/MISP-STIX-Converter')
+    stix = open_stix(stix)
+    return buildEvent(stix, distribution=distribution,
+                      threat_level_id=threat_level_id, analysis=analysis)
+
+
+def make_stix_package(misp_event, to_json=False, to_xml=False):
+    '''Returns a STIXPackage from a MISPEvent.
+
+       Optionally can return the package in json or xml.
+
+    '''
+    if not has_misp_stix_converter:
+        raise Exception('You need to install misp_stix_converter from https://github.com/MISP/MISP-STIX-Converter')
+    package = MISPtoSTIX(misp_event)
+    if to_json:
+        return package.to_json()
+    elif to_xml:
+        return package.to_xml()
+    else:
+        return package