diff --git a/examples/graphdb/make_neo4j.py b/examples/graphdb/make_neo4j.py new file mode 100755 index 0000000..6393813 --- /dev/null +++ b/examples/graphdb/make_neo4j.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from pymisp import PyMISP +from pymisp import Neo4j +from pymisp import MISPEvent +from keys import misp_url, misp_key +import argparse + +""" +Sample Neo4J query: + + +MATCH ()-[r:has]->(n) +WITH n, count(r) as rel_cnt +WHERE rel_cnt > 5 +MATCH (m)-[r:has]->(n) +RETURN m, n LIMIT 200; +""" + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Get all the events matching a value.') + parser.add_argument("-s", "--search", required=True, help="String to search.") + parser.add_argument("--host", default='localhost:7474', help="Host where neo4j is running.") + parser.add_argument("-u", "--user", default='neo4j', help="User on neo4j.") + parser.add_argument("-p", "--password", default='neo4j', help="Password on neo4j.") + parser.add_argument("-d", "--deleteall", action="store_true", default=False, help="Delete all nodes from the database") + args = parser.parse_args() + + neo4j = Neo4j(args.host, args.user, args.password) + if args.deleteall: + neo4j.del_all() + misp = PyMISP(misp_url, misp_key) + result = misp.search_all(args.search) + for json_event in result['response']: + if not json_event['Event']: + print(json_event) + continue + print('Importing', json_event['Event']['info'], json_event['Event']['id']) + try: + misp_event = MISPEvent() + misp_event.load(json_event) + neo4j.import_event(misp_event) + except: + print('broken') diff --git a/pymisp/__init__.py b/pymisp/__init__.py index 7cbe0ee..dac6257 100644 --- a/pymisp/__init__.py +++ b/pymisp/__init__.py @@ -3,3 +3,5 @@ __version__ = '2.4.53' from .exceptions import PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey from .api import PyMISP from .mispevent import MISPEvent, MISPAttribute, EncodeUpdate, EncodeFull +from .tools.neo4j import Neo4j +from .tools import stix diff --git a/pymisp/api.py b/pymisp/api.py index 3b7eb04..1468c38 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -3,6 +3,7 @@ """Python API using the REST interface of MISP""" +import sys import json import datetime import os @@ -136,7 +137,7 @@ class PyMISP(object): {'Authorization': self.key, 'Accept': 'application/{}'.format(output), 'content-type': 'application/{}'.format(output), - 'User-Agent': 'PyMISP {}'.format(__version__)}) + 'User-Agent': 'PyMISP {} - Python {}.{}.{}'.format(__version__, *sys.version_info)}) return session def flatten_error_messages(self, response): diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index b436aab..e0ecb73 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -6,6 +6,7 @@ import time import json from json import JSONEncoder import os +import warnings try: from dateutil.parser import parse except ImportError: @@ -20,10 +21,12 @@ from .exceptions import PyMISPError, NewEventError, NewAttributeError # Least dirty way to support python 2 and 3 try: - warnings.warn("You're using python 2, it is strongly recommended to use python >=3.3") basestring + unicode + warnings.warn("You're using python 2, it is strongly recommended to use python >=3.3") except NameError: basestring = str + unicode = str class MISPAttribute(object): @@ -231,6 +234,19 @@ class MISPEvent(object): self._reinitialize_event() self.set_all_values(**e) + def set_date(self, date, ignore_invalid=False): + if isinstance(date, basestring) or isinstance(date, unicode): + self.date = parse(date).date() + elif isinstance(date, datetime.datetime): + self.date = date.date() + elif isinstance(date, datetime.date): + self.date = date + else: + if ignore_invalid: + self.date = datetime.date.today() + else: + raise NewEventError('Invalid format for the date: {} - {}'.format(date, type(date))) + def set_all_values(self, **kwargs): # Required value if kwargs.get('info'): @@ -254,14 +270,7 @@ class MISPEvent(object): if kwargs.get('published') is not None: self.publish() if kwargs.get('date'): - if isinstance(kwargs['date'], basestring) or isinstance(kwargs['date'], unicode): - self.date = parse(kwargs['date']).date() - elif isinstance(kwargs['date'], datetime.datetime): - self.date = kwargs['date'].date() - elif isinstance(kwargs['date'], datetime.date): - self.date = kwargs['date'] - else: - raise NewEventError('Invalid format for the date: {} - {}'.format(kwargs['date'], type(kwargs['date']))) + self.set_date(kwargs['date']) if kwargs.get('Attribute'): for a in kwargs['Attribute']: attribute = MISPAttribute(self.describe_types) diff --git a/pymisp/tools/__init__.py b/pymisp/tools/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pymisp/tools/neo4j.py b/pymisp/tools/neo4j.py new file mode 100644 index 0000000..af5ba2c --- /dev/null +++ b/pymisp/tools/neo4j.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import glob +import os +from pymisp import MISPEvent + +try: + from py2neo import authenticate, Graph, Node, Relationship + has_py2neo = True +except ImportError: + has_py2neo = False + + +class Neo4j(): + + def __init__(self, host='localhost:7474', username='neo4j', password='neo4j'): + if not has_py2neo: + raise Exception('py2neo is required, please install: pip install py2neo') + authenticate(host, username, password) + self.graph = Graph() + + def load_events_directory(self, directory): + self.events = [] + for path in glob.glob(os.path.join(directory, '*.json')): + e = MISPEvent() + e.load(path) + self.import_event(e) + + def del_all(self): + self.graph.delete_all() + + def import_event(self, event): + tx = self.graph.begin() + event_node = Node('Event', uuid=event.uuid, name=event.info) + # event_node['distribution'] = event.distribution + # event_node['threat_level_id'] = event.threat_level_id + # event_node['analysis'] = event.analysis + # event_node['published'] = event.published + # event_node['date'] = event.date.isoformat() + tx.create(event_node) + for a in event.attributes: + attr_node = Node('Attribute', a.type, uuid=a.uuid) + attr_node['category'] = a.category + attr_node['name'] = a.value + # attr_node['to_ids'] = a.to_ids + # attr_node['comment'] = a.comment + # attr_node['distribution'] = a.distribution + tx.create(attr_node) + member_rel = Relationship(event_node, "is member", attr_node) + tx.create(member_rel) + val = Node('Value', name=a.value) + ev = Relationship(event_node, "has", val) + av = Relationship(attr_node, "is", val) + s = val | ev | av + tx.merge(s) + tx.graph.push(s) + tx.commit() diff --git a/pymisp/tools/openioc.py b/pymisp/tools/openioc.py new file mode 100644 index 0000000..e6be33d --- /dev/null +++ b/pymisp/tools/openioc.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from pymisp import MISPEvent +try: + from bs4 import BeautifulSoup + has_bs4 = True +except ImportError: + has_bs4 = False + + +iocMispMapping = { + 'DriverItem/DriverName': {'category': 'Artifacts dropped', 'type': 'other', 'comment': 'DriverName.'}, + + 'DnsEntryItem/Host': {'type': 'domain'}, + + 'Email/To': {'type': 'target-email'}, + 'Email/Date': {'type': 'comment', 'comment': 'EmailDate.'}, + 'Email/Body': {'type': 'email-subject'}, + 'Email/From': {'type': 'email-dst'}, + 'Email/Subject': {'type': 'email-subject'}, + 'Email/Attachment/Name': {'type': 'email-attachment'}, + + 'FileItem/Md5sum': {'type': 'md5'}, + 'FileItem/Sha1sum': {'type': 'sha1'}, + 'TaskItem/Sha1sum': {'type': 'sha1'}, + 'FileItem/Sha256sum': {'type': 'sha256'}, + 'FileItem/FileName': {'type': 'filename'}, + 'FileItem/FullPath': {'type': 'filename'}, + 'FileItem/FilePath': {'type': 'filename'}, + 'DriverItem/DriverName': {'type': 'filename'}, + + 'Network/URI': {'type': 'uri'}, + 'Network/DNS': {'type': 'domain'}, + 'Network/String': {'type': 'ip-dst'}, + 'RouteEntryItem/Destination': {'type': 'ip-dst'}, + 'Network/UserAgent': {'type': 'user-agent'}, + + 'PortItem/localIP': {'type': 'ip-dst'}, + + 'ProcessItem/name': {'type': 'pattern-in-memory', 'comment': 'ProcessName.'}, + 'ProcessItem/path': {'type': 'pattern-in-memory', 'comment': 'ProcessPath.'}, + 'ProcessItem/Mutex': {'type': 'mutex'}, + 'ProcessItem/Pipe/Name': {'type': 'named pipe'}, + 'ProcessItem/Mutex/Name': {'type': 'mutex', 'comment': 'MutexName.'}, + + 'CookieHistoryItem/HostName': {'type': 'hostname'}, + 'FormHistoryItem/HostName': {'type': 'hostname'}, + 'SystemInfoItem/HostName': {'type': 'hostname'}, + 'UrlHistoryItem/HostName': {'type': 'hostname'}, + 'DnsEntryItem/RecordName': {'type': 'hostname'}, + 'DnsEntryItem/Host': {'type': 'hostname'}, + + # Is it the regkey value? + # 'RegistryItem/Text': {'type': 'regkey', 'RegistryText. '}, + 'RegistryItem/KeyPath': {'type': 'regkey'}, + 'RegistryItem/Path': {'type': 'regkey'}, + + 'ServiceItem/name': {'type': 'windows-service-name'}, + 'ServiceItem/type': {'type': 'pattern-in-memory', 'comment': 'ServiceType. '}, + + 'Snort/Snort': {'type': 'snort'}, +} + + +def extract_field(report, field_name): + data = report.find(field_name.lower()) + if data and hasattr(data, 'text'): + return data.text + return None + + +def load_openioc(openioc): + if not has_bs4: + raise Exception('You need to install BeautifulSoup: pip install bs4') + misp_event = MISPEvent() + with open(openioc, "r") as ioc_file: + iocreport = BeautifulSoup(ioc_file, "lxml") + # Set event fields + info = extract_field(iocreport, 'short_description') + if info: + misp_event.info = info + date = extract_field(iocreport, 'authored_date') + if date: + misp_event.set_date(date) + # Set special attributes + description = extract_field(iocreport, 'description') + if description: + misp_event.add_attribute('comment', description) + author = extract_field(iocreport, 'authored_by') + if author: + misp_event.add_attribute('comment', author) + misp_event = set_all_attributes(iocreport, misp_event) + return misp_event + + +def get_mapping(openioc_type): + t = openioc_type.lower() + for k, v in iocMispMapping.items(): + if k.lower() == t: + return v + return False + + +def set_all_attributes(openioc, misp_event): + for item in openioc.find_all("indicatoritem"): + attribute_values = {'comment': ''} + if item.find('context'): + mapping = get_mapping(item.find('context')['search']) + if mapping: + attribute_values.update(mapping) + else: + # Unknown mapping, ignoring + # print(item.find('context')) + continue + else: + continue + value = extract_field(openioc, 'Content') + if value: + attribute_values['value'] = value + else: + # No value, ignoring + continue + comment = extract_field(openioc, 'Comment') + if comment: + attribute_values["comment"] = '{} {}'.format(attribute_values["comment"], comment) + misp_event.add_attribute(**attribute_values) + return misp_event diff --git a/pymisp/tools/stix.py b/pymisp/tools/stix.py new file mode 100644 index 0000000..b964295 --- /dev/null +++ b/pymisp/tools/stix.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +try: + from misp_stix_converter.converters.buildMISPAttribute import buildEvent, open_stix + from misp_stix_converter.converters.convert import MISPtoSTIX + has_misp_stix_converter = True +except ImportError: + has_misp_stix_converter = False + + +def load_stix(stix, distribution=3, threat_level_id=2, analysis=0): + '''Returns a MISPEvent object from a STIX package''' + if not has_misp_stix_converter: + raise Exception('You need to install misp_stix_converter from https://github.com/MISP/MISP-STIX-Converter') + stix = open_stix(stix) + return buildEvent(stix, distribution=distribution, + threat_level_id=threat_level_id, analysis=analysis) + + +def make_stix_package(misp_event, to_json=False, to_xml=False): + '''Returns a STIXPackage from a MISPEvent. + + Optionally can return the package in json or xml. + + ''' + if not has_misp_stix_converter: + raise Exception('You need to install misp_stix_converter from https://github.com/MISP/MISP-STIX-Converter') + package = MISPtoSTIX(misp_event) + if to_json: + return package.to_json() + elif to_xml: + return package.to_xml() + else: + return package