MISP
/
PyMISP
mirror of https://github.com/MISP/PyMISP
2
0
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #68 from MISP/tooling 

[WIP] Tooling
pull/30/head
Raphaël Vinot 2016-11-03 16:04:28 -04:00 committed by GitHub
parent dbba8ed0f9 9f0737c34a
commit 75ebedae5c
8 changed files with 288 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
examples/make_neo4j.py Executable file
View File

@ -0,0 +1,45 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from pymisp import PyMISP
from pymisp import Neo4j
from pymisp import MISPEvent
from keys import misp_url, misp_key
import argparse
"""
Sample Neo4J query:
MATCH ()-[r:has]->(n)
WITH n, count(r) as rel_cnt
WHERE rel_cnt > 5
MATCH (m)-[r:has]->(n)
RETURN m, n LIMIT 200;
"""
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Get all the events matching a value.')
parser.add_argument("-s", "--search", required=True, help="String to search.")
parser.add_argument("--host", default='localhost:7474', help="Host where neo4j is running.")
parser.add_argument("-u", "--user", default='neo4j', help="User on neo4j.")
parser.add_argument("-p", "--password", default='neo4j', help="Password on neo4j.")
parser.add_argument("-d", "--deleteall", action="store_true", default=False, help="Delete all nodes from the database")
args = parser.parse_args()
neo4j = Neo4j(args.host, args.user, args.password)
if args.deleteall:
neo4j.del_all()
misp = PyMISP(misp_url, misp_key)
result = misp.search_all(args.search)
for json_event in result['response']:
if not json_event['Event']:
print(json_event)
continue
print('Importing', json_event['Event']['info'], json_event['Event']['id'])
try:
misp_event = MISPEvent()
misp_event.load(json_event)
neo4j.import_event(misp_event)
except:
print('broken')

2
pymisp/__init__.py
View File

@ -3,3 +3,5 @@ __version__ = '2.4.53'
from .exceptions import PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey
from .api import PyMISP
from .mispevent import MISPEvent, MISPAttribute, EncodeUpdate, EncodeFull
from .tools.neo4j import Neo4j
from .tools import stix

3
pymisp/api.py
View File

@ -3,6 +3,7 @@
"""Python API using the REST interface of MISP"""
import sys
import json
import datetime
import os
@ -136,7 +137,7 @@ class PyMISP(object):
{'Authorization': self.key,
'Accept': 'application/{}'.format(output),
'content-type': 'application/{}'.format(output),
'User-Agent': 'PyMISP {}'.format(__version__)})
'User-Agent': 'PyMISP {} - Python {}.{}.{}'.format(__version__, *sys.version_info)})
return session
def flatten_error_messages(self, response):

27
pymisp/mispevent.py
View File

@ -6,6 +6,7 @@ import time
import json
from json import JSONEncoder
import os
import warnings
try:
from dateutil.parser import parse
except ImportError:
@ -20,10 +21,12 @@ from .exceptions import PyMISPError, NewEventError, NewAttributeError
# Least dirty way to support python 2 and 3
try:
warnings.warn("You're using python 2, it is strongly recommended to use python >=3.3")
basestring
unicode
warnings.warn("You're using python 2, it is strongly recommended to use python >=3.3")
except NameError:
basestring = str
unicode = str
class MISPAttribute(object):
@ -231,6 +234,19 @@ class MISPEvent(object):
self._reinitialize_event()
self.set_all_values(**e)
def set_date(self, date, ignore_invalid=False):
if isinstance(date, basestring) or isinstance(date, unicode):
self.date = parse(date).date()
elif isinstance(date, datetime.datetime):
self.date = date.date()
elif isinstance(date, datetime.date):
self.date = date
else:
if ignore_invalid:
self.date = datetime.date.today()
else:
raise NewEventError('Invalid format for the date: {} - {}'.format(date, type(date)))
def set_all_values(self, **kwargs):
# Required value
if kwargs.get('info'):
@ -254,14 +270,7 @@ class MISPEvent(object):
if kwargs.get('published') is not None:
self.publish()
if kwargs.get('date'):
if isinstance(kwargs['date'], basestring) or isinstance(kwargs['date'], unicode):
self.date = parse(kwargs['date']).date()
elif isinstance(kwargs['date'], datetime.datetime):
self.date = kwargs['date'].date()
elif isinstance(kwargs['date'], datetime.date):
self.date = kwargs['date']
else:
raise NewEventError('Invalid format for the date: {} - {}'.format(kwargs['date'], type(kwargs['date'])))
self.set_date(kwargs['date'])
if kwargs.get('Attribute'):
for a in kwargs['Attribute']:
attribute = MISPAttribute(self.describe_types)

0
pymisp/tools/__init__.py Normal file
View File

58
pymisp/tools/neo4j.py Normal file
View File

@ -0,0 +1,58 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import glob
import os
from pymisp import MISPEvent
try:
from py2neo import authenticate, Graph, Node, Relationship
has_py2neo = True
except ImportError:
has_py2neo = False
class Neo4j():
def __init__(self, host='localhost:7474', username='neo4j', password='neo4j'):
if not has_py2neo:
raise Exception('py2neo is required, please install: pip install py2neo')
authenticate(host, username, password)
self.graph = Graph()
def load_events_directory(self, directory):
self.events = []
for path in glob.glob(os.path.join(directory, '*.json')):
e = MISPEvent()
e.load(path)
self.import_event(e)
def del_all(self):
self.graph.delete_all()
def import_event(self, event):
tx = self.graph.begin()
event_node = Node('Event', uuid=event.uuid, name=event.info)
# event_node['distribution'] = event.distribution
# event_node['threat_level_id'] = event.threat_level_id
# event_node['analysis'] = event.analysis
# event_node['published'] = event.published
# event_node['date'] = event.date.isoformat()
tx.create(event_node)
for a in event.attributes:
attr_node = Node('Attribute', a.type, uuid=a.uuid)
attr_node['category'] = a.category
attr_node['name'] = a.value
# attr_node['to_ids'] = a.to_ids
# attr_node['comment'] = a.comment
# attr_node['distribution'] = a.distribution
tx.create(attr_node)
member_rel = Relationship(event_node, "is member", attr_node)
tx.create(member_rel)
val = Node('Value', name=a.value)
ev = Relationship(event_node, "has", val)
av = Relationship(attr_node, "is", val)
s = val | ev | av
tx.merge(s)
tx.graph.push(s)
tx.commit()

128
pymisp/tools/openioc.py Normal file
View File

@ -0,0 +1,128 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from pymisp import MISPEvent
try:
from bs4 import BeautifulSoup
has_bs4 = True
except ImportError:
has_bs4 = False
iocMispMapping = {
'DriverItem/DriverName': {'category': 'Artifacts dropped', 'type': 'other', 'comment': 'DriverName.'},
'DnsEntryItem/Host': {'type': 'domain'},
'Email/To': {'type': 'target-email'},
'Email/Date': {'type': 'comment', 'comment': 'EmailDate.'},
'Email/Body': {'type': 'email-subject'},
'Email/From': {'type': 'email-dst'},
'Email/Subject': {'type': 'email-subject'},
'Email/Attachment/Name': {'type': 'email-attachment'},
'FileItem/Md5sum': {'type': 'md5'},
'FileItem/Sha1sum': {'type': 'sha1'},
'TaskItem/Sha1sum': {'type': 'sha1'},
'FileItem/Sha256sum': {'type': 'sha256'},
'FileItem/FileName': {'type': 'filename'},
'FileItem/FullPath': {'type': 'filename'},
'FileItem/FilePath': {'type': 'filename'},
'DriverItem/DriverName': {'type': 'filename'},
'Network/URI': {'type': 'uri'},
'Network/DNS': {'type': 'domain'},
'Network/String': {'type': 'ip-dst'},
'RouteEntryItem/Destination': {'type': 'ip-dst'},
'Network/UserAgent': {'type': 'user-agent'},
'PortItem/localIP': {'type': 'ip-dst'},
'ProcessItem/name': {'type': 'pattern-in-memory', 'comment': 'ProcessName.'},
'ProcessItem/path': {'type': 'pattern-in-memory', 'comment': 'ProcessPath.'},
'ProcessItem/Mutex': {'type': 'mutex'},
'ProcessItem/Pipe/Name': {'type': 'named pipe'},
'ProcessItem/Mutex/Name': {'type': 'mutex', 'comment': 'MutexName.'},
'CookieHistoryItem/HostName': {'type': 'hostname'},
'FormHistoryItem/HostName': {'type': 'hostname'},
'SystemInfoItem/HostName': {'type': 'hostname'},
'UrlHistoryItem/HostName': {'type': 'hostname'},
'DnsEntryItem/RecordName': {'type': 'hostname'},
'DnsEntryItem/Host': {'type': 'hostname'},
# Is it the regkey value?
# 'RegistryItem/Text': {'type': 'regkey', 'RegistryText. '},
'RegistryItem/KeyPath': {'type': 'regkey'},
'RegistryItem/Path': {'type': 'regkey'},
'ServiceItem/name': {'type': 'windows-service-name'},
'ServiceItem/type': {'type': 'pattern-in-memory', 'comment': 'ServiceType. '},
'Snort/Snort': {'type': 'snort'},
}
def extract_field(report, field_name):
data = report.find(field_name.lower())
if data and hasattr(data, 'text'):
return data.text
return None
def load_openioc(openioc):
if not has_bs4:
raise Exception('You need to install BeautifulSoup: pip install bs4')
misp_event = MISPEvent()
with open(openioc, "r") as ioc_file:
iocreport = BeautifulSoup(ioc_file, "lxml")
# Set event fields
info = extract_field(iocreport, 'short_description')
if info:
misp_event.info = info
date = extract_field(iocreport, 'authored_date')
if date:
misp_event.set_date(date)
# Set special attributes
description = extract_field(iocreport, 'description')
if description:
misp_event.add_attribute('comment', description)
author = extract_field(iocreport, 'authored_by')
if author:
misp_event.add_attribute('comment', author)
misp_event = set_all_attributes(iocreport, misp_event)
return misp_event
def get_mapping(openioc_type):
t = openioc_type.lower()
for k, v in iocMispMapping.items():
if k.lower() == t:
return v
return False
def set_all_attributes(openioc, misp_event):
for item in openioc.find_all("indicatoritem"):
attribute_values = {'comment': ''}
if item.find('context'):
mapping = get_mapping(item.find('context')['search'])
if mapping:
attribute_values.update(mapping)
else:
# Unknown mapping, ignoring
# print(item.find('context'))
continue
else:
continue
value = extract_field(openioc, 'Content')
if value:
attribute_values['value'] = value
else:
# No value, ignoring
continue
comment = extract_field(openioc, 'Comment')
if comment:
attribute_values["comment"] = '{} {}'.format(attribute_values["comment"], comment)
misp_event.add_attribute(**attribute_values)
return misp_event

35
pymisp/tools/stix.py Normal file
View File

@ -0,0 +1,35 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
try:
from misp_stix_converter.converters.buildMISPAttribute import buildEvent, open_stix
from misp_stix_converter.converters.convert import MISPtoSTIX
has_misp_stix_converter = True
except ImportError:
has_misp_stix_converter = False
def load_stix(stix, distribution=3, threat_level_id=2, analysis=0):
'''Returns a MISPEvent object from a STIX package'''
if not has_misp_stix_converter:
raise Exception('You need to install misp_stix_converter from https://github.com/MISP/MISP-STIX-Converter')
stix = open_stix(stix)
return buildEvent(stix, distribution=distribution,
threat_level_id=threat_level_id, analysis=analysis)
def make_stix_package(misp_event, to_json=False, to_xml=False):
'''Returns a STIXPackage from a MISPEvent.
Optionally can return the package in json or xml.
'''
if not has_misp_stix_converter:
raise Exception('You need to install misp_stix_converter from https://github.com/MISP/MISP-STIX-Converter')
package = MISPtoSTIX(misp_event)
if to_json:
return package.to_json()
elif to_xml:
return package.to_xml()
else:
return package