diff --git a/MISP_export/ObjectConstructor/AuthFailureMISPObject.py b/MISP_export/ObjectConstructor/AuthFailureMISPObject.py new file mode 100644 index 0000000..4ed48e6 --- /dev/null +++ b/MISP_export/ObjectConstructor/AuthFailureMISPObject.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 + +import time +from pymisp.tools.abstractgenerator import AbstractMISPObjectGenerator + + +class AuthFailureMISPObject(AbstractMISPObjectGenerator): + def __init__(self, dico_val, **kargs): + self._dico_val = dico_val + + # Enforce attribute date with timestamp + super(AuthFailureMISPObject, self).__init__('authentication-failure-report', + default_attributes_parameters={'timestamp': int(time.time())}, + **kargs) + self.name = "authentication-failure-report" + self.generate_attributes() + + def generate_attributes(self): + valid_object_attributes = self._definition['attributes'].keys() + for object_relation, value in self._dico_val.items(): + if object_relation not in valid_object_attributes: + continue + + if object_relation == 'timestamp': + # Date already in ISO format, removing trailing Z + value = value.rstrip('Z') + + if isinstance(value, dict): + self.add_attribute(object_relation, **value) + else: + # uniformize value, sometimes empty array + if isinstance(value, list) and len(value) == 0: + value = '' + self.add_attribute(object_relation, value=value) diff --git a/MISP_export/fromredis.py b/MISP_export/fromredis.py new file mode 100755 index 0000000..47dd20f --- /dev/null +++ b/MISP_export/fromredis.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import argparse +import datetime +import json +import sys +import time + +import redis + +import settings +from generator import FeedGenerator + + +def beautyful_sleep(sleep, additional): + length = 20 + sleeptime = float(sleep) / float(length) + for i in range(length): + temp_string = '|'*i + ' '*(length-i-1) + print('sleeping [{}]\t{}'.format(temp_string, additional), end='\r', sep='') + sys.stdout.flush() + time.sleep(sleeptime) + + +class RedisToMISPFeed: + SUFFIX_SIGH = '_sighting' + SUFFIX_ATTR = '_attribute' + SUFFIX_OBJ = '_object' + SUFFIX_NO = '' + SUFFIX_LIST = [SUFFIX_SIGH, SUFFIX_ATTR, SUFFIX_OBJ, SUFFIX_NO] + + def __init__(self): + self.host = settings.host + self.port = settings.port + self.db = settings.db + self.serv = redis.StrictRedis(self.host, self.port, self.db, decode_responses=True) + + self.generator = FeedGenerator() + + self.keynames = [] + for k in settings.keyname_pop: + for s in self.SUFFIX_LIST: + self.keynames.append(k+s) + + self.keynameError = settings.keyname_error + + self.update_last_action("Init system") + + def consume(self): + self.update_last_action("Started consuming redis") + while True: + for key in self.keynames: + while True: + data = self.pop(key) + if data is None: + break + try: + self.perform_action(key, data) + except Exception as error: + self.save_error_to_redis(error, data) + + try: + beautyful_sleep(5, self.format_last_action()) + except KeyboardInterrupt: + sys.exit(130) + + def pop(self, key): + popped = self.serv.rpop(key) + if popped is None: + return None + try: + popped = json.loads(popped) + except ValueError as error: + self.save_error_to_redis(error, popped) + except ValueError as error: + self.save_error_to_redis(error, popped) + return popped + + def perform_action(self, key, data): + # sighting + if key.endswith(self.SUFFIX_SIGH): + if self.generator.add_sighting_on_attribute(): + self.update_last_action("Added sighting") + else: + self.update_last_action("Error while adding sighting") + + # attribute + elif key.endswith(self.SUFFIX_ATTR): + attr_type = data.pop('type') + attr_value = data.pop('value') + if self.generator.add_attribute_to_event(attr_type, attr_value, **data): + self.update_last_action("Added attribute") + else: + self.update_last_action("Error while adding attribute") + + # object + elif key.endswith(self.SUFFIX_OBJ): + # create the MISP object + obj_name = data.pop('name') + if self.generator.add_object_to_event(obj_name, **data): + self.update_last_action("Added object") + else: + self.update_last_action("Error while adding object") + + else: + # Suffix not provided, try to add anyway + if settings.fallback_MISP_type == 'attribute': + new_key = key + self.SUFFIX_ATTR + # Add atribute type from the config + if 'type' not in data and settings.fallback_attribute_type: + data['type'] = settings.fallback_attribute_type + else: + new_key = None + + elif settings.fallback_MISP_type == 'object': + new_key = key + self.SUFFIX_OBJ + # Add object template name from the config + if 'name' not in data and settings.fallback_object_template_name: + data['name'] = settings.fallback_object_template_name + else: + new_key = None + + elif settings.fallback_MISP_type == 'sighting': + new_key = key + self.SUFFIX_SIGH + + else: + new_key = None + + if new_key is None: + self.update_last_action("Redis key suffix not supported and automatic not configured") + else: + self.perform_action(new_key, data) + + # OTHERS + def update_last_action(self, action): + self.last_action = action + self.last_action_time = datetime.datetime.now() + + def format_last_action(self): + return "Last action: [{}] @ {}".format( + self.last_action, + self.last_action_time.isoformat().replace('T', ' '), + ) + + + def save_error_to_redis(self, error, item): + to_push = {'error': str(error), 'item': str(item)} + print('Error:', str(error), '\nOn adding:', item) + self.serv.lpush(self.keynameError, to_push) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description="Pop item fom redis and add " + + "it to the MISP feed. By default, each action are pushed into a " + + "daily named event. Configuration taken from the file settings.py.") + args = parser.parse_args() + + redisToMISP = RedisToMISPFeed() + redisToMISP.consume() diff --git a/MISP_export/generator.py b/MISP_export/generator.py new file mode 100755 index 0000000..1f3d87a --- /dev/null +++ b/MISP_export/generator.py @@ -0,0 +1,294 @@ +#!/usr/bin/env python3 + +import datetime +import hashlib +import json +import os +import sys +import time +import uuid +import pdb + +from pymisp import MISPEvent + +import settings + + +def get_system_templates(): + """Fetch all MISP-Object template present on the local system. + + Returns: + dict: A dictionary listing all MISP-Object templates + + """ + misp_objects_path = os.path.join( + os.path.abspath(os.path.dirname(sys.modules['pymisp'].__file__)), + 'data', 'misp-objects', 'objects') + + templates = {} + for root, dirs, files in os.walk(misp_objects_path, topdown=False): + for def_file in files: + obj_name = root.split('/')[-1] + template_path = os.path.join(root, def_file) + with open(template_path, 'r') as f: + definition = json.load(f) + templates[obj_name] = definition + return templates + + +def gen_uuid(): + """Generate a random UUID and returns its string representation""" + return str(uuid.uuid4()) + + +class FeedGenerator: + """Helper object to create MISP feed. + + Configuration taken from the file settings.py""" + + def __init__(self): + """This object can be use to easily create a daily MISP-feed. + + It handles the event creation, manifest file and cache file + (hashes.csv). + + """ + self.sys_templates = get_system_templates() + self.constructor_dict = settings.constructor_dict + + self.flushing_interval = settings.flushing_interval + self.flushing_next = time.time() + self.flushing_interval + + self.manifest = {} + self.attributeHashes = [] + + self.daily_event_name = settings.daily_event_name + ' {}' + event_date_str, self.current_event_uuid, self.event_name = self.get_last_event_from_manifest() + temp = [int(x) for x in event_date_str.split('-')] + self.current_event_date = datetime.date(temp[0], temp[1], temp[2]) + self.current_event = self._get_event_from_id(self.current_event_uuid) + + def add_sighting_on_attribute(self, sight_type, attr_uuid, **data): + """Add a sighting on an attribute. + + Not supported for the moment.""" + self.update_daily_event_id() + self._after_addition() + return False + + def add_attribute_to_event(self, attr_type, attr_value, **attr_data): + """Add an attribute to the daily event""" + self.update_daily_event_id() + self.current_event.add_attribute(attr_type, attr_value, **attr_data) + self._add_hash(attr_type, attr_value) + self._after_addition() + return True + + def add_object_to_event(self, obj_name, **data): + """Add an object to the daily event""" + self.update_daily_event_id() + if obj_name not in self.sys_templates: + print('Unkown object template') + return False + + # Get MISP object constructor + obj_constr = self.constructor_dict.get(obj_name, None) + pdb.set_trace() + # Constructor not known, using the generic one + if obj_constr is None: + obj_constr = self.constructor_dict.get('generic') + misp_object = obj_constr(obj_name) + # Fill generic object + for k, v in data.items(): + # attribute is not in the object template definition + if k not in self.sys_templates[obj_name]['attributes']: + # add it with type text + misp_object.add_attribute(k, **{'value': v, 'type': 'text'}) + else: + misp_object.add_attribute(k, **{'value': v}) + + else: + misp_object = obj_constr(data) + + self.current_event.add_object(misp_object) + for attr_type, attr_value in data.items(): + self._add_hash(attr_type, attr_value) + + self._after_addition() + return True + + def _after_addition(self): + """Write event on disk""" + now = time.time() + if self.flushing_next <= now: + self.flush_event() + self.flushing_next = now + self.flushing_interval + + # Cache + def _add_hash(self, attr_type, attr_value): + if ('|' in attr_type or attr_type == 'malware-sample'): + split = attr_value.split('|') + self.attributeHashes.append([ + hashlib.md5(str(split[0]).encode("utf-8")).hexdigest(), + self.current_event_uuid + ]) + self.attributeHashes.append([ + hashlib.md5(str(split[1]).encode("utf-8")).hexdigest(), + self.current_event_uuid + ]) + else: + self.attributeHashes.append([ + hashlib.md5(str(attr_value).encode("utf-8")).hexdigest(), + self.current_event_uuid + ]) + + # Manifest + def _init_manifest(self): + # check if outputdir exists and try to create it if not + if not os.path.exists(settings.outputdir): + try: + os.makedirs(settings.outputdir) + except PermissionError as error: + print(error) + print("Please fix the above error and try again.") + sys.exit(126) + + # create an empty manifest + try: + with open(os.path.join(settings.outputdir, 'manifest.json'), 'w'): + pass + except PermissionError as error: + print(error) + print("Please fix the above error and try again.") + sys.exit(126) + + # create new event and save manifest + self.create_daily_event() + + def flush_event(self, new_event=None): + print('Writting event on disk'+' '*50) + if new_event is not None: + event_uuid = new_event['uuid'] + event = new_event + else: + event_uuid = self.current_event_uuid + event = self.current_event + + eventFile = open(os.path.join(settings.outputdir, event_uuid+'.json'), 'w') + eventFile.write(event.to_json()) + eventFile.close() + + self.save_hashes() + + def save_manifest(self): + try: + manifestFile = open(os.path.join(settings.outputdir, 'manifest.json'), 'w') + manifestFile.write(json.dumps(self.manifest)) + manifestFile.close() + print('Manifest saved') + except Exception as e: + print(e) + sys.exit('Could not create the manifest file.') + + def save_hashes(self): + if len(self.attributeHashes) == 0: + return False + try: + hashFile = open(os.path.join(settings.outputdir, 'hashes.csv'), 'a') + for element in self.attributeHashes: + hashFile.write('{},{}\n'.format(element[0], element[1])) + hashFile.close() + self.attributeHashes = [] + print('Hash saved' + ' '*30) + except Exception as e: + print(e) + sys.exit('Could not create the quick hash lookup file.') + + def _addEventToManifest(self, event): + event_dict = event.to_dict() + tags = [] + for eventTag in event_dict.get('EventTag', []): + tags.append({'name': eventTag['Tag']['name'], + 'colour': eventTag['Tag']['colour']}) + return { + 'Orgc': event_dict.get('Orgc', []), + 'Tag': tags, + 'info': event_dict['info'], + 'date': event_dict['date'], + 'analysis': event_dict['analysis'], + 'threat_level_id': event_dict['threat_level_id'], + 'timestamp': event_dict.get('timestamp', int(time.time())) + } + + def get_last_event_from_manifest(self): + """Retreive last event from the manifest. + + If the manifest doesn't exists or if it is empty, initialize it. + + """ + try: + manifest_path = os.path.join(settings.outputdir, 'manifest.json') + with open(manifest_path, 'r') as f: + man = json.load(f) + dated_events = [] + for event_uuid, event_json in man.items(): + # add events to manifest + self.manifest[event_uuid] = event_json + dated_events.append([ + event_json['date'], + event_uuid, + event_json['info'] + ]) + # Sort by date then by event name + dated_events.sort(key=lambda k: (k[0], k[2]), reverse=True) + return dated_events[0] + except FileNotFoundError as e: + print('Manifest not found, generating a fresh one') + self._init_manifest() + return self.get_last_event_from_manifest() + + # DAILY + def update_daily_event_id(self): + if self.current_event_date != datetime.date.today(): # create new event + # save current event on disk + self.flush_event() + self.current_event = self.create_daily_event() + self.current_event_date = datetime.date.today() + self.current_event_uuid = self.current_event.get('uuid') + self.event_name = self.current_event.info + + def _get_event_from_id(self, event_uuid): + with open(os.path.join(settings.outputdir, '%s.json' % event_uuid), 'r') as f: + event_dict = json.load(f) + event = MISPEvent() + event.from_dict(**event_dict) + return event + + def create_daily_event(self): + new_uuid = gen_uuid() + today = str(datetime.date.today()) + event_dict = { + 'uuid': new_uuid, + 'id': len(self.manifest)+1, + 'Tag': settings.Tag, + 'info': self.daily_event_name.format(today), + 'analysis': settings.analysis, # [0-2] + 'threat_level_id': settings.threat_level_id, # [1-4] + 'published': settings.published, + 'date': today + } + event = MISPEvent() + event.from_dict(**event_dict) + + # reference org + org_dict = {} + org_dict['name'] = settings.org_name + org_dict['uuid'] = settings.org_uuid + event['Orgc'] = org_dict + + # save event on disk + self.flush_event(new_event=event) + # add event to manifest + self.manifest[event['uuid']] = self._addEventToManifest(event) + self.save_manifest() + return event diff --git a/MISP_export/install.sh b/MISP_export/install.sh new file mode 100644 index 0000000..0e65dcb --- /dev/null +++ b/MISP_export/install.sh @@ -0,0 +1,4 @@ +#!/bin/bash +virtualenv -p python3 serv-env +. ./serv-env/bin/activate +pip3 install -U flask Flask-AutoIndex redis diff --git a/MISP_export/server.py b/MISP_export/server.py new file mode 100755 index 0000000..1dd6873 --- /dev/null +++ b/MISP_export/server.py @@ -0,0 +1,12 @@ +#!/usr/bin/env python3 + +import os.path +from flask import Flask +from flask_autoindex import AutoIndex +from settings import outputdir + +app = Flask(__name__) +AutoIndex(app, browse_root=os.path.join(os.path.curdir, outputdir)) + +if __name__ == '__main__': + app.run(host='0.0.0.0') diff --git a/MISP_export/settings.default.py b/MISP_export/settings.default.py new file mode 100755 index 0000000..2d12744 --- /dev/null +++ b/MISP_export/settings.default.py @@ -0,0 +1,65 @@ +""" REDIS RELATED """ +# Your redis server +host='127.0.0.1' +port=6385 +db=3 +## The keynames to POP element from +keyname_pop=['authf'] + +# OTHERS +## If key prefix not provided, data will be added as either object, attribute or sighting +fallback_MISP_type = 'object' +### How to handle the fallback +fallback_object_template_name = 'generic' # MISP-Object only +fallback_attribute_category = 'comment' # MISP-Attribute only + +## How frequent the event should be written on disk +#flushing_interval=5*60 +flushing_interval=15 +## The redis list keyname in which to put items that generated an error +keyname_error='feed-generation-error' + +""" FEED GENERATOR CONFIGURATION """ + +# The output dir for the feed. This will drop a lot of files, so make +# sure that you use a directory dedicated to the feed +outputdir = 'output' + +# Event meta data +## Required +### The organisation id that generated this feed +org_name='myOrg' +### Your organisation UUID +org_uuid='' +### The daily event name to be used in MISP. +### (e.g. honeypot_1, will produce each day an event of the form honeypot_1 dd-mm-yyyy) +daily_event_name='PyMISP default event name' + +## Optional +analysis=0 +threat_level_id=3 +published=False +Tag=[ + { + "colour": "#ffffff", + "name": "tlp:white" + }, + { + "colour": "#ff00ff", + "name": "my:custom:feed" + } +] + +# MISP Object constructor +from ObjectConstructor.AuthFailureMISPObject import AuthFailureMISPObject +#from pymisp.pymisp.tools import GenericObjectGenerator +from pymisp.tools import GenericObjectGenerator + +constructor_dict = { + 'authentication-failure-report': AuthFailureMISPObject, + 'generic': GenericObjectGenerator +} + +# Others +## Redis pooling time +sleep=60 diff --git a/logcompiler/compiler.go b/logcompiler/compiler.go index 4e4740a..fa62835 100644 --- a/logcompiler/compiler.go +++ b/logcompiler/compiler.go @@ -20,6 +20,7 @@ type ( SetReader(io.Reader) Pull(chan error) Flush() error + MISPexport() error } // CompilerStruct will implements Compiler, and should be embedded in diff --git a/logcompiler/sshd.go b/logcompiler/sshd.go index 3b6d6f6..355f77a 100644 --- a/logcompiler/sshd.go +++ b/logcompiler/sshd.go @@ -39,6 +39,12 @@ type GrokedSSHD struct { SshdInvalidUser string `json:"sshd_invalid_user"` } +type MISP_auth_failure_sshd_username struct { + mtype string `json:"type"` + username string `json:"username"` + total string `json:"total"` +} + // Flush recomputes statistics and recompile HTML output // TODO : review after refacto func (s *SSHDCompiler) Flush() error { @@ -523,6 +529,36 @@ func csvStats(s *SSHDCompiler, v string) error { return nil } +func MISPexport(s *SSHDCompiler) error { + + today := time.Now() + dstr := fmt.Sprintf("%v%v%v", today.Year(), fmt.Sprintf("%02d", int(today.Month())), fmt.Sprintf("%02d", int(today.Day()))) + + r0 := *s.r0 + r1 := *s.r1 + zrank, err := redis.Strings(r0.Do("ZRANGEBYSCORE", fmt.Sprintf("%q:statsusername", dstr), "-inf", "+inf", "WITHSCORES")) + if err != nil { + return err + } + + mispobject := new(MISP_auth_failure_sshd_username) + mispobject.mtype = "sshd" + for k, v := range zrank { + // pair: keys + if (k % 2) == 0 { + mispobject.username = v + // even: values + } else { + mispobject.total = v + } + } + + b, err := json.Marshal(mispobject) + r1.Do("LPUSH", "authf_object", b) + + return nil +} + func plotStats(s *SSHDCompiler, v string) error { r := *s.r0 zrank, err := redis.Strings(r.Do("ZRANGEBYSCORE", v, "-inf", "+inf", "WITHSCORES"))