From 188c452a390a63e13ebb1629de972837ae3da269 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Thu, 8 Mar 2018 12:01:35 +0100 Subject: [PATCH 01/17] Init draft of redis to feed --- examples/feed-generator-from-redis/README.md | 13 + .../feed-generator-from-redis/generate.py | 315 ++++++++++++++++++ .../feed-generator-from-redis/output/empty | 0 .../settings.default.py | 38 +++ 4 files changed, 366 insertions(+) create mode 100644 examples/feed-generator-from-redis/README.md create mode 100755 examples/feed-generator-from-redis/generate.py create mode 100644 examples/feed-generator-from-redis/output/empty create mode 100755 examples/feed-generator-from-redis/settings.default.py diff --git a/examples/feed-generator-from-redis/README.md b/examples/feed-generator-from-redis/README.md new file mode 100644 index 0000000..8bdd7cc --- /dev/null +++ b/examples/feed-generator-from-redis/README.md @@ -0,0 +1,13 @@ +# What + +This python script can be used to generate a MISP feed based on data stored in redis. + +# Installation + +```` +git clone https://github.com/CIRCL/PyMISP +cd examples/feed-generator-from-redis +cp settings-default.py settings.py +vi settings.py #adjust your settings +python3 generate.py +```` diff --git a/examples/feed-generator-from-redis/generate.py b/examples/feed-generator-from-redis/generate.py new file mode 100755 index 0000000..3734e8f --- /dev/null +++ b/examples/feed-generator-from-redis/generate.py @@ -0,0 +1,315 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import sys +import json +import os +import hashlib +import argparse +import datetime, time +import uuid +import threading +import redis +from pymisp import MISPEvent, MISPAttribute +from redis import StrictRedis as Redis +import settings + +evtObj=thr=None # animation thread + +def gen_uuid(): + return str(uuid.uuid4()) + +def processing_animation(evtObj, buffer_state, refresh_rate=5): + i = 0 + buffer_state_str = 'attributes: {}, objects: {}, sightings: {}'.format(buffer_state['attribute'], buffer_state['object'], buffer_state['sighting']) + while True: + if evtObj.is_set(): + print(" "*(len(buffer_state_str)+20), end="\r", sep="") # overwrite last characters + sys.stdout.flush() + return + i += 1 + print("Remaining: { %s }\t" % buffer_state_str + "/-\|"[i%4], end="\r", sep="") + sys.stdout.flush() + time.sleep(1.0/float(refresh_rate)) + +def beautyful_sleep(sleep): + length = 20 + sleeptime = float(sleep) / float(length) + for i in range(length): + temp_string = '|'*i + ' '*(length-i-1) + print('sleeping [{}]'.format(temp_string), end='\r', sep='') + sys.stdout.flush() + time.sleep(sleeptime) + + + + +class RedisToMISPFeed: + SUFFIX_SIGH = '_sighting' + SUFFIX_ATTR = '_attribute' + SUFFIX_OBJ = '_object' + SUFFIX_LIST = [SUFFIX_SIGH, SUFFIX_ATTR, SUFFIX_OBJ] + + def __init__(self): + self.host = settings.host + self.port = settings.port + self.db = settings.db + self.serv = redis.StrictRedis(self.host, self.port, self.db, decode_responses=True) + + self.keynames = [] + for k in settings.keyname_pop: + for s in self.SUFFIX_LIST: + self.keynames.append(k+s) + + self.sleep = settings.sleep + self.flushing_interval = settings.flushing_interval + self.flushing_next = time.time() + self.flushing_interval + + self.manifest = {} + self.attributeHashes = [] + + self.keynameError = settings.keyname_error + self.allow_animation = settings.allow_animation + + self.daily_event_name = settings.daily_event_name + ' {}' + _, self.current_event_uuid, self.event_name = self.get_last_event_from_manifest() + self.current_date = datetime.date.today() + self.current_event = self.get_event_from_id(self.current_event_uuid) + + global evtObj, thr + self.evtObj = evtObj + self.thr = thr + + def consume(self): + while True: + flag_empty = True + for key in self.keynames: + while True: + data = self.pop(key) + if data is None: + break + try: + self.perform_action(key, data) + except Exception as error: + self.save_error_to_redis(error, data) + flag_empty = False + + + if flag_empty and self.flushing_next <= time.time(): + self.flush_event() + flushing_next = time.time() + flushing_interval + + beautyful_sleep(5) + + def pop(self, key): + popped = self.serv.rpop(key) + if popped is None: + return None + try: + popped = json.loads(popped) + except ValueError as error: + self.save_error_to_redis(error, popped) + except ValueError as error: + self.save_error_to_redis(error, popped) + return popped + + + def perform_action(self, key, data): + self.update_daily_event_id() + + # sighting + if key.endswith(self.SUFFIX_SIGH): + pass + #r = self.pymisphelper.add_sighting_per_json(data) + + # attribute + elif key.endswith(self.SUFFIX_ATTR): + attr_type = data.pop('type') + attr_value = data.pop('value') + self.current_event.add_attribute(attr_type, attr_value, **data) + self.add_hash(attr_type, attr_value) + + # object + elif key.endswith(self.SUFFIX_OBJ): + self.current_event.add_object(**data) + for attr_type, attr_value in data.items(): + self.add_hash(attr_type, attr_value) + + else: + raise NoValidKey("Can't define action to perform") + + if r is not None and 'errors' in r: + self.save_error_to_redis(r, data) + + def add_hash(self, attr_type, attr_value): + if ('|' in attr_type or attr_type == 'malware-sample'): + split = attr_value.split('|') + self.attributeHashes.append([hashlib.md5(split[0].encode("utf-8")).hexdigest(), self.current_event_uuid]) + self.attributeHashes.append([hashlib.md5(split[1].encode("utf-8")).hexdigest(), self.current_event_uuid]) + else: + self.attributeHashes.append([hashlib.md5(attr_value.encode("utf-8")).hexdigest(), self.current_event_uuid]) + + # Manifest + def init_manifest(self): + # create an empty manifest + with open(os.path.join(settings.outputdir, 'manifest.json'), 'w') as f: + pass + # create new event and save manifest + self.create_daily_event() + + + def flush_event(self, new_event=None): + print('Writting event on disk') + self.print_processing() + if new_event is not None: + event_uuid = new_event['uuid'] + event = new_event + else: + event_uuid = self.current_event_uuid + event = self.current_event + + eventFile = open(os.path.join(settings.outputdir, event_uuid + '.json'), 'w') + eventFile.write(event.to_json()) + eventFile.close() + if self.allow_animation: + self.evtObj.set() + self.thr.join() + print('Event written') + + def saveManifest(self): + try: + manifestFile = open(os.path.join(settings.outputdir, 'manifest.json'), 'w') + manifestFile.write(json.dumps(self.manifest)) + manifestFile.close() + print('Manifest saved') + except Exception as e: + print(e) + sys.exit('Could not create the manifest file.') + + def saveHashes(): + if len(self.attributeHashes) == 0: + return False + try: + hashFile = open(os.path.join(settings.outputdir, 'hashes.csv'), 'a') + for element in self.attributeHashes: + hashFile.write('{},{}\n'.format(element[0], element[1])) + hashFile.close() + self.attributeHashes = [] + print('Hash saved') + except Exception as e: + print(e) + sys.exit('Could not create the quick hash lookup file.') + + + def __addEventToManifest(self, event): + event_dict = event.to_dict()['Event'] + tags = [] + for eventTag in event_dict.get('EventTag', []): + tags.append({'name': eventTag['Tag']['name'], + 'colour': eventTag['Tag']['colour']}) + return { + 'Orgc': event_dict.get('Orgc', []), + 'Tag': tags, + 'info': event_dict['info'], + 'date': event_dict['date'], + 'analysis': event_dict['analysis'], + 'threat_level_id': event_dict['threat_level_id'], + 'timestamp': event_dict.get('timestamp', int(time.time())) + } + + # Retreive last event from the manifest, if the manifest doesn't exists + # or if it is empty, initialize it. + def get_last_event_from_manifest(self): + try: + with open(os.path.join(settings.outputdir, 'manifest.json'), 'r') as f: + man = json.load(f) + dated_events = [] + for event_uuid, event_json in man.items(): + # add events to manifest + self.manifest[event_uuid] = event_json + dated_events.append([event_json['date'], event_uuid, event_json['info']]) + dated_events.sort(key=lambda k: (k[0], k[2]), reverse=True) # sort by date then by event name + return dated_events[0] + except FileNotFoundError as e: + print('Manifest not found, generating a fresh one') + self.init_manifest() + return self.get_last_event_from_manifest() + + # DAILY + def update_daily_event_id(self): + if self.current_date != datetime.date.today(): # create new event + # save current event on disk + self.flush_event() + self.current_event = create_daily_event() + self.current_event_uuid = self.current_event.get('uuid') + self.event_name = self.current_event.info + + def get_event_from_id(self, event_uuid): + with open(os.path.join(settings.outputdir, '%s.json' % event_uuid), 'r') as f: + event_dict = json.load(f)['Event'] + event = MISPEvent() + event.from_dict(**event_dict) + return event + + def create_daily_event(self): + new_uuid = gen_uuid() + today = str(datetime.date.today()) + event_dict = { + 'uuid': new_uuid, + 'id': len(self.manifest)+1, + 'Tag': settings.Tag, + 'info': self.daily_event_name.format(today), + 'analysis': settings.analysis, # [0-2] + 'threat_level_id': settings.threat_level_id, # [1-4] + 'published': settings.published, + 'date': today + } + event = MISPEvent() + event.from_dict(**event_dict) + + # reference org + org_dict = {} + org_dict['name'] = settings.org_name + org_dict['uui'] = settings.org_uuid + event['Orgc'] = org_dict + + # save event on disk + self.flush_event(new_event=event) + # add event to manifest + self.manifest[event['uuid']] = self.__addEventToManifest(event) + self.saveManifest() + return event + + # OTHERS + def get_buffer_state(self): + buffer_state = {'attribute': 0, 'object': 0, 'sighting': 0} + for k in self.keynames: + _ , suffix = k.rsplit('_', 1) + buffer_state[suffix] += self.serv.llen(k) + return buffer_state + + + def print_processing(self): + if self.allow_animation: + buff_states = self.get_buffer_state() + self.evtObj = threading.Event() + self.thr = threading.Thread(name="processing-animation", target=processing_animation, args=(self.evtObj, buff_states, )) + self.thr.start() + + def save_error_to_redis(self, error, item): + to_push = {'error': str(error), 'item': str(item)} + print('Error:', str(error), '\nOn adding:', item) + self.serv.lpush(self.keynameError, to_push) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description="Pop item fom redis and add it to the MISP feed. By default, each action are pushed into a daily named event. Configuration taken from the file settings.py.") + args = parser.parse_args() + + redisToMISP = RedisToMISPFeed() + try: + redisToMISP.consume() + except (KeyboardInterrupt, SystemExit): + if evtObj is not None: + evtObj.set() + thr.join() diff --git a/examples/feed-generator-from-redis/output/empty b/examples/feed-generator-from-redis/output/empty new file mode 100644 index 0000000..e69de29 diff --git a/examples/feed-generator-from-redis/settings.default.py b/examples/feed-generator-from-redis/settings.default.py new file mode 100755 index 0000000..570d483 --- /dev/null +++ b/examples/feed-generator-from-redis/settings.default.py @@ -0,0 +1,38 @@ +# Your redis server +host='127.0.0.1' +port=6379 +db=0 +## The keynames to POP element from +keyname_pop='misp_feed_generator_key' + +# The output dir for the feed. This will drop a lot of files, so make +# sure that you use a directory dedicated to the feed +outputdir = 'output' + +# Event meta data +## Required +### The organisation id that generated this feed +org_name='myOrg' +### Your organisation UUID +org_uuid='' +### The daily event name to be used in MISP. (e.g. honeypot_1, will produce each day an event of the form honeypot_1 dd-mm-yyyy) +daily_event_name='PyMISP default event name' + +## Optional +analysis=0 +threat_level_id=3 +published=False +Tag=[{ + "colour": "#ffffff", + "name": "tlp:white" + }] + +# Others +## Redis pooling time +sleep=1 +## The redis list keyname in which to put items that generated an error +keyname_error='feed-generation-error' +## Display an animation while adding element to MISP +allow_animation=True +## How frequent the event should be written on disk +flushing_interval=60*5 From 22efb64f147c7ee1f6f8c0eb227fbc4917b6a8f7 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Thu, 8 Mar 2018 14:19:28 +0100 Subject: [PATCH 02/17] Overhall seems to work, need testing --- .../feed-generator-from-redis/generate.py | 20 +++++++++---------- .../feed-generator-from-redis/output/empty | 0 .../settings.default.py | 5 +++-- 3 files changed, 12 insertions(+), 13 deletions(-) delete mode 100644 examples/feed-generator-from-redis/output/empty diff --git a/examples/feed-generator-from-redis/generate.py b/examples/feed-generator-from-redis/generate.py index 3734e8f..391d598 100755 --- a/examples/feed-generator-from-redis/generate.py +++ b/examples/feed-generator-from-redis/generate.py @@ -97,7 +97,7 @@ class RedisToMISPFeed: if flag_empty and self.flushing_next <= time.time(): self.flush_event() - flushing_next = time.time() + flushing_interval + flushing_next = time.time() + self.flushing_interval beautyful_sleep(5) @@ -120,7 +120,6 @@ class RedisToMISPFeed: # sighting if key.endswith(self.SUFFIX_SIGH): pass - #r = self.pymisphelper.add_sighting_per_json(data) # attribute elif key.endswith(self.SUFFIX_ATTR): @@ -138,16 +137,14 @@ class RedisToMISPFeed: else: raise NoValidKey("Can't define action to perform") - if r is not None and 'errors' in r: - self.save_error_to_redis(r, data) def add_hash(self, attr_type, attr_value): if ('|' in attr_type or attr_type == 'malware-sample'): split = attr_value.split('|') - self.attributeHashes.append([hashlib.md5(split[0].encode("utf-8")).hexdigest(), self.current_event_uuid]) - self.attributeHashes.append([hashlib.md5(split[1].encode("utf-8")).hexdigest(), self.current_event_uuid]) + self.attributeHashes.append([hashlib.md5(str(split[0]).encode("utf-8")).hexdigest(), self.current_event_uuid]) + self.attributeHashes.append([hashlib.md5(str(split[1]).encode("utf-8")).hexdigest(), self.current_event_uuid]) else: - self.attributeHashes.append([hashlib.md5(attr_value.encode("utf-8")).hexdigest(), self.current_event_uuid]) + self.attributeHashes.append([hashlib.md5(str(attr_value).encode("utf-8")).hexdigest(), self.current_event_uuid]) # Manifest def init_manifest(self): @@ -159,7 +156,7 @@ class RedisToMISPFeed: def flush_event(self, new_event=None): - print('Writting event on disk') + print('Writting event on disk'+' '*20) self.print_processing() if new_event is not None: event_uuid = new_event['uuid'] @@ -171,10 +168,11 @@ class RedisToMISPFeed: eventFile = open(os.path.join(settings.outputdir, event_uuid + '.json'), 'w') eventFile.write(event.to_json()) eventFile.close() + + self.saveHashes() if self.allow_animation: self.evtObj.set() self.thr.join() - print('Event written') def saveManifest(self): try: @@ -186,7 +184,7 @@ class RedisToMISPFeed: print(e) sys.exit('Could not create the manifest file.') - def saveHashes(): + def saveHashes(self): if len(self.attributeHashes) == 0: return False try: @@ -195,7 +193,7 @@ class RedisToMISPFeed: hashFile.write('{},{}\n'.format(element[0], element[1])) hashFile.close() self.attributeHashes = [] - print('Hash saved') + print('Hash saved' + ' '*30) except Exception as e: print(e) sys.exit('Could not create the quick hash lookup file.') diff --git a/examples/feed-generator-from-redis/output/empty b/examples/feed-generator-from-redis/output/empty deleted file mode 100644 index e69de29..0000000 diff --git a/examples/feed-generator-from-redis/settings.default.py b/examples/feed-generator-from-redis/settings.default.py index 570d483..49c121c 100755 --- a/examples/feed-generator-from-redis/settings.default.py +++ b/examples/feed-generator-from-redis/settings.default.py @@ -3,7 +3,8 @@ host='127.0.0.1' port=6379 db=0 ## The keynames to POP element from -keyname_pop='misp_feed_generator_key' +#keyname_pop='misp_feed_generator_key' +keyname_pop=['cowrie'] # The output dir for the feed. This will drop a lot of files, so make # sure that you use a directory dedicated to the feed @@ -35,4 +36,4 @@ keyname_error='feed-generation-error' ## Display an animation while adding element to MISP allow_animation=True ## How frequent the event should be written on disk -flushing_interval=60*5 +flushing_interval=2*5 From c04a3709f9f96c85d1941b1ecb5ea9a67f1c4cc1 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Thu, 8 Mar 2018 17:33:39 +0100 Subject: [PATCH 03/17] Added support of MISP Object --- .../feed-generator-from-redis/generate.py | 37 ++++++++++++++++++- examples/feed-generator-from-redis/server.py | 12 ++++++ .../settings.default.py | 9 +++-- 3 files changed, 53 insertions(+), 5 deletions(-) create mode 100755 examples/feed-generator-from-redis/server.py diff --git a/examples/feed-generator-from-redis/generate.py b/examples/feed-generator-from-redis/generate.py index 391d598..8e89d69 100755 --- a/examples/feed-generator-from-redis/generate.py +++ b/examples/feed-generator-from-redis/generate.py @@ -10,12 +10,31 @@ import datetime, time import uuid import threading import redis -from pymisp import MISPEvent, MISPAttribute + from redis import StrictRedis as Redis import settings +from pymisp import MISPEvent, MISPAttribute +from pymisp.tools import GenericObjectGenerator + evtObj=thr=None # animation thread +def get_system_templates(): + misp_objects_path = os.path.join( + os.path.abspath(os.path.dirname(sys.modules['pymisp'].__file__)), + 'data', 'misp-objects', 'objects') + + templates = {} + for root, dirs, files in os.walk(misp_objects_path, topdown=False): + for def_file in files: + obj_name = root.split('/')[-1] + template_path = os.path.join(root, def_file) + with open(template_path, 'r') as f: + definition = json.load(f) + templates[obj_name] = definition + return templates + + def gen_uuid(): return str(uuid.uuid4()) @@ -61,6 +80,9 @@ class RedisToMISPFeed: for s in self.SUFFIX_LIST: self.keynames.append(k+s) + # get all templates + self.sys_templates = get_system_templates() + self.sleep = settings.sleep self.flushing_interval = settings.flushing_interval self.flushing_next = time.time() + self.flushing_interval @@ -130,10 +152,21 @@ class RedisToMISPFeed: # object elif key.endswith(self.SUFFIX_OBJ): - self.current_event.add_object(**data) + # create the MISP object + obj_name = data.pop('name') + misp_object = GenericObjectGenerator(obj_name) + for k, v in data.items(): + if k not in self.sys_templates[obj_name]['attributes']: # attribute is not in the object template definition + # add it with type text + misp_object.add_attribute(k, **{'value': v, 'type': 'text'}) + else: + misp_object.add_attribute(k, **{'value': v}) + + self.current_event.add_object(misp_object) for attr_type, attr_value in data.items(): self.add_hash(attr_type, attr_value) + else: raise NoValidKey("Can't define action to perform") diff --git a/examples/feed-generator-from-redis/server.py b/examples/feed-generator-from-redis/server.py new file mode 100755 index 0000000..5a2ee16 --- /dev/null +++ b/examples/feed-generator-from-redis/server.py @@ -0,0 +1,12 @@ +#!/usr/bin/env python3 + +import os.path +from flask import Flask +from flask.ext.autoindex import AutoIndex +from settings import outputdir + +app = Flask(__name__) +AutoIndex(app, browse_root=os.path.join(os.path.curdir, outputdir)) + +if __name__ == '__main__': + app.run(host='0.0.0.0') diff --git a/examples/feed-generator-from-redis/settings.default.py b/examples/feed-generator-from-redis/settings.default.py index 49c121c..c3fce6d 100755 --- a/examples/feed-generator-from-redis/settings.default.py +++ b/examples/feed-generator-from-redis/settings.default.py @@ -26,14 +26,17 @@ published=False Tag=[{ "colour": "#ffffff", "name": "tlp:white" - }] + }, + "colour": "#ff00ff", + "name": "my:custom:feed" + ] # Others ## Redis pooling time -sleep=1 +sleep=60 ## The redis list keyname in which to put items that generated an error keyname_error='feed-generation-error' ## Display an animation while adding element to MISP allow_animation=True ## How frequent the event should be written on disk -flushing_interval=2*5 +flushing_interval=5*60 From 61ce67cd1c1804f75e34aee79ff1201317f61e16 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Thu, 8 Mar 2018 17:39:14 +0100 Subject: [PATCH 04/17] Added install script --- examples/feed-generator-from-redis/install.sh | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 examples/feed-generator-from-redis/install.sh diff --git a/examples/feed-generator-from-redis/install.sh b/examples/feed-generator-from-redis/install.sh new file mode 100644 index 0000000..0e65dcb --- /dev/null +++ b/examples/feed-generator-from-redis/install.sh @@ -0,0 +1,4 @@ +#!/bin/bash +virtualenv -p python3 serv-env +. ./serv-env/bin/activate +pip3 install -U flask Flask-AutoIndex redis From fdaa4c790c3fbe54cc241cd74c652dd5b3fac4bf Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Fri, 9 Mar 2018 15:31:13 +0100 Subject: [PATCH 05/17] Creation of the generator object which permit to easily add attributes and objects to daily events, stored as a MISP feed. Plus, script fromredis which pops queue element in redis to put them in the feed --- .../feed-generator-from-redis/fromredis.py | 152 ++++++++ .../feed-generator-from-redis/generate.py | 346 ------------------ .../feed-generator-from-redis/generator.py | 236 ++++++++++++ .../settings.default.py | 25 +- 4 files changed, 404 insertions(+), 355 deletions(-) create mode 100755 examples/feed-generator-from-redis/fromredis.py delete mode 100755 examples/feed-generator-from-redis/generate.py create mode 100755 examples/feed-generator-from-redis/generator.py diff --git a/examples/feed-generator-from-redis/fromredis.py b/examples/feed-generator-from-redis/fromredis.py new file mode 100755 index 0000000..f4f68b7 --- /dev/null +++ b/examples/feed-generator-from-redis/fromredis.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import sys +import json +import argparse +import datetime +import time +import redis + +import settings + +from generator import FeedGenerator + + +def beautyful_sleep(sleep, additional): + length = 20 + sleeptime = float(sleep) / float(length) + for i in range(length): + temp_string = '|'*i + ' '*(length-i-1) + print('sleeping [{}]\t{}'.format(temp_string, additional), end='\r', sep='') + sys.stdout.flush() + time.sleep(sleeptime) + + +class RedisToMISPFeed: + SUFFIX_SIGH = '_sighting' + SUFFIX_ATTR = '_attribute' + SUFFIX_OBJ = '_object' + SUFFIX_LIST = [SUFFIX_SIGH, SUFFIX_ATTR, SUFFIX_OBJ] + + def __init__(self): + self.host = settings.host + self.port = settings.port + self.db = settings.db + self.serv = redis.StrictRedis(self.host, self.port, self.db, decode_responses=True) + + self.flushing_interval = settings.flushing_interval + self.flushing_next = time.time() + self.flushing_interval + + self.generator = FeedGenerator() + + self.keynames = [] + for k in settings.keyname_pop: + for s in self.SUFFIX_LIST: + self.keynames.append(k+s) + + self.keynameError = settings.keyname_error + + self.last_flush = datetime.datetime.now() + self.update_last_action("Init system") + + def consume(self): + self.update_last_action("Started consuming redis") + while True: + flag_empty = True + for key in self.keynames: + while True: + data = self.pop(key) + if data is None: + break + try: + self.perform_action(key, data) + except Exception as error: + self.save_error_to_redis(error, data) + flag_empty = False + + # Define when to write event on disk + if flag_empty and self.flushing_next <= time.time() and self.last_action_time>self.last_flush: + self.update_last_action('Flushed on disk') + self.generator.flush_event() + self.flushing_next = time.time() + self.flushing_interval + self.last_flush = datetime.datetime.now() + + beautyful_sleep(5, self.format_last_action()) + + def pop(self, key): + popped = self.serv.rpop(key) + if popped is None: + return None + try: + popped = json.loads(popped) + except ValueError as error: + self.save_error_to_redis(error, popped) + except ValueError as error: + self.save_error_to_redis(error, popped) + return popped + + def perform_action(self, key, data): + # sighting + if key.endswith(self.SUFFIX_SIGH): + if self.generator.add_sighting_on_attribute(): + self.update_last_action("Added sighting") + else: + self.update_last_action("Error while adding sighting") + + # attribute + elif key.endswith(self.SUFFIX_ATTR): + attr_type = data.pop('type') + attr_value = data.pop('value') + if self.generator.add_attribute_to_event(attr_type, attr_value, **data): + self.update_last_action("Added attribute") + else: + self.update_last_action("Error while adding attribute") + + # object + elif key.endswith(self.SUFFIX_OBJ): + # create the MISP object + obj_name = data.pop('name') + if self.generator.add_object_to_event(obj_name, **data): + self.update_last_action("Added object") + else: + self.update_last_action("Error while adding object") + + else: + # Suffix not valid + self.update_last_action("Redis key suffix not supported") + + # OTHERS + def update_last_action(self, action): + self.last_action = action + self.last_action_time = datetime.datetime.now() + + def format_last_action(self): + temp = datetime.datetime.now() - self.last_flush + return "Last action: [{}] @ {}.\tLast flush: {} ago".format( + self.last_action, + self.last_action_time.isoformat().replace('T', ' '), + str(temp).split('.')[0] + ) + + def get_buffer_state(self): + buffer_state = {'attribute': 0, 'object': 0, 'sighting': 0} + for k in self.keynames: + _ , suffix = k.rsplit('_', 1) + buffer_state[suffix] += self.serv.llen(k) + return buffer_state + + + def save_error_to_redis(self, error, item): + to_push = {'error': str(error), 'item': str(item)} + print('Error:', str(error), '\nOn adding:', item) + self.serv.lpush(self.keynameError, to_push) + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description="Pop item fom redis and add " + + "it to the MISP feed. By default, each action are pushed into a " + + "daily named event. Configuration taken from the file settings.py.") + args = parser.parse_args() + + redisToMISP = RedisToMISPFeed() + redisToMISP.consume() diff --git a/examples/feed-generator-from-redis/generate.py b/examples/feed-generator-from-redis/generate.py deleted file mode 100755 index 8e89d69..0000000 --- a/examples/feed-generator-from-redis/generate.py +++ /dev/null @@ -1,346 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import sys -import json -import os -import hashlib -import argparse -import datetime, time -import uuid -import threading -import redis - -from redis import StrictRedis as Redis -import settings - -from pymisp import MISPEvent, MISPAttribute -from pymisp.tools import GenericObjectGenerator - -evtObj=thr=None # animation thread - -def get_system_templates(): - misp_objects_path = os.path.join( - os.path.abspath(os.path.dirname(sys.modules['pymisp'].__file__)), - 'data', 'misp-objects', 'objects') - - templates = {} - for root, dirs, files in os.walk(misp_objects_path, topdown=False): - for def_file in files: - obj_name = root.split('/')[-1] - template_path = os.path.join(root, def_file) - with open(template_path, 'r') as f: - definition = json.load(f) - templates[obj_name] = definition - return templates - - -def gen_uuid(): - return str(uuid.uuid4()) - -def processing_animation(evtObj, buffer_state, refresh_rate=5): - i = 0 - buffer_state_str = 'attributes: {}, objects: {}, sightings: {}'.format(buffer_state['attribute'], buffer_state['object'], buffer_state['sighting']) - while True: - if evtObj.is_set(): - print(" "*(len(buffer_state_str)+20), end="\r", sep="") # overwrite last characters - sys.stdout.flush() - return - i += 1 - print("Remaining: { %s }\t" % buffer_state_str + "/-\|"[i%4], end="\r", sep="") - sys.stdout.flush() - time.sleep(1.0/float(refresh_rate)) - -def beautyful_sleep(sleep): - length = 20 - sleeptime = float(sleep) / float(length) - for i in range(length): - temp_string = '|'*i + ' '*(length-i-1) - print('sleeping [{}]'.format(temp_string), end='\r', sep='') - sys.stdout.flush() - time.sleep(sleeptime) - - - - -class RedisToMISPFeed: - SUFFIX_SIGH = '_sighting' - SUFFIX_ATTR = '_attribute' - SUFFIX_OBJ = '_object' - SUFFIX_LIST = [SUFFIX_SIGH, SUFFIX_ATTR, SUFFIX_OBJ] - - def __init__(self): - self.host = settings.host - self.port = settings.port - self.db = settings.db - self.serv = redis.StrictRedis(self.host, self.port, self.db, decode_responses=True) - - self.keynames = [] - for k in settings.keyname_pop: - for s in self.SUFFIX_LIST: - self.keynames.append(k+s) - - # get all templates - self.sys_templates = get_system_templates() - - self.sleep = settings.sleep - self.flushing_interval = settings.flushing_interval - self.flushing_next = time.time() + self.flushing_interval - - self.manifest = {} - self.attributeHashes = [] - - self.keynameError = settings.keyname_error - self.allow_animation = settings.allow_animation - - self.daily_event_name = settings.daily_event_name + ' {}' - _, self.current_event_uuid, self.event_name = self.get_last_event_from_manifest() - self.current_date = datetime.date.today() - self.current_event = self.get_event_from_id(self.current_event_uuid) - - global evtObj, thr - self.evtObj = evtObj - self.thr = thr - - def consume(self): - while True: - flag_empty = True - for key in self.keynames: - while True: - data = self.pop(key) - if data is None: - break - try: - self.perform_action(key, data) - except Exception as error: - self.save_error_to_redis(error, data) - flag_empty = False - - - if flag_empty and self.flushing_next <= time.time(): - self.flush_event() - flushing_next = time.time() + self.flushing_interval - - beautyful_sleep(5) - - def pop(self, key): - popped = self.serv.rpop(key) - if popped is None: - return None - try: - popped = json.loads(popped) - except ValueError as error: - self.save_error_to_redis(error, popped) - except ValueError as error: - self.save_error_to_redis(error, popped) - return popped - - - def perform_action(self, key, data): - self.update_daily_event_id() - - # sighting - if key.endswith(self.SUFFIX_SIGH): - pass - - # attribute - elif key.endswith(self.SUFFIX_ATTR): - attr_type = data.pop('type') - attr_value = data.pop('value') - self.current_event.add_attribute(attr_type, attr_value, **data) - self.add_hash(attr_type, attr_value) - - # object - elif key.endswith(self.SUFFIX_OBJ): - # create the MISP object - obj_name = data.pop('name') - misp_object = GenericObjectGenerator(obj_name) - for k, v in data.items(): - if k not in self.sys_templates[obj_name]['attributes']: # attribute is not in the object template definition - # add it with type text - misp_object.add_attribute(k, **{'value': v, 'type': 'text'}) - else: - misp_object.add_attribute(k, **{'value': v}) - - self.current_event.add_object(misp_object) - for attr_type, attr_value in data.items(): - self.add_hash(attr_type, attr_value) - - - else: - raise NoValidKey("Can't define action to perform") - - - def add_hash(self, attr_type, attr_value): - if ('|' in attr_type or attr_type == 'malware-sample'): - split = attr_value.split('|') - self.attributeHashes.append([hashlib.md5(str(split[0]).encode("utf-8")).hexdigest(), self.current_event_uuid]) - self.attributeHashes.append([hashlib.md5(str(split[1]).encode("utf-8")).hexdigest(), self.current_event_uuid]) - else: - self.attributeHashes.append([hashlib.md5(str(attr_value).encode("utf-8")).hexdigest(), self.current_event_uuid]) - - # Manifest - def init_manifest(self): - # create an empty manifest - with open(os.path.join(settings.outputdir, 'manifest.json'), 'w') as f: - pass - # create new event and save manifest - self.create_daily_event() - - - def flush_event(self, new_event=None): - print('Writting event on disk'+' '*20) - self.print_processing() - if new_event is not None: - event_uuid = new_event['uuid'] - event = new_event - else: - event_uuid = self.current_event_uuid - event = self.current_event - - eventFile = open(os.path.join(settings.outputdir, event_uuid + '.json'), 'w') - eventFile.write(event.to_json()) - eventFile.close() - - self.saveHashes() - if self.allow_animation: - self.evtObj.set() - self.thr.join() - - def saveManifest(self): - try: - manifestFile = open(os.path.join(settings.outputdir, 'manifest.json'), 'w') - manifestFile.write(json.dumps(self.manifest)) - manifestFile.close() - print('Manifest saved') - except Exception as e: - print(e) - sys.exit('Could not create the manifest file.') - - def saveHashes(self): - if len(self.attributeHashes) == 0: - return False - try: - hashFile = open(os.path.join(settings.outputdir, 'hashes.csv'), 'a') - for element in self.attributeHashes: - hashFile.write('{},{}\n'.format(element[0], element[1])) - hashFile.close() - self.attributeHashes = [] - print('Hash saved' + ' '*30) - except Exception as e: - print(e) - sys.exit('Could not create the quick hash lookup file.') - - - def __addEventToManifest(self, event): - event_dict = event.to_dict()['Event'] - tags = [] - for eventTag in event_dict.get('EventTag', []): - tags.append({'name': eventTag['Tag']['name'], - 'colour': eventTag['Tag']['colour']}) - return { - 'Orgc': event_dict.get('Orgc', []), - 'Tag': tags, - 'info': event_dict['info'], - 'date': event_dict['date'], - 'analysis': event_dict['analysis'], - 'threat_level_id': event_dict['threat_level_id'], - 'timestamp': event_dict.get('timestamp', int(time.time())) - } - - # Retreive last event from the manifest, if the manifest doesn't exists - # or if it is empty, initialize it. - def get_last_event_from_manifest(self): - try: - with open(os.path.join(settings.outputdir, 'manifest.json'), 'r') as f: - man = json.load(f) - dated_events = [] - for event_uuid, event_json in man.items(): - # add events to manifest - self.manifest[event_uuid] = event_json - dated_events.append([event_json['date'], event_uuid, event_json['info']]) - dated_events.sort(key=lambda k: (k[0], k[2]), reverse=True) # sort by date then by event name - return dated_events[0] - except FileNotFoundError as e: - print('Manifest not found, generating a fresh one') - self.init_manifest() - return self.get_last_event_from_manifest() - - # DAILY - def update_daily_event_id(self): - if self.current_date != datetime.date.today(): # create new event - # save current event on disk - self.flush_event() - self.current_event = create_daily_event() - self.current_event_uuid = self.current_event.get('uuid') - self.event_name = self.current_event.info - - def get_event_from_id(self, event_uuid): - with open(os.path.join(settings.outputdir, '%s.json' % event_uuid), 'r') as f: - event_dict = json.load(f)['Event'] - event = MISPEvent() - event.from_dict(**event_dict) - return event - - def create_daily_event(self): - new_uuid = gen_uuid() - today = str(datetime.date.today()) - event_dict = { - 'uuid': new_uuid, - 'id': len(self.manifest)+1, - 'Tag': settings.Tag, - 'info': self.daily_event_name.format(today), - 'analysis': settings.analysis, # [0-2] - 'threat_level_id': settings.threat_level_id, # [1-4] - 'published': settings.published, - 'date': today - } - event = MISPEvent() - event.from_dict(**event_dict) - - # reference org - org_dict = {} - org_dict['name'] = settings.org_name - org_dict['uui'] = settings.org_uuid - event['Orgc'] = org_dict - - # save event on disk - self.flush_event(new_event=event) - # add event to manifest - self.manifest[event['uuid']] = self.__addEventToManifest(event) - self.saveManifest() - return event - - # OTHERS - def get_buffer_state(self): - buffer_state = {'attribute': 0, 'object': 0, 'sighting': 0} - for k in self.keynames: - _ , suffix = k.rsplit('_', 1) - buffer_state[suffix] += self.serv.llen(k) - return buffer_state - - - def print_processing(self): - if self.allow_animation: - buff_states = self.get_buffer_state() - self.evtObj = threading.Event() - self.thr = threading.Thread(name="processing-animation", target=processing_animation, args=(self.evtObj, buff_states, )) - self.thr.start() - - def save_error_to_redis(self, error, item): - to_push = {'error': str(error), 'item': str(item)} - print('Error:', str(error), '\nOn adding:', item) - self.serv.lpush(self.keynameError, to_push) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser(description="Pop item fom redis and add it to the MISP feed. By default, each action are pushed into a daily named event. Configuration taken from the file settings.py.") - args = parser.parse_args() - - redisToMISP = RedisToMISPFeed() - try: - redisToMISP.consume() - except (KeyboardInterrupt, SystemExit): - if evtObj is not None: - evtObj.set() - thr.join() diff --git a/examples/feed-generator-from-redis/generator.py b/examples/feed-generator-from-redis/generator.py new file mode 100755 index 0000000..ced808d --- /dev/null +++ b/examples/feed-generator-from-redis/generator.py @@ -0,0 +1,236 @@ +#!/usr/bin/env python + +import sys +import json +import os +import hashlib +import datetime +import time +import uuid + +from pymisp import MISPEvent +from pymisp.tools import GenericObjectGenerator + +import settings + + +def get_system_templates(): + """Fetch all MISP-Object template present on the local system. + + Returns: + dict: A dictionary listing all MISP-Object templates + + """ + misp_objects_path = os.path.join( + os.path.abspath(os.path.dirname(sys.modules['pymisp'].__file__)), + 'data', 'misp-objects', 'objects') + + templates = {} + for root, dirs, files in os.walk(misp_objects_path, topdown=False): + for def_file in files: + obj_name = root.split('/')[-1] + template_path = os.path.join(root, def_file) + with open(template_path, 'r') as f: + definition = json.load(f) + templates[obj_name] = definition + return templates + + +def gen_uuid(): + """Generate a random UUID and returns its string representation""" + return str(uuid.uuid4()) + + +class FeedGenerator: + + def __init__(self): + self.sys_templates = get_system_templates() + + self.manifest = {} + self.attributeHashes = [] + + self.daily_event_name = settings.daily_event_name + ' {}' + _, self.current_event_uuid, self.event_name = self.get_last_event_from_manifest() + self.current_date = datetime.date.today() + self.current_event = self._get_event_from_id(self.current_event_uuid) + + def add_sighting_on_attribute(self, sight_type, attr_uuid, **data): + self.update_daily_event_id() + return False + + def add_attribute_to_event(self, attr_type, attr_value, **attr_data): + self.update_daily_event_id() + self.current_event.add_attribute(attr_type, attr_value, **attr_data) + self._add_hash(attr_type, attr_value) + return True + + def add_object_to_event(self, obj_name, **data): + self.update_daily_event_id() + # create the MISP object + misp_object = GenericObjectGenerator(obj_name) + if obj_name not in self.sys_templates: + print('Unkown object template') + return False + + for k, v in data.items(): + # attribute is not in the object template definition + if k not in self.sys_templates[obj_name]['attributes']: + # add it with type text + misp_object.add_attribute(k, **{'value': v, 'type': 'text'}) + else: + misp_object.add_attribute(k, **{'value': v}) + + self.current_event.add_object(misp_object) + for attr_type, attr_value in data.items(): + self._add_hash(attr_type, attr_value) + + return True + + # Cache + def _add_hash(self, attr_type, attr_value): + if ('|' in attr_type or attr_type == 'malware-sample'): + split = attr_value.split('|') + self.attributeHashes.append([ + hashlib.md5(str(split[0]).encode("utf-8")).hexdigest(), + self.current_event_uuid + ]) + self.attributeHashes.append([ + hashlib.md5(str(split[1]).encode("utf-8")).hexdigest(), + self.current_event_uuid + ]) + else: + self.attributeHashes.append([ + hashlib.md5(str(attr_value).encode("utf-8")).hexdigest(), + self.current_event_uuid + ]) + + # Manifest + def _init_manifest(self): + # create an empty manifest + with open(os.path.join(settings.outputdir, 'manifest.json'), 'w'): + pass + # create new event and save manifest + self.create_daily_event() + + def flush_event(self, new_event=None): + print('Writting event on disk'+' '*20) + if new_event is not None: + event_uuid = new_event['uuid'] + event = new_event + else: + event_uuid = self.current_event_uuid + event = self.current_event + + eventFile = open(os.path.join(settings.outputdir, event_uuid+'.json'), 'w') + eventFile.write(event.to_json()) + eventFile.close() + + self.save_hashes() + + def save_manifest(self): + try: + manifestFile = open(os.path.join(settings.outputdir, 'manifest.json'), 'w') + manifestFile.write(json.dumps(self.manifest)) + manifestFile.close() + print('Manifest saved') + except Exception as e: + print(e) + sys.exit('Could not create the manifest file.') + + def save_hashes(self): + if len(self.attributeHashes) == 0: + return False + try: + hashFile = open(os.path.join(settings.outputdir, 'hashes.csv'), 'a') + for element in self.attributeHashes: + hashFile.write('{},{}\n'.format(element[0], element[1])) + hashFile.close() + self.attributeHashes = [] + print('Hash saved' + ' '*30) + except Exception as e: + print(e) + sys.exit('Could not create the quick hash lookup file.') + + def _addEventToManifest(self, event): + event_dict = event.to_dict()['Event'] + tags = [] + for eventTag in event_dict.get('EventTag', []): + tags.append({'name': eventTag['Tag']['name'], + 'colour': eventTag['Tag']['colour']}) + return { + 'Orgc': event_dict.get('Orgc', []), + 'Tag': tags, + 'info': event_dict['info'], + 'date': event_dict['date'], + 'analysis': event_dict['analysis'], + 'threat_level_id': event_dict['threat_level_id'], + 'timestamp': event_dict.get('timestamp', int(time.time())) + } + + def get_last_event_from_manifest(self): + """Retreive last event from the manifest, if the manifest doesn't + exists or if it is empty, initialize it. + """ + try: + with open(os.path.join(settings.outputdir, 'manifest.json'), 'r') as f: + man = json.load(f) + dated_events = [] + for event_uuid, event_json in man.items(): + # add events to manifest + self.manifest[event_uuid] = event_json + dated_events.append([ + event_json['date'], + event_uuid, event_json['info'] + ]) + # Sort by date then by event name + dated_events.sort(key=lambda k: (k[0], k[2]), reverse=True) + return dated_events[0] + except FileNotFoundError as e: + print('Manifest not found, generating a fresh one') + self._init_manifest() + return self.get_last_event_from_manifest() + + # DAILY + def update_daily_event_id(self): + if self.current_date != datetime.date.today(): # create new event + # save current event on disk + self.flush_event() + self.current_event = self.create_daily_event() + self.current_event_uuid = self.current_event.get('uuid') + self.event_name = self.current_event.info + + def _get_event_from_id(self, event_uuid): + with open(os.path.join(settings.outputdir, '%s.json' % event_uuid), 'r') as f: + event_dict = json.load(f)['Event'] + event = MISPEvent() + event.from_dict(**event_dict) + return event + + def create_daily_event(self): + new_uuid = gen_uuid() + today = str(datetime.date.today()) + event_dict = { + 'uuid': new_uuid, + 'id': len(self.manifest)+1, + 'Tag': settings.Tag, + 'info': self.daily_event_name.format(today), + 'analysis': settings.analysis, # [0-2] + 'threat_level_id': settings.threat_level_id, # [1-4] + 'published': settings.published, + 'date': today + } + event = MISPEvent() + event.from_dict(**event_dict) + + # reference org + org_dict = {} + org_dict['name'] = settings.org_name + org_dict['uui'] = settings.org_uuid + event['Orgc'] = org_dict + + # save event on disk + self.flush_event(new_event=event) + # add event to manifest + self.manifest[event['uuid']] = self._addEventToManifest(event) + self.save_manifest() + return event diff --git a/examples/feed-generator-from-redis/settings.default.py b/examples/feed-generator-from-redis/settings.default.py index c3fce6d..ad4a3cd 100755 --- a/examples/feed-generator-from-redis/settings.default.py +++ b/examples/feed-generator-from-redis/settings.default.py @@ -1,3 +1,4 @@ +""" REDIS RELATED """ # Your redis server host='127.0.0.1' port=6379 @@ -6,6 +7,14 @@ db=0 #keyname_pop='misp_feed_generator_key' keyname_pop=['cowrie'] +# OTHERS +## How frequent the event should be written on disk +flushing_interval=5*60 +## The redis list keyname in which to put items that generated an error +keyname_error='feed-generation-error' + +""" FEED GENERATOR CONFIGURATION """ + # The output dir for the feed. This will drop a lot of files, so make # sure that you use a directory dedicated to the feed outputdir = 'output' @@ -16,27 +25,25 @@ outputdir = 'output' org_name='myOrg' ### Your organisation UUID org_uuid='' -### The daily event name to be used in MISP. (e.g. honeypot_1, will produce each day an event of the form honeypot_1 dd-mm-yyyy) +### The daily event name to be used in MISP. +### (e.g. honeypot_1, will produce each day an event of the form honeypot_1 dd-mm-yyyy) daily_event_name='PyMISP default event name' ## Optional analysis=0 threat_level_id=3 published=False -Tag=[{ +Tag=[ + { "colour": "#ffffff", "name": "tlp:white" }, + { "colour": "#ff00ff", "name": "my:custom:feed" - ] + } +] # Others ## Redis pooling time sleep=60 -## The redis list keyname in which to put items that generated an error -keyname_error='feed-generation-error' -## Display an animation while adding element to MISP -allow_animation=True -## How frequent the event should be written on disk -flushing_interval=5*60 From 828aa8e6e3d0478e44d4f4ac0f3373e609a18c23 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Fri, 9 Mar 2018 15:39:19 +0100 Subject: [PATCH 06/17] Updated README --- examples/feed-generator-from-redis/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/feed-generator-from-redis/README.md b/examples/feed-generator-from-redis/README.md index 8bdd7cc..36c9364 100644 --- a/examples/feed-generator-from-redis/README.md +++ b/examples/feed-generator-from-redis/README.md @@ -8,6 +8,6 @@ This python script can be used to generate a MISP feed based on data stored in r git clone https://github.com/CIRCL/PyMISP cd examples/feed-generator-from-redis cp settings-default.py settings.py -vi settings.py #adjust your settings -python3 generate.py +vi settings.py # adjust your settings +python3 fromredis.py ```` From f6828c43946c984b371b492572178c70d0db4691 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Fri, 9 Mar 2018 15:51:26 +0100 Subject: [PATCH 07/17] Added description of generator object --- examples/feed-generator-from-redis/generator.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/examples/feed-generator-from-redis/generator.py b/examples/feed-generator-from-redis/generator.py index ced808d..6683109 100755 --- a/examples/feed-generator-from-redis/generator.py +++ b/examples/feed-generator-from-redis/generator.py @@ -42,8 +42,17 @@ def gen_uuid(): class FeedGenerator: + def __init__(self, auto_flush=60*5): + """This object can be use to easily create a daily MISP-feed. - def __init__(self): + It handles the event creation, manifest file and cache file (hashes.csv). + + Attributes: + auto_flush (int) : (In seconds) If a positive value for auto_flush + is provided the FeedGenerator will the current event when + possible if the last flushed time is greater than auto_flush. + + """ self.sys_templates = get_system_templates() self.manifest = {} From e1a9fe1d85b5f243bbb9b8cddf9ef88b5ff22b3d Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Fri, 9 Mar 2018 17:06:00 +0100 Subject: [PATCH 08/17] Generator handles file flushing itself --- .../feed-generator-from-redis/fromredis.py | 20 ++--------- .../feed-generator-from-redis/generator.py | 33 +++++++++++++------ 2 files changed, 26 insertions(+), 27 deletions(-) diff --git a/examples/feed-generator-from-redis/fromredis.py b/examples/feed-generator-from-redis/fromredis.py index f4f68b7..38f33cc 100755 --- a/examples/feed-generator-from-redis/fromredis.py +++ b/examples/feed-generator-from-redis/fromredis.py @@ -35,9 +35,6 @@ class RedisToMISPFeed: self.db = settings.db self.serv = redis.StrictRedis(self.host, self.port, self.db, decode_responses=True) - self.flushing_interval = settings.flushing_interval - self.flushing_next = time.time() + self.flushing_interval - self.generator = FeedGenerator() self.keynames = [] @@ -47,13 +44,11 @@ class RedisToMISPFeed: self.keynameError = settings.keyname_error - self.last_flush = datetime.datetime.now() self.update_last_action("Init system") def consume(self): self.update_last_action("Started consuming redis") while True: - flag_empty = True for key in self.keynames: while True: data = self.pop(key) @@ -63,14 +58,6 @@ class RedisToMISPFeed: self.perform_action(key, data) except Exception as error: self.save_error_to_redis(error, data) - flag_empty = False - - # Define when to write event on disk - if flag_empty and self.flushing_next <= time.time() and self.last_action_time>self.last_flush: - self.update_last_action('Flushed on disk') - self.generator.flush_event() - self.flushing_next = time.time() + self.flushing_interval - self.last_flush = datetime.datetime.now() beautyful_sleep(5, self.format_last_action()) @@ -122,17 +109,15 @@ class RedisToMISPFeed: self.last_action_time = datetime.datetime.now() def format_last_action(self): - temp = datetime.datetime.now() - self.last_flush - return "Last action: [{}] @ {}.\tLast flush: {} ago".format( + return "Last action: [{}] @ {}".format( self.last_action, self.last_action_time.isoformat().replace('T', ' '), - str(temp).split('.')[0] ) def get_buffer_state(self): buffer_state = {'attribute': 0, 'object': 0, 'sighting': 0} for k in self.keynames: - _ , suffix = k.rsplit('_', 1) + _, suffix = k.rsplit('_', 1) buffer_state[suffix] += self.serv.llen(k) return buffer_state @@ -142,6 +127,7 @@ class RedisToMISPFeed: print('Error:', str(error), '\nOn adding:', item) self.serv.lpush(self.keynameError, to_push) + if __name__ == '__main__': parser = argparse.ArgumentParser(description="Pop item fom redis and add " + "it to the MISP feed. By default, each action are pushed into a " diff --git a/examples/feed-generator-from-redis/generator.py b/examples/feed-generator-from-redis/generator.py index 6683109..7e232fc 100755 --- a/examples/feed-generator-from-redis/generator.py +++ b/examples/feed-generator-from-redis/generator.py @@ -42,19 +42,18 @@ def gen_uuid(): class FeedGenerator: - def __init__(self, auto_flush=60*5): + def __init__(self): """This object can be use to easily create a daily MISP-feed. - It handles the event creation, manifest file and cache file (hashes.csv). - - Attributes: - auto_flush (int) : (In seconds) If a positive value for auto_flush - is provided the FeedGenerator will the current event when - possible if the last flushed time is greater than auto_flush. + It handles the event creation, manifest file and cache file + (hashes.csv). """ self.sys_templates = get_system_templates() + self.flushing_interval = settings.flushing_interval + self.flushing_next = time.time() + self.flushing_interval + self.manifest = {} self.attributeHashes = [] @@ -65,12 +64,14 @@ class FeedGenerator: def add_sighting_on_attribute(self, sight_type, attr_uuid, **data): self.update_daily_event_id() + self.after_addition() return False def add_attribute_to_event(self, attr_type, attr_value, **attr_data): self.update_daily_event_id() self.current_event.add_attribute(attr_type, attr_value, **attr_data) self._add_hash(attr_type, attr_value) + self.after_addition() return True def add_object_to_event(self, obj_name, **data): @@ -93,8 +94,17 @@ class FeedGenerator: for attr_type, attr_value in data.items(): self._add_hash(attr_type, attr_value) + self.after_addition() return True + def after_addition(self): + # Write event on disk + now = time.time() + if self.flushing_next <= now: + self.update_last_action('Flushed on disk') + self.flush_event() + self.flushing_next = now + self.flushing_interval + # Cache def _add_hash(self, attr_type, attr_value): if ('|' in attr_type or attr_type == 'malware-sample'): @@ -177,11 +187,14 @@ class FeedGenerator: } def get_last_event_from_manifest(self): - """Retreive last event from the manifest, if the manifest doesn't - exists or if it is empty, initialize it. + """Retreive last event from the manifest. + + If the manifest doesn't exists or if it is empty, initialize it. + """ try: - with open(os.path.join(settings.outputdir, 'manifest.json'), 'r') as f: + manifest_path = os.path.join(settings.outputdir, 'manifest.json') + with open(manifest_path, 'r') as f: man = json.load(f) dated_events = [] for event_uuid, event_json in man.items(): From aa3f3b85f03628781d9ccee2c66697b945301666 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Mon, 12 Mar 2018 10:19:05 +0100 Subject: [PATCH 09/17] removed unused function --- examples/feed-generator-from-redis/fromredis.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/examples/feed-generator-from-redis/fromredis.py b/examples/feed-generator-from-redis/fromredis.py index 38f33cc..e8d8584 100755 --- a/examples/feed-generator-from-redis/fromredis.py +++ b/examples/feed-generator-from-redis/fromredis.py @@ -114,13 +114,6 @@ class RedisToMISPFeed: self.last_action_time.isoformat().replace('T', ' '), ) - def get_buffer_state(self): - buffer_state = {'attribute': 0, 'object': 0, 'sighting': 0} - for k in self.keynames: - _, suffix = k.rsplit('_', 1) - buffer_state[suffix] += self.serv.llen(k) - return buffer_state - def save_error_to_redis(self, error, item): to_push = {'error': str(error), 'item': str(item)} From 81d35328771677089f8a0d3fd43a6a1c1fe90f42 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Mon, 12 Mar 2018 10:25:25 +0100 Subject: [PATCH 10/17] Added brief object description --- examples/feed-generator-from-redis/generator.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/examples/feed-generator-from-redis/generator.py b/examples/feed-generator-from-redis/generator.py index 7e232fc..da2fb82 100755 --- a/examples/feed-generator-from-redis/generator.py +++ b/examples/feed-generator-from-redis/generator.py @@ -42,6 +42,11 @@ def gen_uuid(): class FeedGenerator: + """Helper object to create MISP feed. + + Configuration taken from the file settings.py""" + + def __init__(self): """This object can be use to easily create a daily MISP-feed. From d898bb38579718fb44f9b368d56541ffcc86d5d2 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Mon, 12 Mar 2018 15:17:25 +0100 Subject: [PATCH 11/17] feature: Added support of MISP object constructor instead of the generic_generator --- .../feed-generator-from-redis/generator.py | 58 ++++++++++++------- examples/feed-generator-from-redis/server.py | 2 +- .../settings.default.py | 11 +++- 3 files changed, 47 insertions(+), 24 deletions(-) diff --git a/examples/feed-generator-from-redis/generator.py b/examples/feed-generator-from-redis/generator.py index da2fb82..ed88743 100755 --- a/examples/feed-generator-from-redis/generator.py +++ b/examples/feed-generator-from-redis/generator.py @@ -9,7 +9,6 @@ import time import uuid from pymisp import MISPEvent -from pymisp.tools import GenericObjectGenerator import settings @@ -46,7 +45,6 @@ class FeedGenerator: Configuration taken from the file settings.py""" - def __init__(self): """This object can be use to easily create a daily MISP-feed. @@ -55,6 +53,7 @@ class FeedGenerator: """ self.sys_templates = get_system_templates() + self.constructor_dict = settings.constructor_dict self.flushing_interval = settings.flushing_interval self.flushing_next = time.time() + self.flushing_interval @@ -63,50 +62,63 @@ class FeedGenerator: self.attributeHashes = [] self.daily_event_name = settings.daily_event_name + ' {}' - _, self.current_event_uuid, self.event_name = self.get_last_event_from_manifest() - self.current_date = datetime.date.today() + event_date_str, self.current_event_uuid, self.event_name = self.get_last_event_from_manifest() + temp = [int(x) for x in event_date_str.split('-')] + self.current_event_date = datetime.date(temp[0], temp[1], temp[2]) self.current_event = self._get_event_from_id(self.current_event_uuid) def add_sighting_on_attribute(self, sight_type, attr_uuid, **data): + """Add a sighting on an attribute. + + Not supported for the moment.""" self.update_daily_event_id() - self.after_addition() + self._after_addition() return False def add_attribute_to_event(self, attr_type, attr_value, **attr_data): + """Add an attribute to the daily event""" self.update_daily_event_id() self.current_event.add_attribute(attr_type, attr_value, **attr_data) self._add_hash(attr_type, attr_value) - self.after_addition() + self._after_addition() return True def add_object_to_event(self, obj_name, **data): + """Add an object to the daily event""" self.update_daily_event_id() - # create the MISP object - misp_object = GenericObjectGenerator(obj_name) if obj_name not in self.sys_templates: print('Unkown object template') return False - for k, v in data.items(): - # attribute is not in the object template definition - if k not in self.sys_templates[obj_name]['attributes']: - # add it with type text - misp_object.add_attribute(k, **{'value': v, 'type': 'text'}) - else: - misp_object.add_attribute(k, **{'value': v}) + # Get MISP object constructor + obj_constr = self.constructor_dict.get(obj_name, None) + # Constructor not known, using the generic one + if obj_constr is None: + obj_constr = self.constructor_dict.get('generic') + misp_object = obj_constr(obj_name) + # Fill generic object + for k, v in data.items(): + # attribute is not in the object template definition + if k not in self.sys_templates[obj_name]['attributes']: + # add it with type text + misp_object.add_attribute(k, **{'value': v, 'type': 'text'}) + else: + misp_object.add_attribute(k, **{'value': v}) + + else: + misp_object = obj_constr(data) self.current_event.add_object(misp_object) for attr_type, attr_value in data.items(): self._add_hash(attr_type, attr_value) - self.after_addition() + self._after_addition() return True - def after_addition(self): - # Write event on disk + def _after_addition(self): + """Write event on disk""" now = time.time() if self.flushing_next <= now: - self.update_last_action('Flushed on disk') self.flush_event() self.flushing_next = now + self.flushing_interval @@ -137,7 +149,7 @@ class FeedGenerator: self.create_daily_event() def flush_event(self, new_event=None): - print('Writting event on disk'+' '*20) + print('Writting event on disk'+' '*50) if new_event is not None: event_uuid = new_event['uuid'] event = new_event @@ -207,7 +219,8 @@ class FeedGenerator: self.manifest[event_uuid] = event_json dated_events.append([ event_json['date'], - event_uuid, event_json['info'] + event_uuid, + event_json['info'] ]) # Sort by date then by event name dated_events.sort(key=lambda k: (k[0], k[2]), reverse=True) @@ -219,10 +232,11 @@ class FeedGenerator: # DAILY def update_daily_event_id(self): - if self.current_date != datetime.date.today(): # create new event + if self.current_event_date != datetime.date.today(): # create new event # save current event on disk self.flush_event() self.current_event = self.create_daily_event() + self.current_event_date = datetime.date.today() self.current_event_uuid = self.current_event.get('uuid') self.event_name = self.current_event.info diff --git a/examples/feed-generator-from-redis/server.py b/examples/feed-generator-from-redis/server.py index 5a2ee16..1dd6873 100755 --- a/examples/feed-generator-from-redis/server.py +++ b/examples/feed-generator-from-redis/server.py @@ -2,7 +2,7 @@ import os.path from flask import Flask -from flask.ext.autoindex import AutoIndex +from flask_autoindex import AutoIndex from settings import outputdir app = Flask(__name__) diff --git a/examples/feed-generator-from-redis/settings.default.py b/examples/feed-generator-from-redis/settings.default.py index ad4a3cd..0b4a478 100755 --- a/examples/feed-generator-from-redis/settings.default.py +++ b/examples/feed-generator-from-redis/settings.default.py @@ -25,7 +25,7 @@ outputdir = 'output' org_name='myOrg' ### Your organisation UUID org_uuid='' -### The daily event name to be used in MISP. +### The daily event name to be used in MISP. ### (e.g. honeypot_1, will produce each day an event of the form honeypot_1 dd-mm-yyyy) daily_event_name='PyMISP default event name' @@ -44,6 +44,15 @@ Tag=[ } ] +# MISP Object constructor +from CowrieMISPObject import CowrieMISPObject +from pymisp.tools import GenericObjectGenerator + +constructor_dict = { + 'cowrie': CowrieMISPObject, + 'generic': GenericObjectGenerator +} + # Others ## Redis pooling time sleep=60 From 38c22ba95412d246dc4be3702a94de54cb19bcb1 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Mon, 12 Mar 2018 15:22:58 +0100 Subject: [PATCH 12/17] Moved object constructor into their own folder --- .../ObjectConstructor/CowrieMISPObject.py | 32 +++++++++++++++++++ .../settings.default.py | 2 +- 2 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 examples/feed-generator-from-redis/ObjectConstructor/CowrieMISPObject.py diff --git a/examples/feed-generator-from-redis/ObjectConstructor/CowrieMISPObject.py b/examples/feed-generator-from-redis/ObjectConstructor/CowrieMISPObject.py new file mode 100644 index 0000000..e2e8f64 --- /dev/null +++ b/examples/feed-generator-from-redis/ObjectConstructor/CowrieMISPObject.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 + +import time + +from pymisp.tools.abstractgenerator import AbstractMISPObjectGenerator + + +class CowrieMISPObject(AbstractMISPObjectGenerator): + def __init__(self, dico_val, **kargs): + self._dico_val = dico_val + self.name = "cowrie" + + # Enforce attribute date with timestamp + super(CowrieMISPObject, self).__init__('cowrie', + default_attributes_parameters={'timestamp': int(time.time())}, + **kargs) + self.generate_attributes() + + def generate_attributes(self): + skip_list = ['time', 'duration', 'isError', 'ttylog'] + for object_relation, value in self._dico_val.items(): + if object_relation in skip_list or 'log_' in object_relation: + continue + # cast to datetime + if object_relation == 'timestamp': + # Date already in ISO format, removing trailing Z + value = value.rstrip('Z') + + if isinstance(value, dict): + self.add_attribute(object_relation, **value) + else: + self.add_attribute(object_relation, value=value) diff --git a/examples/feed-generator-from-redis/settings.default.py b/examples/feed-generator-from-redis/settings.default.py index 0b4a478..0f6457c 100755 --- a/examples/feed-generator-from-redis/settings.default.py +++ b/examples/feed-generator-from-redis/settings.default.py @@ -45,7 +45,7 @@ Tag=[ ] # MISP Object constructor -from CowrieMISPObject import CowrieMISPObject +from ObjectConstructor.CowrieMISPObject import CowrieMISPObject from pymisp.tools import GenericObjectGenerator constructor_dict = { From 80517aaf41f5f1cc2cf4729d3dbe29ec8a2311c2 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Mon, 12 Mar 2018 15:34:12 +0100 Subject: [PATCH 13/17] Updated readme --- .../ObjectConstructor/CowrieMISPObject.py | 2 +- examples/feed-generator-from-redis/README.md | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/examples/feed-generator-from-redis/ObjectConstructor/CowrieMISPObject.py b/examples/feed-generator-from-redis/ObjectConstructor/CowrieMISPObject.py index e2e8f64..6c1a40b 100644 --- a/examples/feed-generator-from-redis/ObjectConstructor/CowrieMISPObject.py +++ b/examples/feed-generator-from-redis/ObjectConstructor/CowrieMISPObject.py @@ -21,7 +21,7 @@ class CowrieMISPObject(AbstractMISPObjectGenerator): for object_relation, value in self._dico_val.items(): if object_relation in skip_list or 'log_' in object_relation: continue - # cast to datetime + if object_relation == 'timestamp': # Date already in ISO format, removing trailing Z value = value.rstrip('Z') diff --git a/examples/feed-generator-from-redis/README.md b/examples/feed-generator-from-redis/README.md index 36c9364..faf7b46 100644 --- a/examples/feed-generator-from-redis/README.md +++ b/examples/feed-generator-from-redis/README.md @@ -5,9 +5,16 @@ This python script can be used to generate a MISP feed based on data stored in r # Installation ```` +# Feed generator git clone https://github.com/CIRCL/PyMISP cd examples/feed-generator-from-redis -cp settings-default.py settings.py +cp settings.default.py settings.py vi settings.py # adjust your settings + python3 fromredis.py + +# Serving file to MISP +bash install.sh +. ./serv-env/bin/activate +python3 server.py ```` From 39fc05aad9ee384ae500fb094bd2248157f9e024 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Mon, 12 Mar 2018 15:41:02 +0100 Subject: [PATCH 14/17] Updated readme 2 --- examples/feed-generator-from-redis/README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/examples/feed-generator-from-redis/README.md b/examples/feed-generator-from-redis/README.md index faf7b46..0fac8fa 100644 --- a/examples/feed-generator-from-redis/README.md +++ b/examples/feed-generator-from-redis/README.md @@ -1,6 +1,9 @@ # What -This python script can be used to generate a MISP feed based on data stored in redis. +- ``generator.py`` exposes a class allowing to generate a MISP feed in real time. +- ``fromredis.py`` uses ``generator.py`` to generate a MISP feed based on data stored in redis. +- ``server.py`` is a simple script using *Flask_autoindex* to serve data to MISP + # Installation From 91262662c4debc8ee724aa69d4960faec967357e Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Mon, 12 Mar 2018 16:13:34 +0100 Subject: [PATCH 15/17] Added MISPItemToRedis and updated readme accordingly --- .../MISPItemToRedis.py | 89 +++++++++++++++++++ examples/feed-generator-from-redis/README.md | 3 +- 2 files changed, 91 insertions(+), 1 deletion(-) create mode 100644 examples/feed-generator-from-redis/MISPItemToRedis.py diff --git a/examples/feed-generator-from-redis/MISPItemToRedis.py b/examples/feed-generator-from-redis/MISPItemToRedis.py new file mode 100644 index 0000000..7c038b4 --- /dev/null +++ b/examples/feed-generator-from-redis/MISPItemToRedis.py @@ -0,0 +1,89 @@ +import redis +import json + + +class MISPItemToRedis: + """This class provides a simple normalization to add MISP item to + redis, so that they can easily be processed and added to MISP later on.""" + SUFFIX_SIGH = '_sighting' + SUFFIX_ATTR = '_attribute' + SUFFIX_OBJ = '_object' + SUFFIX_LIST = [SUFFIX_SIGH, SUFFIX_ATTR, SUFFIX_OBJ] + + def __init__(self, keyname, host='localhost', port=6379, db=0): + self.host = host + self.port = port + self.db = db + self.keyname = keyname + self.serv = redis.StrictRedis(self.host, self.port, self.db) + + def push_json(self, jdata, keyname, action): + all_action = [s.lstrip('_') for s in self.SUFFIX_LIST] + if action not in all_action: + raise('Error: Invalid action. (Allowed: {})'.format(all_action)) + key = keyname + '_' + action + self.serv.lpush(key, jdata) + + def push_attribute(self, type_value, value, category=None, to_ids=False, + comment=None, distribution=None, proposal=False, **kwargs): + to_push = {} + to_push['type'] = type_value + to_push['value'] = value + if category is not None: + to_push['category'] = category + if to_ids is not None: + to_push['to_ids'] = to_ids + if comment is not None: + to_push['comment'] = comment + if distribution is not None: + to_push['distribution'] = distribution + if proposal is not None: + to_push['proposal'] = proposal + for k, v in kwargs.items(): + to_push[k] = v + key = self.keyname + self.SUFFIX_ATTR + self.serv.lpush(key, json.dumps(to_push)) + + def push_attribute_obj(self, MISP_Attribute, keyname): + key = keyname + self.SUFFIX_ATTR + jdata = MISP_Attribute.to_json() + self.serv.lpush(key, jdata) + + def push_object(self, dict_values): + # check that 'name' field is present + if 'name' not in dict_values: + print("Error: JSON must contain the field 'name'") + key = self.keyname + self.SUFFIX_OBJ + self.serv.lpush(key, json.dumps(dict_values)) + + def push_object_obj(self, MISP_Object, keyname): + key = keyname + self.SUFFIX_OBJ + jdata = MISP_Object.to_json() + self.serv.lpush(key, jdata) + + def push_sighting(self, value=None, uuid=None, id=None, source=None, + type=0, timestamp=None, **kargs): + to_push = {} + if value is not None: + to_push['value'] = value + if uuid is not None: + to_push['uuid'] = uuid + if id is not None: + to_push['id'] = id + if source is not None: + to_push['source'] = source + if type is not None: + to_push['type'] = type + if timestamp is not None: + to_push['timestamp'] = timestamp + + for k, v in kargs.items(): + if v is not None: + to_push[k] = v + key = self.keyname + self.SUFFIX_SIGH + self.serv.lpush(key, json.dumps(to_push)) + + def push_sighting_obj(self, MISP_Sighting, keyname): + key = keyname + self.SUFFIX_SIGH + jdata = MISP_Sighting.to_json() + self.serv.lpush(key, jdata) diff --git a/examples/feed-generator-from-redis/README.md b/examples/feed-generator-from-redis/README.md index 0fac8fa..1e31868 100644 --- a/examples/feed-generator-from-redis/README.md +++ b/examples/feed-generator-from-redis/README.md @@ -2,7 +2,8 @@ - ``generator.py`` exposes a class allowing to generate a MISP feed in real time. - ``fromredis.py`` uses ``generator.py`` to generate a MISP feed based on data stored in redis. -- ``server.py`` is a simple script using *Flask_autoindex* to serve data to MISP +- ``server.py`` is a simple script using *Flask_autoindex* to serve data to MISP. +- ``MISPItemToRedis.py`` permits to push (in redis) items to be added in MISP by the ``fromredis.py`` script. # Installation From 364d685e0c7143d9fd6b2426eab225dd27949edc Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Mon, 12 Mar 2018 16:40:06 +0100 Subject: [PATCH 16/17] Added usage in README --- examples/feed-generator-from-redis/README.md | 40 +++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/examples/feed-generator-from-redis/README.md b/examples/feed-generator-from-redis/README.md index 1e31868..2a3a94a 100644 --- a/examples/feed-generator-from-redis/README.md +++ b/examples/feed-generator-from-redis/README.md @@ -1,6 +1,6 @@ # What -- ``generator.py`` exposes a class allowing to generate a MISP feed in real time. +- ``generator.py`` exposes a class allowing to generate a MISP feed in real time, where each items can be added on daily generated events. - ``fromredis.py`` uses ``generator.py`` to generate a MISP feed based on data stored in redis. - ``server.py`` is a simple script using *Flask_autoindex* to serve data to MISP. - ``MISPItemToRedis.py`` permits to push (in redis) items to be added in MISP by the ``fromredis.py`` script. @@ -22,3 +22,41 @@ bash install.sh . ./serv-env/bin/activate python3 server.py ```` + + +# Utilisation + +### Adding items to MISP + +``` +# create helper object +>>> helper = MISPItemToRedis("redis_list_keyname") + +# push an attribute to redis +>>> helper.push_attribute("ip-src", "8.8.8.8", category="Network activity") + +# push an object to redis +>>> helper.push_object({ "name": "cowrie", "session": "session_id", "username": "admin", "password": "admin", "protocol": "telnet" }) + +# push a sighting to redis +>>> helper.push_sighting(uuid="5a9e9e26-fe40-4726-8563-5585950d210f") +``` + +### Generator + +``` +# Create the FeedGenerator object using the configuration provided in the file settings.py +# It will create daily event in which attributes and object will be added +generator = FeedGenerator() + +# Add an attribute to the daily event +attr_type = "ip-src" +attr_value = "8.8.8.8" +additional_data = {} +generator.add_attribute_to_event(attr_type, attr_value, **additional_data) + +# Add a cowrie object to the daily event +obj_name = "cowrie" +obj_data = { "session": "session_id", "username": "admin", "password": "admin", "protocol": "telnet" } +generator.add_object_to_event(obj_name, **obj_data) +``` From 6553519e3bcc6c2782160d53d75d710ab48474ff Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Mon, 12 Mar 2018 16:55:21 +0100 Subject: [PATCH 17/17] Added more examples --- examples/feed-generator-from-redis/README.md | 36 +++++++++++++++----- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/examples/feed-generator-from-redis/README.md b/examples/feed-generator-from-redis/README.md index 2a3a94a..4ccd91f 100644 --- a/examples/feed-generator-from-redis/README.md +++ b/examples/feed-generator-from-redis/README.md @@ -26,6 +26,11 @@ python3 server.py # Utilisation +``` +# Activate virtualenv +. ./serv-env/bin/activate +``` + ### Adding items to MISP ``` @@ -42,21 +47,34 @@ python3 server.py >>> helper.push_sighting(uuid="5a9e9e26-fe40-4726-8563-5585950d210f") ``` -### Generator +### Generate the feed ``` # Create the FeedGenerator object using the configuration provided in the file settings.py # It will create daily event in which attributes and object will be added -generator = FeedGenerator() +>>> generator = FeedGenerator() # Add an attribute to the daily event -attr_type = "ip-src" -attr_value = "8.8.8.8" -additional_data = {} -generator.add_attribute_to_event(attr_type, attr_value, **additional_data) +>>> attr_type = "ip-src" +>>> attr_value = "8.8.8.8" +>>> additional_data = {} +>>> generator.add_attribute_to_event(attr_type, attr_value, **additional_data) # Add a cowrie object to the daily event -obj_name = "cowrie" -obj_data = { "session": "session_id", "username": "admin", "password": "admin", "protocol": "telnet" } -generator.add_object_to_event(obj_name, **obj_data) +>>> obj_name = "cowrie" +>>> obj_data = { "session": "session_id", "username": "admin", "password": "admin", "protocol": "telnet" } +>>> generator.add_object_to_event(obj_name, **obj_data) +``` + +### Consume stored data in redis + +``` +# Configuration provided in the file settings.py +>>> python3 fromredis.py +``` + +### Serve data to MISP + +``` +>>> python3 server.py ```