From acf888165819f1f841777cbe571239ee53bdbf23 Mon Sep 17 00:00:00 2001 From: Tristan METAYER Date: Thu, 12 May 2016 17:33:13 +0200 Subject: [PATCH] init for ioc-2-misp --- examples/ioc-2-misp/README.md | 25 +++ examples/ioc-2-misp/ioc2misp.py | 348 +++++++++++++++++++++++++++++ examples/ioc-2-misp/keys.py.sample | 94 ++++++++ examples/ioc-2-misp/taxonomy.csv | 12 + 4 files changed, 479 insertions(+) create mode 100644 examples/ioc-2-misp/README.md create mode 100644 examples/ioc-2-misp/ioc2misp.py create mode 100644 examples/ioc-2-misp/keys.py.sample create mode 100644 examples/ioc-2-misp/taxonomy.csv diff --git a/examples/ioc-2-misp/README.md b/examples/ioc-2-misp/README.md new file mode 100644 index 0000000..dc22d58 --- /dev/null +++ b/examples/ioc-2-misp/README.md @@ -0,0 +1,25 @@ +### Description + +Python script for ioc import to misp + +### requires + +> python 2.7 +> PyMISP +> BeautifulSoup (apt-get install python-bs4 lxml) + +### Usage + +```bash +python ioc2misp.py -i myioc -t "tag:mytag='sample','tag:other='foo'" +``` + +```bash +time find /iocsample -type f|while read line ;do python ioc2misp.py -i ${line};done +``` + +### Conf + + * rename keys.py.sample as keys.py + * add your url and api key in keys.py + * use command in terminal \ No newline at end of file diff --git a/examples/ioc-2-misp/ioc2misp.py b/examples/ioc-2-misp/ioc2misp.py new file mode 100644 index 0000000..ecbeb60 --- /dev/null +++ b/examples/ioc-2-misp/ioc2misp.py @@ -0,0 +1,348 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Format description +# @variables : camelCase +# @functions : snake_case + +from keys import mispUrl, mispKey, csvTaxonomyFile, iocMispMapping + +try: + from pymisp import PyMISP +except: + print "you need pymisp form github" + import sys + sys.exit(1) + +import json +import os +import argparse + +try: + from bs4 import BeautifulSoup +except: + print "install BeautifulSoup : sudo apt-get install python-bs4 python-lxml" + import sys + sys.exit(1) + +def misp_init(url, key): + return PyMISP(url, key, False, 'json') + +def check_valid_ioc(): + + (filepath, filename) = os.path.split(iocDescriptions["iocfile"]) + (shortname, extension) = os.path.splitext(filename) + + if (("ioc" in extension)) and (sum(1 for _ in open(iocDescriptions["iocfile"])) > 1): + iocDescriptions['filename'] = filename + return True + return False + +def get_parse_ioc_file(): + + return BeautifulSoup(open(iocDescriptions["iocfile"]) , "lxml") + +def parse_ioc_search_content(iocContextSearch): + for k,v in iocMispMapping.items(): + if str(k).lower() == str(iocContextSearch).lower(): + return v + return False + +def create_attribute_json(iocContextSearch, attributeValue, attributeComment,force=False): + ##################################### + # force used for description to upload + if force: + parseResult=("Other","comment") + else: + parseResult = parse_ioc_search_content(iocContextSearch) + + if parseResult is False: + + print "/!\ Not implemented :: {0} :: {1} :: Item add as 'Other','Comment'. Add it in your keys.py".format(iocContextSearch,attributeValue) + ######################################## + # force import to misp + parseResult=("Other","comment") + + comment = "" + try : + comment= parseResult[2]+attributeComment + except: + comment= attributeComment + + attribute = { + "category": parseResult[0], + "type": parseResult[1], + "value": attributeValue, + "timestamp": "0", + "to_ids": "0", + "distribution": "0", + "comment": comment, + } + return attribute + +def create_attributes_from_ioc_json(soup): + attributes = [] + + IndicatorItemValues = {} + for item in soup.find_all("indicatoritem"): + + if item.find('context'): + IndicatorItemValues["context"]=str(item.find('context')['search']) + else: + IndicatorItemValues["context"]="" + if item.find('content'): + IndicatorItemValues["content"]=str(item.find('content').text) + else: + IndicatorItemValues["content"]="" + if item.find('comment'): + IndicatorItemValues["comment"]=str(item.find('comment').text) + else: + IndicatorItemValues["comment"]="" + + + jsonAttribute = create_attribute_json(IndicatorItemValues["context"],IndicatorItemValues["content"],IndicatorItemValues["comment"]) + attributes.append(jsonAttribute) + + return attributes + + +def create_misp_event_json(attributes): + import time + if iocDescriptions["authored_by"]: + attributes.append( + create_attribute_json(None,"authored_by",iocDescriptions["authored_by"],True) + ) + if iocDescriptions["authored_date"]: + attributes.append( + create_attribute_json(None,"authored_date",iocDescriptions["authored_date"],True) + ) + + ################################################## + # make short-description in "info field + # if not exist make description + # if "info"="short-description" make descrption as comment + mispInfoFild = "" + if iocDescriptions["short_description"]: + mispInfoFild = iocDescriptions["short_description"] + if iocDescriptions["description"]: + attributes.append( + create_attribute_json(None,"description",iocDescriptions["description"],True) + ) + else: + if iocDescriptions["description"]: + mispInfoFild = iocDescriptions["description"] + else: + mispInfoFild = "No description or short_description from IOC find." + + eventJson = { + "Event": { + "info": mispInfoFild, + "timestamp": "1", + "attribute_count": 0, + "analysis": "0", + "date": time.strftime("%Y-%m-%d"), + "org": "", + "distribution": "0", + "Attribute": [], + "proposal_email_lock": False, + "threat_level_id": "4", + } + } + + eventJson["Event"]["Attribute"] = attributes + + return eventJson + + +def get_descriptions(soup, description): + if soup.find(description.lower()): + return soup.find(description.lower()).text + return "" + +def save_ioc_description(soup): + list_description = ["short_description","authored_by","authored_date","description"] + + for description in list_description: + iocDescriptions[description]=get_descriptions(soup, description) + + return + + + +def get_taxonomy(soup): + import csv + taxonomy = [] + reader = csv.reader(open(csvTaxonomyFile, 'rb'), delimiter=';') + ##################################### + # save file in a dict + # r[0] = @link from csv + # r[1] = @value from csv + # = value + # r[2] = @keep + # 0 : don't creat tag + # 1 : tag created + # r[3] = @taxonomy + + csvdic = {i:r for i,r in enumerate(reader)} + + ######################################### + # find all link with soup + for n in soup.find_all('link', rel=True): + rel = str(n.attrs['rel'][0]).lower() + + ########################## + # build special taxo + # special string because link if a html value + relValue = str(n.next_sibling).strip() + if rel == 'family': + if len(relValue)>0: + taxonomy.append("malware_classification:malware-family='"+relValue+"'") + elif rel == 'threatgroup': + if len(relValue)>0: + taxonomy.append("malware_classification:malware-threatgroup='"+relValue+"'") + + ######################### + # build taxo from csv match + else: + taxo = [r[3] for r in + {i:r for i,r in csvdic.items() + if r[0].lower() == rel and str(r[2])=="1" + }.values() + if r[1].lower() == relValue.lower() and str(r[2])=="1" + ] + + # taxo find in correspondance file + if (len(taxo) > 0 and taxo[0] != '') : + taxonomy.append(taxo[0]) + # not find + return taxonomy + +def custum_color_tag(tagg): + color="#00ace6" + if ":amber" in tagg :color="#ffc200" + if ":green:" in tagg :color="#009933" + if "tlp:green" in tagg :color="#009933" + if ":red:" in tagg :color="#ff0000" + if "tlp:red" in tagg :color="#ff0000" + if "tlp:white" in tagg :color="#fafafa" + return color + +def push_event_to_misp(jsonEvent): + global misp + + #################### + # upload json event + r = misp.add_event(jsonEvent) + event=r.json() + + # save event id for file upload and tagg + iocDescriptions["misp_event_id"]=event["Event"]["id"] + + return + +def upload_file(): + + # filename,path, eid, distrib, ids, categ, info, ids, analysis, threat + misp.upload_sample( + iocDescriptions['filename'], + iocDescriptions["iocfile"], + iocDescriptions["misp_event_id"], + "0", + False, + "External analysis", + iocDescriptions["short_description"], + None, + "1", + "4", + ) + + return + +def update_tag(listOfTagg): + for tagg in listOfTagg: + color = custum_color_tag(tagg) + + ############################# + # creatz tag in MISP + + r = misp.new_tag(str(tagg), str(color)) + ############################# + # link tag to MISP event + toPost={} + toPost['Event']={'id':iocDescriptions["misp_event_id"]} + misp.add_tag( + toPost, + str(tagg)) + return + + +def main(): + global misp + global iocDescriptions + iocDescriptions = {} + + + ################################ + # parse for valid argments + parser = argparse.ArgumentParser(description='Get an event from a MISP instance.') + parser.add_argument("-i", "--input", required=True, help="Input file") + parser.add_argument("-t", "--tag", help="Add custom tags 'tlp:red,cossi:tmp=test'") + args = parser.parse_args() + + iocDescriptions["iocfile"]=os.path.abspath(args.input) + + ################################ + # check if file have ioc extention and if he is not empty + if check_valid_ioc(): + + ################################ + # Try to parse file + iocfileparse = get_parse_ioc_file() + else : + print "/!\ Bad format {0}".format(iocDescriptions["iocfile"]) + return + + ################################ + # save description for create event + save_ioc_description(iocfileparse) + + ################################ + # parse ioc and buid json attributes + jsonAttributes = create_attributes_from_ioc_json(iocfileparse) + + ################################ + # create a json misp event and append attributes + jsonEvent = create_misp_event_json(jsonAttributes) + + + ################################ + # try connection + try: + misp = misp_init(mispUrl, mispKey) + except: + print "/!\ Connection fail, bad url ({0}) or API key : {1}".format(mispUrl,mispKey) + return + + ################################ + # Add event to MSIP + push_event_to_misp(jsonEvent) + + + ################################ + # Upload the IOC file and close tmpfile + upload_file() + + ################################ + # Update MISP Event with tag from IOC + update_tag(get_taxonomy(iocfileparse)) + + ################################ + # Add custom Tag (-t) + if args.tag : + customTag = args.tag + update_tag(customTag.split(",")) + + + +if __name__ == '__main__': + main() diff --git a/examples/ioc-2-misp/keys.py.sample b/examples/ioc-2-misp/keys.py.sample new file mode 100644 index 0000000..5b73563 --- /dev/null +++ b/examples/ioc-2-misp/keys.py.sample @@ -0,0 +1,94 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +mispUrl = '' +mispKey = '' + +############################### +# file use for internal tag +# some sample can be find here : +# https://github.com/eset/malware-ioc +# https://github.com/fireeye/iocs +csvTaxonomyFile = "taxonomy.csv" + +# csv delimiter : ";" with quotechar : " + +############################### +# link sample + #~ + #~ APT + #~ APT12 + #~ Backdoor + #~ Apache 2.0 + #~ + +# @link from csv +# = rel attribut from +# @value from csv +# = value +# @keep +# 0 : don't create tag +# 1 : tag created +# @taxonomy +# define tag for misp +# @comment +# litte description but not use + + +######################################### +# https://www.circl.lu/doc/misp/categories-and-types/index.html +# /\ +# || +# || +# \/ +# http://schemas.mandiant.com/ + +# @index = Context/search form ioc +# @(1, 2, 3) +# 1. categorie mapping +# 2. type mapping +# 3. optionnal comment + + +iocMispMapping = { + + ('DriverItem/DriverName') : (u'Artifacts dropped',u'other', u'DriverName. '), + + ('DnsEntryItem/Host') : (u'Network activity',u'domain'), + + ('Email/To') : (u'Targeting data',u'target-email'), + ('Email/Date') : (u'Other',u'comment',u'EmailDate. '), + ('Email/Body') : (u'Payload delivery',u'email-subject'), + ('Email/From') : (u'Payload delivery',u'email-dst'), + ('Email/Subject') : (u'Payload delivery',u'email-subject'), + ('Email/Attachment/Name') : (u'Payload delivery',u'email-attachment'), + + ('FileItem/Md5sum') : (u'External analysis',u'md5'), + ('FileItem/Sha1sum') : (u'External analysis',u'sha1'), + ('FileItem/FileName') : (u'External analysis',u'filename'), + ('FileItem/FullPath') : (u'External analysis',u'filename'), + ('FileItem/FilePath') : (u'External analysis',u'filename'), + ('FileItem/Sha256sum') : (u'External analysis',u'sha256'), + + ('Network/URI') : (u'Network activity',u'uri'), + ('Network/DNS') : (u'Network activity',u'domain'), + ('Network/String') : (u'Network activity',u'ip-dst'), + ('Network/UserAgent') : (u'Network activity',u'user-agent'), + + ('PortItem/localIP') : (u'Network activity',u'ip-dst'), + + ('ProcessItem/name') : (u'External analysis',u'pattern-in-memory', u'ProcessName. '), + ('ProcessItem/path') : (u'External analysis',u'pattern-in-memory', u'ProcessPath. '), + ('ProcessItem/Mutex') : (u'Artifacts dropped',u'mutex', u'mutex'), + ('ProcessItem/Pipe/Name') : (u'Artifacts dropped',u'named pipe'), + ('ProcessItem/Mutex/Name') : (u'Artifacts dropped',u'mutex', u'MutexName. '), + + ('RegistryItem/Text') : (u'Artifacts dropped',u'regkey', u'RegistryText. '), + ('RegistryItem/Path') : (u'Artifacts dropped',u'regkey', u'RegistryPath. '), + + ('ServiceItem/name') : (u'Artifacts dropped',u'windows-service-name'), + ('ServiceItem/type') : (u'Artifacts dropped',u'pattern-in-memory', u'ServiceType. '), + + ('Snort/Snort') : (u'Network activity',u'snort'), + + } diff --git a/examples/ioc-2-misp/taxonomy.csv b/examples/ioc-2-misp/taxonomy.csv new file mode 100644 index 0000000..73ac977 --- /dev/null +++ b/examples/ioc-2-misp/taxonomy.csv @@ -0,0 +1,12 @@ +link,value,keep,taxonomy,comment +classification,TLP AMBER,1,tlp:amber, +classification,TLP GREEN,1,tlp:green, +confidential,TLP-AMBER,1,tlp:amber, +confidential,TLP GREEN,1,tlp:green, +confidential,TLP-GREEN,1,tlp:green, +confidential,TLP RED,1,tlp:red, +exportable,Yes,0,, +family,APT,1,malware_classification:malware-category='APT', +family,APT3,1,malware_classification:malware-category='APT3',https://github.com/fireeye/iocs/tree/master/APT3 +license,Apache 2.0,0,, +threatcategory,APT3,1,malware_classification:malware-category='APT3',https://github.com/fireeye/iocs/tree/master/APT3