mirror of https://github.com/MISP/PyMISP
commit
dd6381ba11
|
|
@ -0,0 +1,25 @@
|
|||
### Description
|
||||
|
||||
Python script for ioc import to misp
|
||||
|
||||
### requires
|
||||
|
||||
> python 2.7
|
||||
> PyMISP
|
||||
> BeautifulSoup (apt-get install python-bs4 python-lxml)
|
||||
|
||||
### Usage
|
||||
|
||||
```bash
|
||||
python ioc2misp.py -i myioc -t "tag:mytag='sample','tag:other='foo'"
|
||||
```
|
||||
|
||||
```bash
|
||||
time find /iocsample -type f|while read line ;do python ioc2misp.py -i ${line};done
|
||||
```
|
||||
|
||||
### Conf
|
||||
|
||||
* rename keys.py.sample as keys.py
|
||||
* add your url and api key in keys.py
|
||||
* use command in terminal
|
||||
|
|
@ -0,0 +1,348 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Format description
|
||||
# @variables : camelCase
|
||||
# @functions : snake_case
|
||||
|
||||
from keys import mispUrl, mispKey, csvTaxonomyFile, iocMispMapping
|
||||
|
||||
try:
|
||||
from pymisp import PyMISP
|
||||
except:
|
||||
print "you need pymisp form github"
|
||||
import sys
|
||||
sys.exit(1)
|
||||
|
||||
import json
|
||||
import os
|
||||
import argparse
|
||||
|
||||
try:
|
||||
from bs4 import BeautifulSoup
|
||||
except:
|
||||
print "install BeautifulSoup : sudo apt-get install python-bs4 python-lxml"
|
||||
import sys
|
||||
sys.exit(1)
|
||||
|
||||
def misp_init(url, key):
|
||||
return PyMISP(url, key, False, 'json')
|
||||
|
||||
def check_valid_ioc():
|
||||
|
||||
(filepath, filename) = os.path.split(iocDescriptions["iocfile"])
|
||||
(shortname, extension) = os.path.splitext(filename)
|
||||
|
||||
if (("ioc" in extension)) and (sum(1 for _ in open(iocDescriptions["iocfile"])) > 1):
|
||||
iocDescriptions['filename'] = filename
|
||||
return True
|
||||
return False
|
||||
|
||||
def get_parse_ioc_file():
|
||||
|
||||
return BeautifulSoup(open(iocDescriptions["iocfile"]) , "lxml")
|
||||
|
||||
def parse_ioc_search_content(iocContextSearch):
|
||||
for k,v in iocMispMapping.items():
|
||||
if str(k).lower() == str(iocContextSearch).lower():
|
||||
return v
|
||||
return False
|
||||
|
||||
def create_attribute_json(iocContextSearch, attributeValue, attributeComment,force=False):
|
||||
#####################################
|
||||
# force used for description to upload
|
||||
if force:
|
||||
parseResult=("Other","comment")
|
||||
else:
|
||||
parseResult = parse_ioc_search_content(iocContextSearch)
|
||||
|
||||
if parseResult is False:
|
||||
|
||||
print "/!\ Not implemented :: {0} :: {1} :: Item add as 'Other','Comment'. Add it in your keys.py".format(iocContextSearch,attributeValue)
|
||||
########################################
|
||||
# force import to misp
|
||||
parseResult=("Other","comment")
|
||||
|
||||
comment = ""
|
||||
try :
|
||||
comment= parseResult[2]+attributeComment
|
||||
except:
|
||||
comment= attributeComment
|
||||
|
||||
attribute = {
|
||||
"category": parseResult[0],
|
||||
"type": parseResult[1],
|
||||
"value": attributeValue,
|
||||
"timestamp": "0",
|
||||
"to_ids": "0",
|
||||
"distribution": "0",
|
||||
"comment": comment,
|
||||
}
|
||||
return attribute
|
||||
|
||||
def create_attributes_from_ioc_json(soup):
|
||||
attributes = []
|
||||
|
||||
IndicatorItemValues = {}
|
||||
for item in soup.find_all("indicatoritem"):
|
||||
|
||||
if item.find('context'):
|
||||
IndicatorItemValues["context"]=str(item.find('context')['search'])
|
||||
else:
|
||||
IndicatorItemValues["context"]=""
|
||||
if item.find('content'):
|
||||
IndicatorItemValues["content"]=str(item.find('content').text)
|
||||
else:
|
||||
IndicatorItemValues["content"]=""
|
||||
if item.find('comment'):
|
||||
IndicatorItemValues["comment"]=str(item.find('comment').text)
|
||||
else:
|
||||
IndicatorItemValues["comment"]=""
|
||||
|
||||
|
||||
jsonAttribute = create_attribute_json(IndicatorItemValues["context"],IndicatorItemValues["content"],IndicatorItemValues["comment"])
|
||||
attributes.append(jsonAttribute)
|
||||
|
||||
return attributes
|
||||
|
||||
|
||||
def create_misp_event_json(attributes):
|
||||
import time
|
||||
if iocDescriptions["authored_by"]:
|
||||
attributes.append(
|
||||
create_attribute_json(None,"authored_by",iocDescriptions["authored_by"],True)
|
||||
)
|
||||
if iocDescriptions["authored_date"]:
|
||||
attributes.append(
|
||||
create_attribute_json(None,"authored_date",iocDescriptions["authored_date"],True)
|
||||
)
|
||||
|
||||
##################################################
|
||||
# make short-description in "info field
|
||||
# if not exist make description
|
||||
# if "info"="short-description" make descrption as comment
|
||||
mispInfoFild = ""
|
||||
if iocDescriptions["short_description"]:
|
||||
mispInfoFild = iocDescriptions["short_description"]
|
||||
if iocDescriptions["description"]:
|
||||
attributes.append(
|
||||
create_attribute_json(None,"description",iocDescriptions["description"],True)
|
||||
)
|
||||
else:
|
||||
if iocDescriptions["description"]:
|
||||
mispInfoFild = iocDescriptions["description"]
|
||||
else:
|
||||
mispInfoFild = "No description or short_description from IOC find."
|
||||
|
||||
eventJson = {
|
||||
"Event": {
|
||||
"info": mispInfoFild,
|
||||
"timestamp": "1",
|
||||
"attribute_count": 0,
|
||||
"analysis": "0",
|
||||
"date": time.strftime("%Y-%m-%d"),
|
||||
"org": "",
|
||||
"distribution": "0",
|
||||
"Attribute": [],
|
||||
"proposal_email_lock": False,
|
||||
"threat_level_id": "4",
|
||||
}
|
||||
}
|
||||
|
||||
eventJson["Event"]["Attribute"] = attributes
|
||||
|
||||
return eventJson
|
||||
|
||||
|
||||
def get_descriptions(soup, description):
|
||||
if soup.find(description.lower()):
|
||||
return soup.find(description.lower()).text
|
||||
return ""
|
||||
|
||||
def save_ioc_description(soup):
|
||||
list_description = ["short_description","authored_by","authored_date","description"]
|
||||
|
||||
for description in list_description:
|
||||
iocDescriptions[description]=get_descriptions(soup, description)
|
||||
|
||||
return
|
||||
|
||||
|
||||
|
||||
def get_taxonomy(soup):
|
||||
import csv
|
||||
taxonomy = []
|
||||
reader = csv.reader(open(csvTaxonomyFile, 'rb'), delimiter=';')
|
||||
#####################################
|
||||
# save file in a dict
|
||||
# r[0] = @link from csv
|
||||
# r[1] = @value from csv
|
||||
# = value
|
||||
# r[2] = @keep
|
||||
# 0 : don't creat tag
|
||||
# 1 : tag created
|
||||
# r[3] = @taxonomy
|
||||
|
||||
csvdic = {i:r for i,r in enumerate(reader)}
|
||||
|
||||
#########################################
|
||||
# find all link with soup
|
||||
for n in soup.find_all('link', rel=True):
|
||||
rel = str(n.attrs['rel'][0]).lower()
|
||||
|
||||
##########################
|
||||
# build special taxo
|
||||
# special string because link if a html value
|
||||
relValue = str(n.next_sibling).strip()
|
||||
if rel == 'family':
|
||||
if len(relValue)>0:
|
||||
taxonomy.append("malware_classification:malware-family='"+relValue+"'")
|
||||
elif rel == 'threatgroup':
|
||||
if len(relValue)>0:
|
||||
taxonomy.append("malware_classification:malware-threatgroup='"+relValue+"'")
|
||||
|
||||
#########################
|
||||
# build taxo from csv match
|
||||
else:
|
||||
taxo = [r[3] for r in
|
||||
{i:r for i,r in csvdic.items()
|
||||
if r[0].lower() == rel and str(r[2])=="1"
|
||||
}.values()
|
||||
if r[1].lower() == relValue.lower() and str(r[2])=="1"
|
||||
]
|
||||
|
||||
# taxo find in correspondance file
|
||||
if (len(taxo) > 0 and taxo[0] != '') :
|
||||
taxonomy.append(taxo[0])
|
||||
# not find
|
||||
return taxonomy
|
||||
|
||||
def custum_color_tag(tagg):
|
||||
color="#00ace6"
|
||||
if ":amber" in tagg :color="#ffc200"
|
||||
if ":green:" in tagg :color="#009933"
|
||||
if "tlp:green" in tagg :color="#009933"
|
||||
if ":red:" in tagg :color="#ff0000"
|
||||
if "tlp:red" in tagg :color="#ff0000"
|
||||
if "tlp:white" in tagg :color="#fafafa"
|
||||
return color
|
||||
|
||||
def push_event_to_misp(jsonEvent):
|
||||
global misp
|
||||
|
||||
####################
|
||||
# upload json event
|
||||
r = misp.add_event(jsonEvent)
|
||||
event=r.json()
|
||||
|
||||
# save event id for file upload and tagg
|
||||
iocDescriptions["misp_event_id"]=event["Event"]["id"]
|
||||
|
||||
return
|
||||
|
||||
def upload_file():
|
||||
|
||||
# filename,path, eid, distrib, ids, categ, info, ids, analysis, threat
|
||||
misp.upload_sample(
|
||||
iocDescriptions['filename'],
|
||||
iocDescriptions["iocfile"],
|
||||
iocDescriptions["misp_event_id"],
|
||||
"0",
|
||||
False,
|
||||
"External analysis",
|
||||
iocDescriptions["short_description"],
|
||||
None,
|
||||
"1",
|
||||
"4",
|
||||
)
|
||||
|
||||
return
|
||||
|
||||
def update_tag(listOfTagg):
|
||||
for tagg in listOfTagg:
|
||||
color = custum_color_tag(tagg)
|
||||
|
||||
#############################
|
||||
# creatz tag in MISP
|
||||
|
||||
r = misp.new_tag(str(tagg), str(color))
|
||||
#############################
|
||||
# link tag to MISP event
|
||||
toPost={}
|
||||
toPost['Event']={'id':iocDescriptions["misp_event_id"]}
|
||||
misp.add_tag(
|
||||
toPost,
|
||||
str(tagg))
|
||||
return
|
||||
|
||||
|
||||
def main():
|
||||
global misp
|
||||
global iocDescriptions
|
||||
iocDescriptions = {}
|
||||
|
||||
|
||||
################################
|
||||
# parse for valid argments
|
||||
parser = argparse.ArgumentParser(description='Get an event from a MISP instance.')
|
||||
parser.add_argument("-i", "--input", required=True, help="Input file")
|
||||
parser.add_argument("-t", "--tag", help="Add custom tags 'tlp:red,cossi:tmp=test'")
|
||||
args = parser.parse_args()
|
||||
|
||||
iocDescriptions["iocfile"]=os.path.abspath(args.input)
|
||||
|
||||
################################
|
||||
# check if file have ioc extention and if he is not empty
|
||||
if check_valid_ioc():
|
||||
|
||||
################################
|
||||
# Try to parse file
|
||||
iocfileparse = get_parse_ioc_file()
|
||||
else :
|
||||
print "/!\ Bad format {0}".format(iocDescriptions["iocfile"])
|
||||
return
|
||||
|
||||
################################
|
||||
# save description for create event
|
||||
save_ioc_description(iocfileparse)
|
||||
|
||||
################################
|
||||
# parse ioc and buid json attributes
|
||||
jsonAttributes = create_attributes_from_ioc_json(iocfileparse)
|
||||
|
||||
################################
|
||||
# create a json misp event and append attributes
|
||||
jsonEvent = create_misp_event_json(jsonAttributes)
|
||||
|
||||
|
||||
################################
|
||||
# try connection
|
||||
try:
|
||||
misp = misp_init(mispUrl, mispKey)
|
||||
except:
|
||||
print "/!\ Connection fail, bad url ({0}) or API key : {1}".format(mispUrl,mispKey)
|
||||
return
|
||||
|
||||
################################
|
||||
# Add event to MSIP
|
||||
push_event_to_misp(jsonEvent)
|
||||
|
||||
|
||||
################################
|
||||
# Upload the IOC file and close tmpfile
|
||||
upload_file()
|
||||
|
||||
################################
|
||||
# Update MISP Event with tag from IOC
|
||||
update_tag(get_taxonomy(iocfileparse))
|
||||
|
||||
################################
|
||||
# Add custom Tag (-t)
|
||||
if args.tag :
|
||||
customTag = args.tag
|
||||
update_tag(customTag.split(","))
|
||||
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
|
@ -0,0 +1,94 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
mispUrl = ''
|
||||
mispKey = ''
|
||||
|
||||
###############################
|
||||
# file use for internal tag
|
||||
# some sample can be find here :
|
||||
# https://github.com/eset/malware-ioc
|
||||
# https://github.com/fireeye/iocs
|
||||
csvTaxonomyFile = "taxonomy.csv"
|
||||
|
||||
# csv delimiter : ";" with quotechar : "
|
||||
|
||||
###############################
|
||||
# link sample
|
||||
#~ <links>
|
||||
#~ <link rel="threatcategory">APT</link>
|
||||
#~ <link rel="threatgroup">APT12</link>
|
||||
#~ <link rel="category">Backdoor</link>
|
||||
#~ <link rel="license">Apache 2.0</link>
|
||||
#~ </links>
|
||||
|
||||
# @link from csv
|
||||
# = rel attribut from <link>
|
||||
# @value from csv
|
||||
# = value
|
||||
# @keep
|
||||
# 0 : don't create tag
|
||||
# 1 : tag created
|
||||
# @taxonomy
|
||||
# define tag for misp
|
||||
# @comment
|
||||
# litte description but not use
|
||||
|
||||
|
||||
#########################################
|
||||
# https://www.circl.lu/doc/misp/categories-and-types/index.html
|
||||
# /\
|
||||
# ||
|
||||
# ||
|
||||
# \/
|
||||
# http://schemas.mandiant.com/
|
||||
|
||||
# @index = Context/search form ioc
|
||||
# @(1, 2, 3)
|
||||
# 1. categorie mapping
|
||||
# 2. type mapping
|
||||
# 3. optionnal comment
|
||||
|
||||
|
||||
iocMispMapping = {
|
||||
|
||||
('DriverItem/DriverName') : (u'Artifacts dropped',u'other', u'DriverName. '),
|
||||
|
||||
('DnsEntryItem/Host') : (u'Network activity',u'domain'),
|
||||
|
||||
('Email/To') : (u'Targeting data',u'target-email'),
|
||||
('Email/Date') : (u'Other',u'comment',u'EmailDate. '),
|
||||
('Email/Body') : (u'Payload delivery',u'email-subject'),
|
||||
('Email/From') : (u'Payload delivery',u'email-dst'),
|
||||
('Email/Subject') : (u'Payload delivery',u'email-subject'),
|
||||
('Email/Attachment/Name') : (u'Payload delivery',u'email-attachment'),
|
||||
|
||||
('FileItem/Md5sum') : (u'External analysis',u'md5'),
|
||||
('FileItem/Sha1sum') : (u'External analysis',u'sha1'),
|
||||
('FileItem/FileName') : (u'External analysis',u'filename'),
|
||||
('FileItem/FullPath') : (u'External analysis',u'filename'),
|
||||
('FileItem/FilePath') : (u'External analysis',u'filename'),
|
||||
('FileItem/Sha256sum') : (u'External analysis',u'sha256'),
|
||||
|
||||
('Network/URI') : (u'Network activity',u'uri'),
|
||||
('Network/DNS') : (u'Network activity',u'domain'),
|
||||
('Network/String') : (u'Network activity',u'ip-dst'),
|
||||
('Network/UserAgent') : (u'Network activity',u'user-agent'),
|
||||
|
||||
('PortItem/localIP') : (u'Network activity',u'ip-dst'),
|
||||
|
||||
('ProcessItem/name') : (u'External analysis',u'pattern-in-memory', u'ProcessName. '),
|
||||
('ProcessItem/path') : (u'External analysis',u'pattern-in-memory', u'ProcessPath. '),
|
||||
('ProcessItem/Mutex') : (u'Artifacts dropped',u'mutex', u'mutex'),
|
||||
('ProcessItem/Pipe/Name') : (u'Artifacts dropped',u'named pipe'),
|
||||
('ProcessItem/Mutex/Name') : (u'Artifacts dropped',u'mutex', u'MutexName. '),
|
||||
|
||||
('RegistryItem/Text') : (u'Artifacts dropped',u'regkey', u'RegistryText. '),
|
||||
('RegistryItem/Path') : (u'Artifacts dropped',u'regkey', u'RegistryPath. '),
|
||||
|
||||
('ServiceItem/name') : (u'Artifacts dropped',u'windows-service-name'),
|
||||
('ServiceItem/type') : (u'Artifacts dropped',u'pattern-in-memory', u'ServiceType. '),
|
||||
|
||||
('Snort/Snort') : (u'Network activity',u'snort'),
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
link,value,keep,taxonomy,comment
|
||||
classification,TLP AMBER,1,tlp:amber,
|
||||
classification,TLP GREEN,1,tlp:green,
|
||||
confidential,TLP-AMBER,1,tlp:amber,
|
||||
confidential,TLP GREEN,1,tlp:green,
|
||||
confidential,TLP-GREEN,1,tlp:green,
|
||||
confidential,TLP RED,1,tlp:red,
|
||||
exportable,Yes,0,,
|
||||
family,APT,1,malware_classification:malware-category='APT',
|
||||
family,APT3,1,malware_classification:malware-category='APT3',https://github.com/fireeye/iocs/tree/master/APT3
|
||||
license,Apache 2.0,0,,
|
||||
threatcategory,APT3,1,malware_classification:malware-category='APT3',https://github.com/fireeye/iocs/tree/master/APT3
|
||||
|
Loading…
Reference in New Issue