Make pep8 happy

pull/21/head
Raphaël Vinot 2016-06-13 19:14:32 +09:00
parent 03c2a053f4
commit 1da7660934
1 changed files with 266 additions and 276 deletions

542
examples/ioc-2-misp/ioc2misp.py Normal file → Executable file
View File

@ -8,341 +8,331 @@
from keys import mispUrl, mispKey, csvTaxonomyFile, iocMispMapping from keys import mispUrl, mispKey, csvTaxonomyFile, iocMispMapping
try: try:
from pymisp import PyMISP from pymisp import PyMISP
except: except:
print "you need pymisp form github" print("you need pymisp form github")
import sys import sys
sys.exit(1) sys.exit(1)
import json
import os import os
import argparse import argparse
try: try:
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
except: except:
print "install BeautifulSoup : sudo apt-get install python-bs4 python-lxml" print("install BeautifulSoup : sudo apt-get install python-bs4 python-lxml")
import sys import sys
sys.exit(1) sys.exit(1)
def misp_init(url, key): def misp_init(url, key):
return PyMISP(url, key, False, 'json') return PyMISP(url, key, False, 'json')
def check_valid_ioc():
def check_valid_ioc():
(filepath, filename) = os.path.split(iocDescriptions["iocfile"])
(shortname, extension) = os.path.splitext(filename) (filepath, filename) = os.path.split(iocDescriptions["iocfile"])
(shortname, extension) = os.path.splitext(filename)
if (("ioc" in extension)) and (sum(1 for _ in open(iocDescriptions["iocfile"])) > 1):
iocDescriptions['filename'] = filename
return True
return False
if (("ioc" in extension)) and (sum(1 for _ in open(iocDescriptions["iocfile"])) > 1):
iocDescriptions['filename'] = filename
return True
return False
def get_parse_ioc_file(): def get_parse_ioc_file():
return BeautifulSoup(open(iocDescriptions["iocfile"]), "lxml")
return BeautifulSoup(open(iocDescriptions["iocfile"]) , "lxml")
def parse_ioc_search_content(iocContextSearch): def parse_ioc_search_content(iocContextSearch):
for k,v in iocMispMapping.items(): for k, v in iocMispMapping.items():
if str(k).lower() == str(iocContextSearch).lower(): if str(k).lower() == str(iocContextSearch).lower():
return v return v
return False return False
def create_attribute_json(iocContextSearch, attributeValue, attributeComment,force=False):
##################################### def create_attribute_json(iocContextSearch, attributeValue, attributeComment, force=False):
# force used for description to upload #####################################
if force: # force used for description to upload
parseResult=("Other","comment") if force:
else: parseResult = ("Other", "comment")
parseResult = parse_ioc_search_content(iocContextSearch) else:
parseResult = parse_ioc_search_content(iocContextSearch)
if parseResult is False:
if parseResult is False:
print "/!\ Not implemented :: {0} :: {1} :: Item add as 'Other','Comment'. Add it in your keys.py".format(iocContextSearch,attributeValue)
######################################## print("/!\ Not implemented :: {0} :: {1} :: Item add as 'Other','Comment'. Add it in your keys.py".format(iocContextSearch, attributeValue))
# force import to misp ########################################
parseResult=("Other","comment") # force import to misp
parseResult = ("Other", "comment")
comment = ""
try : comment = ""
comment= parseResult[2]+attributeComment try:
except: comment = parseResult[2] + attributeComment
comment= attributeComment except:
comment = attributeComment
attribute = {"category": parseResult[0],
"type": parseResult[1],
"value": attributeValue,
"timestamp": "0",
"to_ids": "0",
"distribution": "0",
"comment": comment
}
return attribute
attribute = {
"category": parseResult[0],
"type": parseResult[1],
"value": attributeValue,
"timestamp": "0",
"to_ids": "0",
"distribution": "0",
"comment": comment,
}
return attribute
def create_attributes_from_ioc_json(soup): def create_attributes_from_ioc_json(soup):
attributes = [] attributes = []
IndicatorItemValues = {} IndicatorItemValues = {}
for item in soup.find_all("indicatoritem"): for item in soup.find_all("indicatoritem"):
if item.find('context'): if item.find('context'):
IndicatorItemValues["context"]=str(item.find('context')['search']) IndicatorItemValues["context"] = str(item.find('context')['search'])
else: else:
IndicatorItemValues["context"]="" IndicatorItemValues["context"] = ""
if item.find('content'): if item.find('content'):
IndicatorItemValues["content"]=str(item.find('content').text) IndicatorItemValues["content"] = str(item.find('content').text)
else: else:
IndicatorItemValues["content"]="" IndicatorItemValues["content"] = ""
if item.find('comment'): if item.find('comment'):
IndicatorItemValues["comment"]=str(item.find('comment').text) IndicatorItemValues["comment"] = str(item.find('comment').text)
else: else:
IndicatorItemValues["comment"]="" IndicatorItemValues["comment"] = ""
jsonAttribute = create_attribute_json(IndicatorItemValues["context"],IndicatorItemValues["content"],IndicatorItemValues["comment"]) jsonAttribute = create_attribute_json(IndicatorItemValues["context"], IndicatorItemValues["content"], IndicatorItemValues["comment"])
attributes.append(jsonAttribute) attributes.append(jsonAttribute)
return attributes return attributes
def create_misp_event_json(attributes): def create_misp_event_json(attributes):
import time import time
if iocDescriptions["authored_by"]: if iocDescriptions["authored_by"]:
attributes.append( attributes.append(create_attribute_json(None, "authored_by", iocDescriptions["authored_by"], True))
create_attribute_json(None,"authored_by",iocDescriptions["authored_by"],True) if iocDescriptions["authored_date"]:
) attributes.append(create_attribute_json(None, "authored_date", iocDescriptions["authored_date"], True))
if iocDescriptions["authored_date"]:
attributes.append(
create_attribute_json(None,"authored_date",iocDescriptions["authored_date"],True)
)
##################################################
# make short-description in "info field
# if not exist make description
# if "info"="short-description" make descrption as comment
mispInfoFild = ""
if iocDescriptions["short_description"]:
mispInfoFild = iocDescriptions["short_description"]
if iocDescriptions["description"]:
attributes.append(
create_attribute_json(None,"description",iocDescriptions["description"],True)
)
else:
if iocDescriptions["description"]:
mispInfoFild = iocDescriptions["description"]
else:
mispInfoFild = "No description or short_description from IOC find."
eventJson = {
"Event": {
"info": mispInfoFild,
"timestamp": "1",
"attribute_count": 0,
"analysis": "0",
"date": time.strftime("%Y-%m-%d"),
"org": "",
"distribution": "0",
"Attribute": [],
"proposal_email_lock": False,
"threat_level_id": "4",
}
}
eventJson["Event"]["Attribute"] = attributes ##################################################
# make short-description in "info field
# if not exist make description
# if "info"="short-description" make descrption as comment
mispInfoFild = ""
if iocDescriptions["short_description"]:
mispInfoFild = iocDescriptions["short_description"]
if iocDescriptions["description"]:
attributes.append(create_attribute_json(None, "description", iocDescriptions["description"], True))
else:
if iocDescriptions["description"]:
mispInfoFild = iocDescriptions["description"]
else:
mispInfoFild = "No description or short_description from IOC find."
return eventJson eventJson = {"Event": {"info": mispInfoFild,
"timestamp": "1",
"attribute_count": 0,
"analysis": "0",
"date": time.strftime("%Y-%m-%d"),
"org": "",
"distribution": "0",
"Attribute": [],
"proposal_email_lock": False,
"threat_level_id": "4",
}}
eventJson["Event"]["Attribute"] = attributes
return eventJson
def get_descriptions(soup, description): def get_descriptions(soup, description):
if soup.find(description.lower()): if soup.find(description.lower()):
return soup.find(description.lower()).text return soup.find(description.lower()).text
return "" return ""
def save_ioc_description(soup): def save_ioc_description(soup):
list_description = ["short_description","authored_by","authored_date","description"] list_description = ["short_description", "authored_by", "authored_date", "description"]
for description in list_description:
iocDescriptions[description]=get_descriptions(soup, description)
return for description in list_description:
iocDescriptions[description] = get_descriptions(soup, description)
return
def get_taxonomy(soup): def get_taxonomy(soup):
import csv import csv
taxonomy = [] taxonomy = []
reader = csv.reader(open(csvTaxonomyFile, 'rb'), delimiter=';') reader = csv.reader(open(csvTaxonomyFile, 'rb'), delimiter=';')
##################################### #####################################
# save file in a dict # save file in a dict
# r[0] = @link from csv # r[0] = @link from csv
# r[1] = @value from csv # r[1] = @value from csv
# = value # = value
# r[2] = @keep # r[2] = @keep
# 0 : don't creat tag # 0 : don't creat tag
# 1 : tag created # 1 : tag created
# r[3] = @taxonomy # r[3] = @taxonomy
csvdic = {i:r for i,r in enumerate(reader)} csvdic = {i: r for i, r in enumerate(reader)}
#########################################
# find all link with soup
for n in soup.find_all('link', rel=True):
rel = str(n.attrs['rel'][0]).lower()
##########################
# build special taxo
# special string because link if a html value
relValue = str(n.next_sibling).strip()
if rel == 'family':
if len(relValue) > 0:
taxonomy.append("malware_classification:malware-family='" + relValue + "'")
elif rel == 'threatgroup':
if len(relValue) > 0:
taxonomy.append("malware_classification:malware-threatgroup='" + relValue + "'")
#########################
# build taxo from csv match
else:
taxo = [r[3] for r in {i: r for i, r in csvdic.items() if r[0].lower() == rel and str(r[2]) == "1"}.values() if r[1].lower() == relValue.lower() and str(r[2]) == "1"]
# taxo find in correspondance file
if (len(taxo) > 0 and taxo[0] != ''):
taxonomy.append(taxo[0])
# not find
return taxonomy
#########################################
# find all link with soup
for n in soup.find_all('link', rel=True):
rel = str(n.attrs['rel'][0]).lower()
##########################
# build special taxo
# special string because link if a html value
relValue = str(n.next_sibling).strip()
if rel == 'family':
if len(relValue)>0:
taxonomy.append("malware_classification:malware-family='"+relValue+"'")
elif rel == 'threatgroup':
if len(relValue)>0:
taxonomy.append("malware_classification:malware-threatgroup='"+relValue+"'")
#########################
# build taxo from csv match
else:
taxo = [r[3] for r in
{i:r for i,r in csvdic.items()
if r[0].lower() == rel and str(r[2])=="1"
}.values()
if r[1].lower() == relValue.lower() and str(r[2])=="1"
]
# taxo find in correspondance file
if (len(taxo) > 0 and taxo[0] != '') :
taxonomy.append(taxo[0])
# not find
return taxonomy
def custum_color_tag(tagg): def custum_color_tag(tagg):
color="#00ace6" color = "#00ace6"
if ":amber" in tagg :color="#ffc200" if ":amber" in tagg:
if ":green:" in tagg :color="#009933" color = "#ffc200"
if "tlp:green" in tagg :color="#009933" if ":green:" in tagg:
if ":red:" in tagg :color="#ff0000" color = "#009933"
if "tlp:red" in tagg :color="#ff0000" if "tlp:green" in tagg:
if "tlp:white" in tagg :color="#fafafa" color = "#009933"
return color if ":red:" in tagg:
color = "#ff0000"
if "tlp:red" in tagg:
color = "#ff0000"
if "tlp:white" in tagg:
color = "#fafafa"
return color
def push_event_to_misp(jsonEvent): def push_event_to_misp(jsonEvent):
global misp global misp
#################### ####################
# upload json event # upload json event
r = misp.add_event(jsonEvent) r = misp.add_event(jsonEvent)
event=r.json() event = r.json()
# save event id for file upload and tagg # save event id for file upload and tagg
iocDescriptions["misp_event_id"]=event["Event"]["id"] iocDescriptions["misp_event_id"] = event["Event"]["id"]
return
return
def upload_file(): def upload_file():
# filename,path, eid, distrib, ids, categ, info, ids, analysis, threat # filename,path, eid, distrib, ids, categ, info, ids, analysis, threat
misp.upload_sample( misp.upload_sample(iocDescriptions['filename'],
iocDescriptions['filename'], iocDescriptions["iocfile"],
iocDescriptions["iocfile"], iocDescriptions["misp_event_id"],
iocDescriptions["misp_event_id"], "0",
"0", False,
False, "External analysis",
"External analysis", iocDescriptions["short_description"],
iocDescriptions["short_description"], None,
None, "1",
"1", "4",
"4", )
) return
return
def update_tag(listOfTagg): def update_tag(listOfTagg):
for tagg in listOfTagg: for tagg in listOfTagg:
color = custum_color_tag(tagg) color = custum_color_tag(tagg)
############################# #############################
# creatz tag in MISP # creatz tag in MISP
r = misp.new_tag(str(tagg), str(color)) misp.new_tag(str(tagg), str(color))
############################# #############################
# link tag to MISP event # link tag to MISP event
toPost={} toPost = {}
toPost['Event']={'id':iocDescriptions["misp_event_id"]} toPost['Event'] = {'id': iocDescriptions["misp_event_id"]}
misp.add_tag( misp.add_tag(toPost, str(tagg))
toPost, return
str(tagg))
return
def main(): def main():
global misp global misp
global iocDescriptions global iocDescriptions
iocDescriptions = {} iocDescriptions = {}
################################
# parse for valid argments
parser = argparse.ArgumentParser(description='Get an event from a MISP instance.')
parser.add_argument("-i", "--input", required=True, help="Input file")
parser.add_argument("-t", "--tag", help="Add custom tags 'tlp:red,cossi:tmp=test'")
args = parser.parse_args()
iocDescriptions["iocfile"]=os.path.abspath(args.input) ################################
# parse for valid argments
################################ parser = argparse.ArgumentParser(description='Get an event from a MISP instance.')
# check if file have ioc extention and if he is not empty parser.add_argument("-i", "--input", required=True, help="Input file")
if check_valid_ioc(): parser.add_argument("-t", "--tag", help="Add custom tags 'tlp:red,cossi:tmp=test'")
args = parser.parse_args()
################################
# Try to parse file
iocfileparse = get_parse_ioc_file()
else :
print "/!\ Bad format {0}".format(iocDescriptions["iocfile"])
return
################################ iocDescriptions["iocfile"] = os.path.abspath(args.input)
# save description for create event
save_ioc_description(iocfileparse)
################################
# parse ioc and buid json attributes
jsonAttributes = create_attributes_from_ioc_json(iocfileparse)
################################ ################################
# create a json misp event and append attributes # check if file have ioc extention and if he is not empty
jsonEvent = create_misp_event_json(jsonAttributes) if check_valid_ioc():
################################
# try connection
try:
misp = misp_init(mispUrl, mispKey)
except:
print "/!\ Connection fail, bad url ({0}) or API key : {1}".format(mispUrl,mispKey)
return
################################
# Add event to MSIP
push_event_to_misp(jsonEvent)
################################
################################ # Try to parse file
# Upload the IOC file and close tmpfile iocfileparse = get_parse_ioc_file()
upload_file() else:
print("/!\ Bad format {0}".format(iocDescriptions["iocfile"]))
return
################################ ################################
# Update MISP Event with tag from IOC # save description for create event
update_tag(get_taxonomy(iocfileparse)) save_ioc_description(iocfileparse)
################################
# Add custom Tag (-t)
if args.tag :
customTag = args.tag
update_tag(customTag.split(","))
################################
# parse ioc and buid json attributes
jsonAttributes = create_attributes_from_ioc_json(iocfileparse)
################################
# create a json misp event and append attributes
jsonEvent = create_misp_event_json(jsonAttributes)
################################
# try connection
try:
misp = misp_init(mispUrl, mispKey)
except:
print("/!\ Connection fail, bad url ({0}) or API key : {1}".format(mispUrl, mispKey))
return
################################
# Add event to MSIP
push_event_to_misp(jsonEvent)
################################
# Upload the IOC file and close tmpfile
upload_file()
################################
# Update MISP Event with tag from IOC
update_tag(get_taxonomy(iocfileparse))
################################
# Add custom Tag (-t)
if args.tag:
customTag = args.tag
update_tag(customTag.split(","))
if __name__ == '__main__': if __name__ == '__main__':
main() main()