PyMISP/examples/ioc_2_misp/ioc2misp.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-

# Format description
# @variables : camelCase
# @functions : snake_case

from keys import mispUrl, mispKey, csvTaxonomyFile, iocMispMapping

try:
        from pymisp import PyMISP
except:
        print("you need pymisp form github")
        import sys
        sys.exit(1)

import os
import argparse

try:
        from bs4 import BeautifulSoup
except:
        print("install BeautifulSoup : sudo apt-get install python-bs4 python-lxml")
        import sys
        sys.exit(1)


def misp_init(url, key):
        return PyMISP(url, key, False, 'json')


def check_valid_ioc():

        (filepath, filename) = os.path.split(iocDescriptions["iocfile"])
        (shortname, extension) = os.path.splitext(filename)

        if (("ioc" in extension)) and (sum(1 for _ in open(iocDescriptions["iocfile"])) > 1):
                iocDescriptions['filename'] = filename
                return True
        return False


def get_parse_ioc_file():
        return BeautifulSoup(open(iocDescriptions["iocfile"]), "lxml")


def parse_ioc_search_content(iocContextSearch):
        for k, v in iocMispMapping.items():
                if str(k).lower() == str(iocContextSearch).lower():
                        return v
        return False


def create_attribute_json(iocContextSearch, attributeValue, attributeComment, force=False):
        #####################################
        # force used for description to upload
        if force:
                parseResult = ("Other", "comment")
        else:
                parseResult = parse_ioc_search_content(iocContextSearch)

        if parseResult is False:

                print("/!\ Not implemented :: {0} :: {1} :: Item add as 'Other','Comment'. Add it in your keys.py".format(iocContextSearch, attributeValue))
                ########################################
                # force import to misp
                parseResult = ("Other", "comment")

        comment = ""
        try:
                comment = parseResult[2] + attributeComment
        except:
                comment = attributeComment

        attribute = {"category": parseResult[0],
                     "type": parseResult[1],
                     "value": attributeValue,
                     "timestamp": "0",
                     "to_ids": "0",
                     "distribution": "0",
                     "comment": comment
                     }
        return attribute


def create_attributes_from_ioc_json(soup):
        attributes = []

        IndicatorItemValues = {}
        for item in soup.find_all("indicatoritem"):

                if item.find('context'):
                        IndicatorItemValues["context"] = str(item.find('context')['search'])
                else:
                        IndicatorItemValues["context"] = ""
                if item.find('content'):
                        IndicatorItemValues["content"] = str(item.find('content').text)
                else:
                        IndicatorItemValues["content"] = ""
                if item.find('comment'):
                        IndicatorItemValues["comment"] = str(item.find('comment').text)
                else:
                        IndicatorItemValues["comment"] = ""

                jsonAttribute = create_attribute_json(IndicatorItemValues["context"], IndicatorItemValues["content"], IndicatorItemValues["comment"])
                attributes.append(jsonAttribute)

        return attributes


def create_misp_event_json(attributes):
        import time
        if iocDescriptions["authored_by"]:
                attributes.append(create_attribute_json(None, "authored_by", iocDescriptions["authored_by"], True))
        if iocDescriptions["authored_date"]:
                attributes.append(create_attribute_json(None, "authored_date", iocDescriptions["authored_date"], True))

        ##################################################
        # make short-description in "info field
        # if not exist make description
        # if "info"="short-description" make descrption as comment
        mispInfoFild = ""
        if iocDescriptions["short_description"]:
                mispInfoFild = iocDescriptions["short_description"]
                if iocDescriptions["description"]:
                        attributes.append(create_attribute_json(None, "description", iocDescriptions["description"], True))
        else:
                if iocDescriptions["description"]:
                        mispInfoFild = iocDescriptions["description"]
                else:
                        mispInfoFild = "No description or short_description from IOC find."

        eventJson = {"Event": {"info": mispInfoFild,
                               "timestamp": "1",
                               "attribute_count": 0,
                               "analysis": "0",
                               "date": time.strftime("%Y-%m-%d"),
                               "org": "",
                               "distribution": "0",
                               "Attribute": [],
                               "proposal_email_lock": False,
                               "threat_level_id": "4",
                               }}

        eventJson["Event"]["Attribute"] = attributes

        return eventJson


def get_descriptions(soup, description):
        if soup.find(description.lower()):
                return soup.find(description.lower()).text
        return ""


def save_ioc_description(soup):
        list_description = ["short_description", "authored_by", "authored_date", "description"]

        for description in list_description:
                iocDescriptions[description] = get_descriptions(soup, description)

        return


def get_taxonomy(soup):
        import csv
        taxonomy = []
        reader = csv.reader(open(csvTaxonomyFile, 'rb'), delimiter=';')
        #####################################
        # save file in a dict
        #       r[0] = @link from csv
        #       r[1] = @value from csv
        #               = value
        #       r[2] = @keep
        #               0 : don't creat tag
        #               1 : tag created
        #       r[3] = @taxonomy

        csvdic = {i: r for i, r in enumerate(reader)}

        #########################################
        # find all link with soup
        for n in soup.find_all('link', rel=True):
                rel = str(n.attrs['rel'][0]).lower()

                ##########################
                # build special taxo
                # special string because link if a html value
                relValue = str(n.next_sibling).strip()
                if rel == 'family':
                        if len(relValue) > 0:
                                taxonomy.append("malware_classification:malware-family='" + relValue + "'")
                elif rel == 'threatgroup':
                        if len(relValue) > 0:
                                taxonomy.append("malware_classification:malware-threatgroup='" + relValue + "'")

                #########################
                # build taxo from csv match
                else:
                        taxo = [r[3] for r in {i: r for i, r in csvdic.items() if r[0].lower() == rel and str(r[2]) == "1"}.values() if r[1].lower() == relValue.lower() and str(r[2]) == "1"]

                        # taxo find in correspondance file
                        if (len(taxo) > 0 and taxo[0] != ''):
                                taxonomy.append(taxo[0])
                        # not find
        return taxonomy


def custum_color_tag(tagg):
        color = "#00ace6"
        if ":amber" in tagg:
            color = "#ffc200"
        if ":green:" in tagg:
            color = "#009933"
        if "tlp:green" in tagg:
            color = "#009933"
        if ":red:" in tagg:
            color = "#ff0000"
        if "tlp:red" in tagg:
            color = "#ff0000"
        if "tlp:white" in tagg:
            color = "#fafafa"
        return color


def push_event_to_misp(jsonEvent):
        global misp

        ####################
        # upload json event
        event = misp.add_event(jsonEvent)

        # save event id for file upload and tagg
        iocDescriptions["misp_event_id"] = event["Event"]["id"]

        return


def upload_file():

        # filename,path, eid, distrib, ids, categ, info, ids, analysis, threat
        misp.upload_sample(iocDescriptions['filename'],
                           iocDescriptions["iocfile"],
                           iocDescriptions["misp_event_id"],
                           "0",
                           False,
                           "External analysis",
                           iocDescriptions["short_description"],
                           None,
                           "1",
                           "4",
                           )
        return


def update_tag(listOfTagg):
        for tagg in listOfTagg:
                color = custum_color_tag(tagg)

                #############################
                # creatz tag in MISP

                misp.new_tag(str(tagg), str(color))
                #############################
                # link tag to MISP event
                toPost = {}
                toPost['Event'] = {'id': iocDescriptions["misp_event_id"]}
                misp.add_tag(toPost, str(tagg))
        return


def main():
        global misp
        global iocDescriptions
        iocDescriptions = {}

        ################################
        # parse for valid argments
        parser = argparse.ArgumentParser(description='Get an event from a MISP instance.')
        parser.add_argument("-i", "--input", required=True, help="Input file")
        parser.add_argument("-t", "--tag", help="Add custom tags 'tlp:red,cossi:tmp=test'")
        args = parser.parse_args()

        iocDescriptions["iocfile"] = os.path.abspath(args.input)

        ################################
        # check if file have ioc extention and if he is not empty
        if check_valid_ioc():

                ################################
                # Try to parse file
                iocfileparse = get_parse_ioc_file()
        else:
                print("/!\ Bad format {0}".format(iocDescriptions["iocfile"]))
                return

        ################################
        # save description for create event
        save_ioc_description(iocfileparse)

        ################################
        # parse ioc and buid json attributes
        jsonAttributes = create_attributes_from_ioc_json(iocfileparse)

        ################################
        # create a json misp event and append attributes
        jsonEvent = create_misp_event_json(jsonAttributes)

        ################################
        # try connection
        try:
                misp = misp_init(mispUrl, mispKey)
        except:
                print("/!\ Connection fail, bad url ({0}) or API key : {1}".format(mispUrl, mispKey))
                return

        ################################
        # Add event to MSIP
        push_event_to_misp(jsonEvent)

        ################################
        # Upload the IOC file and close tmpfile
        upload_file()

        ################################
        # Update MISP Event with tag from IOC
        update_tag(get_taxonomy(iocfileparse))

        ################################
        # Add custom Tag (-t)
        if args.tag:
                customTag = args.tag
                update_tag(customTag.split(","))


if __name__ == '__main__':
        main()