# -*- coding: utf-8 -*- import os from pymisp import MISPEvent try: from bs4 import BeautifulSoup has_bs4 = True except ImportError: has_bs4 = False iocMispMapping = { # ~ @Link https://wiki.ops.fr/doku.php/manuels:misp:event-guidelines 'CookieHistoryItem/HostName': {'type': 'hostname', 'comment': 'CookieHistory.'}, 'DriverItem/DriverName': {'category': 'Artifacts dropped', 'type': 'other', 'comment': 'DriverName.'}, 'DriverItem/CertificateIssuer': {'category': 'Artifacts dropped', 'type': 'other', 'comment': 'DriverCertificateIssuer.'}, 'DriverItem/DeviceItem/AttachedDeviceName': {'category': 'Artifacts dropped', 'type': 'other', 'comment': 'DriverDeviceName. '}, 'DnsEntryItem/Host': {'type': 'domain'}, 'DnsEntryItem/RecordName': {'type': 'domain'}, 'Email/To': {'type': 'target-email'}, 'Email/Date': {'type': 'comment', 'comment': 'EmailDate.'}, 'Email/Body': {'type': 'email-subject'}, 'Email/From': {'type': 'email-dst'}, 'Email/Subject': {'type': 'email-subject'}, 'Email/Attachment/Name': {'type': 'email-attachment'}, 'FileItem/Md5sum': {'type': 'md5'}, 'FileItem/Sha1sum': {'type': 'sha1'}, 'FileItem/FileName': {'type': 'filename'}, 'FileItem/FullPath': {'type': 'filename'}, 'FileItem/FilePath': {'type': 'filename'}, 'FileItem/Sha256sum': {'type': 'sha256'}, 'FileItem/DevicePath': {'type': 'comment', 'comment': 'DevicePath. '}, 'FileItem/SizeInBytes': {'type': 'size-in-bytes'}, 'FileItem/PEInfo/Type': {'type': 'comment', 'comment': 'PE Type. '}, 'FileItem/FileExtension': {'type': 'comment', 'comment': 'FileExtension. '}, 'FileItem/FilenameCreated': {'type': 'filename', 'comment': 'FilenameCreated. '}, 'FileItem/StringList/string': {'type': 'pattern-in-file', 'comment': 'string list. '}, 'FileItem/PEInfo/PETimeStamp': {'type': 'pattern-in-file', 'comment': 'PE TimeStamp. '}, 'FileItem/PEInfo/Exports/DllName': {'type': 'pattern-in-memory', 'comment': 'PE export DllName. '}, 'FileItem/PEInfo/Sections/Section/Name': {'type': 'pattern-in-memory', 'comment': 'PE SectionName. '}, 'FileItem/PEInfo/DetectedAnomalies/string': {'type': 'pattern-in-file', 'comment': 'PE DEtected AnomaliesString. '}, 'FileItem/PEInfo/Exports/NumberOfFunctions': {'type': 'pattern-in-file', 'comment': 'PE Export NumberOfFunctions. '}, 'FileItem/PEInfo/ImportedModules/Module/Name': {'type': 'pattern-in-file', 'comment': 'PE ImportedModulesName. '}, 'FileItem/PEInfo/DigitalSignature/Description': {'type': 'comment', 'comment': 'PE DigitalSignatureDescription. '}, 'FileItem/PEInfo/DigitalSignature/SignatureExists': {'type': 'comment', 'comment': 'PE SignatureExists. '}, 'FileItem/PEInfo/Exports/ExportedFunctions/string': {'type': 'comment', 'comment': 'PE ExportedFunctions. '}, 'FileItem/PEInfo/DigitalSignature/CertificateIssuer': {'type': 'comment', 'comment': 'PE SignatureCertificateIssuer. '}, 'FileItem/PEInfo/DigitalSignature/SignatureVerified': {'type': 'comment', 'comment': 'PE SignatureVerified. '}, 'FileItem/PEInfo/DigitalSignature/CertificateSubject': {'type': 'other', 'comment': 'PE CertificateDigitalSignatureSubject. '}, 'FileItem/PEInfo/ResourceInfoList/ResourceInfoItem/Name': {'type': 'comment', 'comment': 'PE ResourceName. '}, 'FileItem/PEInfo/ResourceInfoList/ResourceInfoItem/Type': {'type': 'comment', 'comment': 'PE ResourceType. '}, 'FileItem/PEInfo/VersionInfoList/VersionInfoItem/Language': {'type': 'pattern-in-file', 'comment': 'PE LanguageVersion. '}, 'FileItem/PEInfo/ResourceInfoList/ResourceInfoItem/Language': {'type': 'pattern-in-file', 'comment': 'PE LanguageResource. '}, 'FileItem/PEInfo/VersionInfoList/VersionInfoItem/CompanyName': {'type': 'pattern-in-file', 'comment': 'PE versionInfo CompanyName. '}, 'FileItem/PEInfo/VersionInfoList/VersionInfoItem/FileVersion': {'type': 'pattern-in-file', 'comment': 'PE Version. '}, 'FileItem/PEInfo/VersionInfoList/VersionInfoItem/ProductName': {'type': 'pattern-in-file', 'comment': 'PE ProductName. '}, 'FileItem/PEInfo/VersionInfoList/VersionInfoItem/InternalName': {'type': 'pattern-in-file', 'comment': 'PE InternalName. '}, 'FileItem/PEInfo/VersionInfoList/VersionInfoItem/LegalCopyright': {'type': 'pattern-in-file', 'comment': 'PE LegalCopyright. '}, 'FileItem/PEInfo/VersionInfoList/VersionInfoItem/ProductVersion': {'type': 'pattern-in-file', 'comment': 'PE ProductVersion. '}, 'FileItem/PEInfo/VersionInfoList/VersionInfoItem/FileDescription': {'type': 'comment', 'comment': 'PE FileDescription .'}, 'FileItem/PEInfo/ImportedModules/Module/ImportedFunctions/string': {'type': 'pattern-in-file', 'comment': 'PE ImportedModules. '}, 'FileItem/PEInfo/VersionInfoList/VersionInfoItem/OriginalFilename': {'type': 'pattern-in-file', 'comment': 'OriginalFilename of PE. '}, 'FormHistoryItem/HostName': {'type': 'hostname', 'comment': 'FormHistory. '}, 'Network/URI': {'type': 'uri'}, 'Network/DNS': {'type': 'domain'}, 'Network/String': {'type': 'url'}, 'Network/IPRange': {'type': 'ip-dst'}, 'Network/UserAgent': {'type': 'user-agent'}, 'PortItem/localIP': {'type': 'ip-src'}, 'PortItem/remoteIP': {'type': 'ip-dst'}, 'PortItem/remotePort': {'type': 'pattern-in-traffic', 'comment': 'RemotePort. '}, 'ProcessItem/name': {'type': 'pattern-in-memory', 'comment': 'ProcessName. '}, 'ProcessItem/path': {'type': 'pattern-in-memory', 'comment': 'ProcessPath. '}, 'ProcessItem/Mutex': {'type': 'mutex', 'comment': 'mutex'}, 'ProcessItem/arguments': {'type': 'pattern-in-memory', 'comment': 'ProcessArguments. '}, 'ProcessItem/NamedPipe': {'type': 'named pipe'}, 'ProcessItem/Pipe/Name': {'type': 'named pipe'}, 'ProcessItem/Mutex/Name': {'type': 'mutex', 'comment': 'MutexName. '}, 'ProcessItem/Event/Name': {'type': 'pattern-in-memory', 'comment': 'ProcessEventName. '}, 'ProcessItem/StringList/string': {'type': 'pattern-in-memory', 'comment': 'StringlistName. '}, 'ProcessItem/HandleList/Handle/Name': {'type': 'pattern-in-memory', 'comment': 'ProcessHandleListName'}, 'ProcessItem/HandleList/Handle/Type': {'type': 'pattern-in-memory', 'comment': 'ProcessHandleType'}, 'ProcessItem/SectionList/MemorySection/Name': {'type': 'pattern-in-memory', 'comment': 'ProcessSectionMemoryName'}, 'ProcessItem/SectionList/MemorySection/PEInfo/Exports/DllName': {'type': 'pattern-in-memory', 'comment': 'ProcessMemoryPEExportsDllName'}, 'ProcessItem/SectionList/MemorySection/PEInfo/Sections/Section/Name': {'type': 'pattern-in-memory', 'comment': 'Section name from PE in process memory section'}, 'RegistryItem/Text': {'type': 'regkey', 'comment': 'RegistryText. '}, 'RegistryItem/Path': {'type': 'regkey', 'comment': 'RegistryPath. '}, 'RegistryItem/Value': {'type': 'regkey', 'comment': 'RegistryValue. '}, 'RegistryItem/KeyPath': {'type': 'regkey', 'comment': 'RegistryKeyPath. '}, 'RegistryItem/ValueName': {'type': 'regkey', 'comment': 'RegistryValueName. '}, 'RouteEntryItem/Destination': {'type': 'ip-dst'}, 'RouteEntryItem/Destination/IP': {'type': 'ip-dst', 'comment': 'RouteDestination. '}, 'RouteEntryItem/Destination/string': {'type': 'url', 'comment': 'RouteDestination. '}, 'ServiceItem/name': {'type': 'windows-service-name'}, 'ServiceItem/type': {'type': 'pattern-in-memory', 'comment': 'ServiceType. '}, 'ServiceItem/startedAs': {'type': 'pattern-in-memory', 'comment': 'ServiceStartedAs. '}, 'ServiceItem/serviceDLL': {'type': 'pattern-in-memory', 'comment': 'ServiceDll. '}, 'ServiceItem/description': {'type': 'comment', 'comment': 'ServiceDescription. '}, 'ServiceItem/descriptiveName': {'type': 'windows-service-displayname'}, 'ServiceItem/serviceDLLmd5sum': {'type': 'md5', 'comment': 'ServiceDLL. '}, 'ServiceItem/serviceDLLsha1sum': {'type': 'sha1', 'comment': 'ServiceDLL. '}, 'ServiceItem/serviceDLLsha256sum': {'type': 'sha256', 'comment': 'ServiceDLL. '}, 'ServiceItem/serviceDLLSignatureVerified': {'type': 'pattern-in-memory', 'comment': 'ServiceDllSignatureVerified. '}, 'Snort/Snort': {'type': 'snort'}, 'SystemInfoItem/HostName': {'type': 'hostname', 'comment': 'SystemInfo. '}, 'TaskItem/Name': {'type': 'windows-scheduled-task', 'comment': 'TaskName. '}, 'TaskItem/sha1sum': {'type': 'windows-scheduled-task', 'comment': 'TashSha1. '}, 'TaskItem/sha256sum': {'type': 'windows-scheduled-task', 'comment': 'TashSha256. '}, 'TaskItem/AccountName': {'type': 'windows-scheduled-task', 'comment': 'TaskAccountName'}, 'TaskItem/ActionList/Action/ExecProgramPath': {'type': 'windows-scheduled-task', 'comment': 'TaskExecProgramPath. '}, 'TaskItem/TriggerList/Trigger/TriggerFrequency': {'type': 'windows-scheduled-task', 'comment': 'TaskTriggerFrequency. '}, 'UrlHistoryItem/URL': {'type': 'url', 'comment': 'UrlHistory. '}, 'UrlHistoryItem/HostName': {'type': 'hostname', 'comment': 'UrlHistory. '}, 'Yara/Yara': {'type': 'yara'} } iocMispCompositeMapping = { # mapping for composite object # maybe later filename|sizeinbyte 'FileItem/FileName|FileItem/Md5sum': {'type': 'filename|md5'}, 'FileItem/FileName|FileItem/Sha1sum': {'type': 'filename|sha1'}, 'FileItem/FileName|FileItem/Sha256sum': {'type': 'filename|sha256'}, 'Network/DNS|PortItem/remoteIP': {'type': 'domain|ip'}, 'PortItem/remoteIP|PortItem/remotePort': {'comment': 'ip-dst|port'}, 'RegistryItem/Path|RegistryItem/Value': {'type': 'regkey|value'}, 'RegistryItem/KeyPath|RegistryItem/Value': {'type': 'regkey|value'}, 'RegistryItem/Path|RegistryItem/Text': {'type': 'regkey|value'} } def extract_field(report, field_name): if report: data = report.find(field_name.lower()) if data and hasattr(data, 'text'): return data.text return '' def load_openioc_file(openioc_path): if not os.path.exists(openioc_path): raise Exception("Path doesn't exists.") with open(openioc_path, 'r') as f: return load_openioc(f) def load_openioc(openioc): # Takes a opened file, or a string if not has_bs4: raise Exception('You need to install BeautifulSoup: pip install bs4') misp_event = MISPEvent() iocreport = BeautifulSoup(openioc, "html.parser") # Set event fields info = extract_field(iocreport, 'short_description') if info: misp_event.info = info date = extract_field(iocreport, 'authored_date') if date: misp_event.set_date(date) # Set special attributes description = extract_field(iocreport, 'description') if description: if not misp_event.info: misp_event.info = description else: misp_event.add_attribute('comment', description) if not misp_event.info: misp_event.info = 'OpenIOC import' author = extract_field(iocreport, 'authored_by') if author: misp_event.add_attribute('comment', author) misp_event = set_all_attributes(iocreport, misp_event) return misp_event def get_mapping(openioc_type, mappingDict=iocMispMapping): t = openioc_type.lower() for k, v in mappingDict.items(): if k.lower() == t: return v return False def set_values(value1, value2=None): attribute_values = {} if value2 is not None: # construct attribut composite value value = "{}|{}".format(extract_field(value1, 'Content'), extract_field(value2, 'Content') ) else: value = extract_field(value1, 'Content') if value: attribute_values['value'] = value else: return None if value2 is not None: # construct attribut composite type compositeMapping = '{}|{}'.format(value1.find('context')['search'], value2.find('context')['search']) mapping = get_mapping(compositeMapping, mappingDict=iocMispCompositeMapping) else: mapping = get_mapping(value1.find('context')['search']) if mapping: attribute_values.update(mapping) else: # Unknown mapping, assign to default attribute_values['category'] = 'External analysis' attribute_values['type'] = 'other' # change value to composite # 127.0.0.1:80 ip-* to 127.0.0.1|80 ip-*|port if attribute_values['type'] in ['ip-src', 'ip-dst'] and attribute_values['value'].count(':') == 1: attribute_values['type'] = attribute_values['type'] + '|port' attribute_values['value'] = attribute_values['value'].replace(':', '|') attribute_values["comment"] = '{}{}'.format(extract_field(value1, 'Comment'), extract_field(value2, 'Comment') ) return attribute_values def set_all_attributes(openioc, misp_event): processed = set() # check for composite item for composite in openioc.find_all("indicator", operator="AND"): # check for composite number under childs = composite.find_all('indicatoritem') if len(childs) == 2: childList = [child.find('context')['search'] for child in childs] def check_and_add(value1, value2): if (value1 and value2) in childList: if childs[0].find('context')['search'] == value1: attribute_values = set_values(childs[0], childs[1]) else: attribute_values = set_values(childs[1], childs[0]) misp_event.add_attribute(**attribute_values) processed.add(childs[0]['id']) processed.add(childs[1]['id']) for k in iocMispCompositeMapping: check_and_add(k.split('|')[0], k.split('|')[1]) for item in openioc.find_all("indicatoritem"): # check if id in processed list if item['id'] in processed: continue attribute_values = set_values(item) if attribute_values is None: continue misp_event.add_attribute(**attribute_values) return misp_event if __name__ == '__main__': import requests # test file for composite url = 'https://raw.githubusercontent.com/fireeye/iocs/master/BlogPosts/9cee306d-5441-4cd3-932d-f3119752634c.ioc' # ~ url = 'https://raw.githubusercontent.com/MISP/misp-modules/master/tests/openioc.xml' x = requests.get(url) mispEvent = load_openioc(x.text) print(mispEvent) # ~ from pymisp import PyMISP # ~ misp = PyMISP('http://misp.local', 'xxxxx') # ~ r = misp.add_event(mispEvent) # ~ print(r)