Added a dictionary to manage patterns and observable objects

pull/2566/head
chrisr3d 2017-10-13 14:43:39 +02:00
parent 4b339751ee
commit dbad853a16
1 changed files with 173 additions and 92 deletions

View File

@ -15,7 +15,7 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import sys, json, os, time, datetime, re
import sys, json, os, datetime, re
from stix2 import *
namespace = ['https://github.com/MISP/MISP', 'MISP']
@ -26,6 +26,105 @@ non_indicator_attributes = ['text', 'comment', 'other', 'link', 'target-user', '
'target-machine', 'target-org', 'target-location', 'target-external',
'vulnerability', 'attachment']
mispTypesMapping = {'md5': {'observable': {'0': {'type': 'file', 'hashes': ''}},
'pattern': 'file:hashes.\'{1}\' = \'{0}\''},
'sha1': {'observable': {'0': {'type': 'file', 'hashes': ''}},
'pattern': 'file:hashes.\'{1}\' = \'{0}\''},
'sha256': {'observable': {'0': {'type': 'file', 'hashes': ''}},
'pattern': 'file:hashes.\'{1}\' = \'{0}\''},
'filename': {'observable': {'0': {'type': 'file', 'name': ''}},
'pattern': 'file:name = \'{0}\''},
'filename|md5': {'observable': {'0': {'type': 'file', 'name': '', 'hashes': ''}},
'pattern': 'file:name = \'{0}\' AND file:hashes.\'{3}\' = \'{1}\''},
'filename|sha1': {'observable': {'0': {'type': 'file', 'name': '', 'hashes': ''}},
'pattern': 'file:name = \'{0}\' AND file:hashes.\'{3}\' = \'{1}\''},
'filename|sha256': {'observable': {'0': {'type': 'file', 'name': '', 'hashes': ''}},
'pattern': 'file:name = \'{0}\' AND file:hashes.\'{3}\' = \'{1}\''},
'ip-src': {'observable': {'0': {'type': '', 'value': ''}},
'pattern': '{0}:value = \'{1}\''},
'ip-dst': {'observable': {'0': {'type': '', 'value': ''}},
'pattern': '{0}:value = \'{1}\''},
'hostname': {'observable': {'0': {'type': 'domain-name', 'value': ''}},
'pattern': 'domain-name:value = \'{0}\''},
'domain': {'observable': {'0': {'type': 'domain-name', 'value': ''}},
'pattern': 'domain-name:value = \'{0}\''},
'domain|ip': {'observable': {'0': {'type': 'domain-name', 'value': '', 'resolves_to_refs': '1'}, '1': {'type': '', 'value': ''}},
'pattern': 'domain-name:value = \'{0}\' AND domain-name:resolves_to_refs[*].value = \'{1}\''},
'email-src': {'observable': {'0': {'type': 'email-addr', 'value': ''}},
'pattern': 'email-addr:value = \'{0}\''},
'email-dst': {'observable': {'0': {'type': 'email-addr', 'value': ''}},
'pattern': 'email-addr:value = \'{0}\''},
'email-subject': {'observable': {'0': {'type': 'email-message', 'subject': '', 'is_multipart': 'false'}},
'pattern': 'email-message:subject = \'{0}\''},
# 'email-attachment': {'observable': '', 'pattern': ''},
'email-body': {'observable': {'0': {'type': 'email-message', 'body': '', 'is_multipart': 'false'}},
'pattern': 'email-message:body = \'{0}\''},
'url': {'observable': {'0': {'type': 'url', 'value': ''}},
'pattern': 'url:value = \'{0}\''},
'regkey': {'observable': {'0': {'type': 'windows-registry-key', 'key': ''}},
'pattern': 'windows-registry-key:key = \'{0}\''},
'regkey|value': {'observable': {'0': {'type': 'windows-registry-key', 'key': '', 'values': {'name': ''}}},
'pattern': 'windows-registry-key:key = \'{0}\' AND windows-registry-key:values = \'{1}\''},
'mutex': {'observable': {'0': {'type': 'mutex', 'name': ''}},
'pattern': 'mutex:name = \'{0}\''},
'uri': {'observable': {'0': {'type': 'url', 'value': ''}},
'pattern': 'url:value = \'{0}\''},
'authentihash': {'observable': {'0': {'type': 'file', 'hashes': ''}},
'pattern': 'file:hashes.\'{2}\' = \'{1}\''},
'ssdeep': {'observable': {'0': {'type': 'file', 'hashes': ''}},
'pattern': 'file:hashes.\'{2}\' = \'{1}\''},
'imphash': {'observable': {'0': {'type': 'file', 'hashes': ''}},
'pattern': 'file:hashes.\'{2}\' = \'{1}\''},
'pehash': {'observable': {'0': {'type': 'file', 'hashes': ''}},
'pattern': 'file:hashes.\'{2}\' = \'{1}\''},
'impfuzzy': {'observable': {'0': {'type': 'file', 'hashes': ''}},
'pattern': 'file:hashes.\'{2}\' = \'{1}\''},
'sha224': {'observable': {'0': {'type': 'file', 'hashes': ''}},
'pattern': 'file:hashes.\'{2}\' = \'{1}\''},
'sha384': {'observable': {'0': {'type': 'file', 'hashes': ''}},
'pattern': 'file:hashes.\'{2}\' = \'{1}\''},
'sha512': {'observable': {'0': {'type': 'file', 'hashes': ''}},
'pattern': 'file:hashes.\'{2}\' = \'{1}\''},
'sha512/224': {'observable': {'0': {'type': 'file', 'hashes': ''}},
'pattern': 'file:hashes.\'{2}\' = \'{1}\''},
'sha512/256': {'observable': {'0': {'type': 'file', 'hashes': ''}},
'pattern': 'file:hashes.\'{2}\' = \'{1}\''},
'tlsh': {'observable': {'0': {'type': 'file', 'hashes': ''}},
'pattern': 'file:hashes.\'{2}\' = \'{1}\''},
'filename|authentihash': {'observable': {'0': {'type': 'file', 'name': '', 'hashes': ''}},
'pattern': 'file:name = \'{0}\' AND file:hashes.\'{3}\' = \'{1}\''},
'filename|ssdeep': {'observable': {'0': {'type': 'file', 'name': '', 'hashes': ''}},
'pattern': 'file:name = \'{0}\' AND file:hashes.\'{3}\' = \'{1}\''},
'filename|imphash': {'observable': {'0': {'type': 'file', 'name': '', 'hashes': ''}},
'pattern': 'file:name = \'{0}\' AND file:hashes.\'{3}\' = \'{1}\''},
'filename|impfuzzy': {'observable': {'0': {'type': 'file', 'name': '', 'hashes': ''}},
'pattern': 'file:name = \'{0}\' AND file:hashes.\'{3}\' = \'{1}\''},
'filename|pehash': {'observable': {'0': {'type': 'file', 'name': '', 'hashes': ''}},
'pattern': 'file:name = \'{0}\' AND file:hashes.\'{3}\' = \'{1}\''},
'filename|sha224': {'observable': {'0': {'type': 'file', 'name': '', 'hashes': ''}},
'pattern': 'file:name = \'{0}\' AND file:hashes.\'{3}\' = \'{1}\''},
'filename|sha384': {'observable': {'0': {'type': 'file', 'name': '', 'hashes': ''}},
'pattern': 'file:name = \'{0}\' AND file:hashes.\'{3}\' = \'{1}\''},
'filename|sha512': {'observable': {'0': {'type': 'file', 'name': '', 'hashes': ''}},
'pattern': 'file:name = \'{0}\' AND file:hashes.\'{3}\' = \'{1}\''},
'filename|sha512/224': {'observable': {'0': {'type': 'file', 'name': '', 'hashes': ''}},
'pattern': 'file:name = \'{0}\' AND file:hashes.\'{3}\' = \'{1}\''},
'filename|sha512/256': {'observable': {'0': {'type': 'file', 'name': '', 'hashes': ''}},
'pattern': 'file:name = \'{0}\' AND file:hashes.\'{3}\' = \'{1}\''},
'filename|tlsh': {'observable': {'0': {'type': 'file', 'name': '', 'hashes': ''}},
'pattern': 'file:name = \'{0}\' AND file:hashes.\'{3}\' = \'{1}\''},
'x509-fingerprint-sha1': {'observable': {'0': {'type': 'x509-certificate', 'hashes': {'sha1': ''}}},
'pattern': 'x509-certificate:hashes = \'{0}\''},
'port': {'observable': {'0': {'type': 'network-traffic', 'dst_port': ''}},
'pattern': 'network-traffic:dst_port = \'{0}\''},
'ip-dst|port': {'observable': {'0': {'type': '', 'value': ''}, '1': {'type': 'network-traffic', 'dst_ref': '0', 'dst_port': ''}},
'pattern': 'network-traffic:dst_port = \'{1}\' AND network-traffic:dst_ref.type = \'{2}\' AND network-traffic:dst_ref.value = \'{0}\''},
'ip-src|port': {'observable': {'0': {'type': '', 'value': ''}, '1': {'type': 'network-traffic', 'src_ref': '0', 'dst_port': ''}},
'pattern': 'network-traffic:dst_port = \'{1}\' AND network-traffic:src_ref.type = \'{2}\' AND network-traffic:src_ref.value = \'{0}\''},
'hostname|port': {'observable': {'0': {'type': 'domain-name', 'value': ''}, '1': {'type': 'traffic-network', 'dst_ref': '0', 'dst_port': ''}},
'pattern': 'domain-name:value = \'{0}\' AND network-traffic:dst_port = \'{1}\''},
}
def loadEvent(args, pathname):
try:
filename = args[1]
@ -55,7 +154,7 @@ def readAttributes(event, identity, object_refs, external_refs):
attributes = []
for attribute in event["Attribute"]:
attr_type = attribute['type']
if '|' in attr_type or attr_type == 'malware-sample':
if attr_type not in mispTypesMapping:
continue
if attr_type in non_indicator_attributes:
if attr_type == "link":
@ -105,7 +204,7 @@ def addCampaign(object_refs, attributes, galaxy, identity):
campaign_id = "campaign--{}".format(cluster['uuid'])
name = cluster['value']
description = cluster['description']
labels = 'misp:to_ids=\"{}\"'.format(attribute['to_ids'])
labels = 'misp:to_ids=\"{}\"'.format(galaxy['to_ids'])
campaign_args = {'id': campaign_id, 'type': 'campaign', 'name': name, 'description': description,
'created_by_ref': identity, 'labels': labels}
meta = cluster['meta']
@ -140,7 +239,7 @@ def addIntrusionSet(object_refs, attributes, galaxy, identity):
intrusionSet_id = "intrusion-set--{}".format(cluster['uuid'])
name = cluster['value']
description = cluster['description']
labels = 'misp:to_ids=\"{}\"'.format(attribute['to_ids'])
labels = 'misp:to_ids=\"{}\"'.format(galaxy['to_ids'])
intrusion_args = {'id': intrusionSet_id, 'type': 'intrusion-set', 'name': name, 'description': description,
'created_by_ref': identity, 'labels': labels}
meta = cluster['meta']
@ -159,9 +258,9 @@ def addMalware(object_refs, attributes, galaxy, identity):
def addObservedData(object_refs, attributes, attribute, identity):
observedData_id = "observed-data--{}".format(attribute['uuid'])
timestamp = getDateFromTimestamp(int(attribute['timestamp']))
object0 = defineObservableType(attribute['type'], attribute['value'])
# OBSERVABLE TYPES ARE CRAP
objects = {'0': object0}
attr_type = attribute['type']
attr_val = attribute['value']
objects = defineObservableObject(attr_type, attr_val)
labels = 'misp:to_ids=\"{}\"'.format(attribute['to_ids'])
observedData_args = {'id': observedData_id, 'type': 'observed-data', 'number_observed': 1,
'first_observed': timestamp, 'last_observed': timestamp, 'objects': objects,
@ -210,59 +309,6 @@ def addAliases(meta, argument):
aliases.append(a)
argument['aliases'] = aliases
def defineObservableType(dtype, val):
object0 = {}
# if dtype == '':
# datatype = 'artifact'
# elif dtype == '':
# datatype = 'autonomous-system'
# elif dtype == '':
# datatype = 'directory'
# elif dtype == '':
# datatype = 'domain-name'
# el
if 'email' in dtype and 'name' not in dtype and ('src' in dtype or 'dst' in dtype or 'target' in dtype):
object0['type'] = 'email-addr'
object0['value'] = val
elif 'email' in dtype and ('body' in dtype or 'subject' in dtype or 'header' in dtype or 'reply' in dtype):
object0['type'] = 'email-message'
object0['subject'] = val
object0['is_multipart'] = 'false'
elif 'attachment' in dtype:
object0['type'] = 'file'
object0['name'] = val
# elif dtype == '':
# datatype = 'ipv4-addr'
# elif dtype == '':
# datatype = 'ipv6-addr'
# elif dtype == '':
# datatype = 'mac-addr'
# elif dtype == 'mutex':
# datatype = 'mutex'
# elif dtype == '':
# datatype = 'network-traffic'
# elif dtype == '':
# datatype = 'process'
# elif dtype == '':
# datatype = 'software'
elif dtype == 'url':
object0['type'] = 'url'
object0['value'] = val
# elif dtype == '':
# datatype = 'user-account'
elif 'regkey' in dtype:
object0['type'] = 'windows-registry-key'
object0['key'] = val
elif 'x509' in dtype:
object0['type'] = 'x509-certificate'
elif 'md5' in dtype or 'sha' in dtype:
object0['type'] = 'file'
object0['hashes'] = {dtype: val}
else:
object0['type'] = 'file' # CRAP BEFORE FINDING HOW TO HANDLE ALL THE CASES \o/
object0['name'] = val
return object0
def handleNonIndicatorAttribute(object_refs, attributes, attribute, identity):
attr_type = attribute['type']
if attr_type == "vulnerability":
@ -277,49 +323,84 @@ def handleIndicatorAttribute(object_refs, attributes, attribute, identity):
killchain = [{'kill_chain_name': 'misp-category',
'phase_name': category}]
labels = 'misp:to_ids=\"{}\"'.format(attribute['to_ids'])
attr_type = attribute['type']
attr_val = attribute['value']
args_indicator = {'valid_from': getDateFromTimestamp(int(attribute['timestamp'])), 'type': 'indicator',
'labels': labels, 'pattern': [definePattern(attribute)], 'id': indic_id,
'created_by_ref': identity}
args_indicator['kill_chain_phases'] = killchain
'labels': labels, 'pattern': [definePattern(attr_type, attr_val)], 'id': indic_id,
'created_by_ref': identity, 'kill_chain_phases': killchain}
if attribute['comment']:
args_indicator['description'] = attribute['comment']
indicator = Indicator(**args_indicator)
# indicator = Indicator(valid_from=getDateFromTimestamp(int(attribute["timestamp"])), type='indicator',
# labels=['malicious activity'], pattern="{}".format(definePattern(attribute)),
# id=indic_id, kill_chain_phases=killchain, created_by_ref=identity)
# indicator.id = indic_id
# indicator.labels = ['malicious activity']
# indicator.pattern = "{}".format(definePattern(attribute))
attributes.append(indicator)
object_refs.append(indic_id)
def buildRelationships():
return
def definePattern(attribute):
attr_type = attribute['type']
attr_val = attribute['value']
pattern = ''
if 'md5' in attr_type or 'sha' in attr_type:
pattern += 'file:hashes.\'{}\' = \'{}\''.format(attr_type, attr_val)
elif 'email' in attr_type and 'name' not in attr_type:
if 'src' in attr_type:
pattern += 'email-message:from_refs.value = \'{}\''.format(attr_val)
if 'dst' in attr_type or 'target' in attr_type:
pattern += 'email-message:to_refs.value = \'{}\''.format(attr_val)
elif 'filename' in attr_type:
pattern += 'file:name = \'{}\''.format(attr_val)
elif attr_type == 'url':
pattern += 'url:value = \'{}\''.format(attr_val)
elif 'ip-' in attr_type:
if '.' in attr_val:
pattern += 'ipv4-addr:value = \'{}\''.format(attr_val)
def defineObservableObject(attr_type, attr_val):
observed_object = mispTypesMapping[attr_type]['observable'].copy()
object0 = observed_object['0']
if '|' in attr_type:
_, attr_type2 = attr_type.split('|')
attr_val1, attr_val2 = attr_val.split('|')
object1 = observed_object['1']
if '|ip' in attr_type:
addr_type = defineAddressType(attr_val2)
object0['value'] = attr_val1
object1['type'] = addr_type
object1['value'] = attr_val2
elif 'ip-' in attr_type:
addr_type = defineAddressType(attr_val2)
object0['type'] = addr_type
object0['value'] = attr_val1
object1['dst_port'] = attr_val2
elif 'hostname' in attr_type:
object0['value'] = attr_val1
object1['dst_port'] = attr_val2
elif 'regkey' in attr_type:
object0['key'] = attr_val1
object0['values']['name'] = attr_val2
else:
pattern += 'ipv6-addr:value = \'{}\''.format(attr_val)
elif 'hostname' in attr_type or 'domain' in attr_type:
pattern += 'domain-name:value = \'{}\''.format(attr_val)
else: # Validator will probably be angry with that but waiting for me to use dictionaries, we will use this
pattern += 'mutex:value = \'{}\''.format(attr_val)
object0['name'] = attr_val1
object0['hashes'] = {attr_type2: attr_val2}
else:
if 'x509' in attr_type:
object0['hashes']['sha1'] = attr_val
return observed_object
elif 'ip-' in attr_type:
addr_type = defineAddressType(attr_val)
object0['type'] = addr_type
for obj_attr in object0:
if obj_attr in ('name', 'value', 'body', 'subject', 'dst_port', 'key'):
object0[obj_attr] = attr_val
if 'hashes' in obj_attr:
object0[obj_attr] = {attr_type: attr_val}
return observed_object
def definePattern(attr_type, attr_val):
if '|' in attr_type:
attr_type1, attr_type2 = attr_type.split('|')
attr_val1, attr_val2 = attr_val.split('|')
if 'ip-' in attr_type1 or 'ip' in attr_type2:
addr_type = defineAddressType(attr_val2)
pattern = mispTypesMapping[attr_type]['pattern'].format(attr_val1, attr_val2, addr_type)
else:
pattern = mispTypesMapping[attr_type]['pattern'].format(attr_val1, attr_val2, attr_type1, attr_type2)
else:
if 'ip-' in attr_type:
addr_type = defineAddressType(attr_val)
pattern = mispTypesMapping[attr_type]['pattern'].format(addr_type, attr_val)
else:
pattern = mispTypesMapping[attr_type]['pattern'].format(attr_val, attr_type)
return pattern
def defineAddressType(attr_val):
if ':' in attr_val:
addr_type = 'ipv6-addr'
else:
addr_type = 'ipv4-addr'
return addr_type
def eventReport(event, identity, object_refs, external_refs):
timestamp = getDateFromTimestamp(int(event["publish_timestamp"]))
name = event["info"]