MISP
/
misp-modules
mirror of https://github.com/MISP/misp-modules
2
0
Fork 0
Code Issues Releases Wiki Activity

Merge branch 'chisholm-taxii21_import_contrib' into main

pull/555/head
Alexandre Dulaunoy 2022-02-14 09:36:11 +01:00
parent 27d7e19c15 30287e3b03
commit 3b7b2748e0
No known key found for this signature in database
GPG Key ID: 09E2CD4944E6CBCD
8 changed files with 2922 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.gitmodules vendored Normal file
View File

@ -0,0 +1,4 @@
[submodule "misp_modules/lib/misp-objects"]
path = misp_modules/lib/misp-objects
url = https://github.com/MISP/misp-objects.git
branch = main

2
REQUIREMENTS
View File

@ -128,9 +128,11 @@ socialscan==1.4.2
socketio-client==0.5.7.4
soupsieve==2.2.1; python_version >= '3'
sparqlwrapper==1.8.5
stix2==3.0.1
stix2-patterns==1.3.2
tabulate==0.8.9
tau-clients==0.1.3
taxii2-client==2.3.0
tldextract==3.1.0; python_version >= '3.5'
tornado==6.1; python_version >= '3.5'
tqdm==4.62.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'

1
misp_modules/lib/misp-objects Submodule

@ -0,0 +1 @@
Subproject commit 9dc7e3578f2165e32a3b7cdd09e9e552f2d98d36

2080
misp_modules/lib/stix2misp.py Normal file
View File

File diff suppressed because it is too large Load Diff

460
misp_modules/lib/stix2misp_mapping.py Normal file
View File

@ -0,0 +1,460 @@
################################################################################
# ATTRIBUTES AND OBJECTS MAPPING #
################################################################################
attributes_mapping = {
'filename': '_parse_name',
'ip-src': '_parse_value',
'ip-dst': '_parse_value',
'hostname': '_parse_value',
'domain': '_parse_value',
'domain|ip': '_parse_domain_ip_attribute',
'email-src': '_parse_value',
'email-dst': '_parse_value',
'email-attachment': '_parse_name',
'url': '_parse_value',
'regkey': '_parse_regkey_attribute',
'regkey|value': '_parse_regkey_value',
'malware-sample': '_parse_malware_sample',
'mutex': '_parse_name',
'uri': '_parse_value',
'port': '_parse_port',
'ip-dst|port': '_parse_network_attribute',
'ip-src|port': '_parse_network_attribute',
'hostname|port': '_parse_network_attribute',
'email-reply-to': '_parse_email_reply_to',
'attachment': '_parse_attachment',
'mac-address': '_parse_value',
'AS': '_parse_number'
}
attributes_type_mapping = {
'md5': '_parse_hash',
'sha1': '_parse_hash',
'sha256': '_parse_hash',
'filename|md5': '_parse_filename_hash',
'filename|sha1': '_parse_filename_hash',
'filename|sha256': '_parse_filename_hash',
'email-subject': '_parse_email_message',
'email-body': '_parse_email_message',
'authentihash': '_parse_hash',
'ssdeep': '_parse_hash',
'imphash': '_parse_hash',
'pehash': '_parse_hash',
'impfuzzy': '_parse_hash',
'sha224': '_parse_hash',
'sha384': '_parse_hash',
'sha512': '_parse_hash',
'sha512/224': '_parse_hash',
'sha512/256': '_parse_hash',
'tlsh': '_parse_hash',
'cdhash': '_parse_hash',
'filename|authentihash': '_parse_filename_hash',
'filename|ssdeep': '_parse_filename_hash',
'filename|imphash': '_parse_filename_hash',
'filename|impfuzzy': '_parse_filename_hash',
'filename|pehash': '_parse_filename_hash',
'filename|sha224': '_parse_filename_hash',
'filename|sha384': '_parse_filename_hash',
'filename|sha512': '_parse_filename_hash',
'filename|sha512/224': '_parse_filename_hash',
'filename|sha512/256': '_parse_filename_hash',
'filename|tlsh': '_parse_filename_hash',
'x509-fingerprint-md5': '_parse_x509_attribute',
'x509-fingerprint-sha1': '_parse_x509_attribute',
'x509-fingerprint-sha256': '_parse_x509_attribute'
}
objects_mapping = {
'asn': {
'observable': 'parse_asn_observable',
'pattern': 'parse_asn_pattern'},
'credential': {
'observable': 'parse_credential_observable',
'pattern': 'parse_credential_pattern'},
'domain-ip': {
'observable': 'parse_domain_ip_observable',
'pattern': 'parse_domain_ip_pattern'},
'email': {
'observable': 'parse_email_observable',
'pattern': 'parse_email_pattern'},
'file': {
'observable': 'parse_file_observable',
'pattern': 'parse_file_pattern'},
'ip-port': {
'observable': 'parse_ip_port_observable',
'pattern': 'parse_ip_port_pattern'},
'network-connection': {
'observable': 'parse_network_connection_observable',
'pattern': 'parse_network_connection_pattern'},
'network-socket': {
'observable': 'parse_network_socket_observable',
'pattern': 'parse_network_socket_pattern'},
'process': {
'observable': 'parse_process_observable',
'pattern': 'parse_process_pattern'},
'registry-key': {
'observable': 'parse_regkey_observable',
'pattern': 'parse_regkey_pattern'},
'url': {
'observable': 'parse_url_observable',
'pattern': 'parse_url_pattern'},
'user-account': {
'observable': 'parse_user_account_observable',
'pattern': 'parse_user_account_pattern'},
'WindowsPEBinaryFile': {
'observable': 'parse_pe_observable',
'pattern': 'parse_pe_pattern'},
'x509': {
'observable': 'parse_x509_observable',
'pattern': 'parse_x509_pattern'}
}
observable_mapping = {
('artifact', 'file'): 'parse_file_observable',
('artifact', 'directory', 'file'): 'parse_file_observable',
('artifact', 'email-addr', 'email-message', 'file'): 'parse_email_observable',
('autonomous-system',): 'parse_asn_observable',
('autonomous-system', 'ipv4-addr'): 'parse_asn_observable',
('autonomous-system', 'ipv6-addr'): 'parse_asn_observable',
('autonomous-system', 'ipv4-addr', 'ipv6-addr'): 'parse_asn_observable',
('directory', 'file'): 'parse_file_observable',
('domain-name',): 'parse_domain_ip_observable',
('domain-name', 'ipv4-addr'): 'parse_domain_ip_observable',
('domain-name', 'ipv6-addr'): 'parse_domain_ip_observable',
('domain-name', 'ipv4-addr', 'ipv6-addr'): 'parse_domain_ip_observable',
('domain-name', 'ipv4-addr', 'network-traffic'): 'parse_domain_ip_network_traffic_observable',
('domain-name', 'ipv6-addr', 'network-traffic'): 'parse_domain_ip_network_traffic_observable',
('domain-name', 'ipv4-addr', 'ipv6-addr', 'network-traffic'): 'parse_domain_ip_network_traffic_observable',
('domain-name', 'network-traffic'): 'parse_domain_network_traffic_observable',
('domain-name', 'network-traffic', 'url'): 'parse_url_observable',
('email-addr',): 'parse_email_address_observable',
('email-addr', 'email-message'): 'parse_email_observable',
('email-addr', 'email-message', 'file'): 'parse_email_observable',
('email-message',): 'parse_email_observable',
('file',): 'parse_file_observable',
('file', 'process'): 'parse_process_observable',
('ipv4-addr',): 'parse_ip_address_observable',
('ipv6-addr',): 'parse_ip_address_observable',
('ipv4-addr', 'network-traffic'): 'parse_ip_network_traffic_observable',
('ipv6-addr', 'network-traffic'): 'parse_ip_network_traffic_observable',
('ipv4-addr', 'ipv6-addr', 'network-traffic'): 'parse_ip_network_traffic_observable',
('mac-addr',): 'parse_mac_address_observable',
('mutex',): 'parse_mutex_observable',
('process',): 'parse_process_observable',
('x509-certificate',): 'parse_x509_observable',
('url',): 'parse_url_observable',
('user-account',): 'parse_user_account_observable',
('windows-registry-key',): 'parse_regkey_observable'
}
pattern_mapping = {
('artifact', 'file'): 'parse_file_pattern',
('artifact', 'directory', 'file'): 'parse_file_pattern',
('autonomous-system', ): 'parse_as_pattern',
('autonomous-system', 'ipv4-addr'): 'parse_as_pattern',
('autonomous-system', 'ipv6-addr'): 'parse_as_pattern',
('autonomous-system', 'ipv4-addr', 'ipv6-addr'): 'parse_as_pattern',
('directory',): 'parse_file_pattern',
('directory', 'file'): 'parse_file_pattern',
('domain-name',): 'parse_domain_ip_port_pattern',
('domain-name', 'ipv4-addr'): 'parse_domain_ip_port_pattern',
('domain-name', 'ipv6-addr'): 'parse_domain_ip_port_pattern',
('domain-name', 'ipv4-addr', 'ipv6-addr'): 'parse_domain_ip_port_pattern',
('domain-name', 'ipv4-addr', 'url'): 'parse_url_pattern',
('domain-name', 'ipv6-addr', 'url'): 'parse_url_pattern',
('domain-name', 'ipv4-addr', 'ipv6-addr', 'url'): 'parse_url_pattern',
('domain-name', 'network-traffic'): 'parse_domain_ip_port_pattern',
('domain-name', 'network-traffic', 'url'): 'parse_url_pattern',
('email-addr',): 'parse_email_address_pattern',
('email-message',): 'parse_email_message_pattern',
('file',): 'parse_file_pattern',
('ipv4-addr',): 'parse_ip_address_pattern',
('ipv6-addr',): 'parse_ip_address_pattern',
('ipv4-addr', 'ipv6-addr'): 'parse_ip_address_pattern',
('mac-addr',): 'parse_mac_address_pattern',
('mutex',): 'parse_mutex_pattern',
('network-traffic',): 'parse_network_traffic_pattern',
('process',): 'parse_process_pattern',
('url',): 'parse_url_pattern',
('user-account',): 'parse_user_account_pattern',
('windows-registry-key',): 'parse_regkey_pattern',
('x509-certificate',): 'parse_x509_pattern'
}
pattern_forbidden_relations = (' LIKE ', ' FOLLOWEDBY ', ' MATCHES ', ' ISSUBSET ', ' ISSUPERSET ', ' REPEATS ')
single_attribute_fields = ('type', 'value', 'to_ids')
################################################################################
# OBSERVABLE OBJECTS AND PATTERNS MAPPING. #
################################################################################
address_family_attribute_mapping = {'type': 'text','object_relation': 'address-family'}
as_number_attribute_mapping = {'type': 'AS', 'object_relation': 'asn'}
description_attribute_mapping = {'type': 'text', 'object_relation': 'description'}
asn_subnet_attribute_mapping = {'type': 'ip-src', 'object_relation': 'subnet-announced'}
cc_attribute_mapping = {'type': 'email-dst', 'object_relation': 'cc'}
credential_attribute_mapping = {'type': 'text', 'object_relation': 'password'}
data_attribute_mapping = {'type': 'text', 'object_relation': 'data'}
data_type_attribute_mapping = {'type': 'text', 'object_relation': 'data-type'}
domain_attribute_mapping = {'type': 'domain', 'object_relation': 'domain'}
domain_family_attribute_mapping = {'type': 'text', 'object_relation': 'domain-family'}
dst_port_attribute_mapping = {'type': 'port', 'object_relation': 'dst-port'}
email_attachment_attribute_mapping = {'type': 'email-attachment', 'object_relation': 'attachment'}
email_date_attribute_mapping = {'type': 'datetime', 'object_relation': 'send-date'}
email_subject_attribute_mapping = {'type': 'email-subject', 'object_relation': 'subject'}
encoding_attribute_mapping = {'type': 'text', 'object_relation': 'file-encoding'}
end_datetime_attribute_mapping = {'type': 'datetime', 'object_relation': 'last-seen'}
entropy_mapping = {'type': 'float', 'object_relation': 'entropy'}
filename_attribute_mapping = {'type': 'filename', 'object_relation': 'filename'}
from_attribute_mapping = {'type': 'email-src', 'object_relation': 'from'}
imphash_mapping = {'type': 'imphash', 'object_relation': 'imphash'}
id_attribute_mapping = {'type': 'text', 'object_relation': 'id'}
ip_attribute_mapping = {'type': 'ip-dst', 'object_relation': 'ip'}
issuer_attribute_mapping = {'type': 'text', 'object_relation': 'issuer'}
key_attribute_mapping = {'type': 'regkey', 'object_relation': 'key'}
malware_sample_attribute_mapping = {'type': 'malware-sample', 'object_relation': 'malware-sample'}
mime_type_attribute_mapping = {'type': 'mime-type', 'object_relation': 'mimetype'}
modified_attribute_mapping = {'type': 'datetime', 'object_relation': 'last-modified'}
name_attribute_mapping = {'type': 'text', 'object_relation': 'name'}
network_traffic_ip = {'type': 'ip-{}', 'object_relation': 'ip-{}'}
number_sections_mapping = {'type': 'counter', 'object_relation': 'number-sections'}
password_mapping = {'type': 'text', 'object_relation': 'password'}
path_attribute_mapping = {'type': 'text', 'object_relation': 'path'}
pe_type_mapping = {'type': 'text', 'object_relation': 'type'}
pid_attribute_mapping = {'type': 'text', 'object_relation': 'pid'}
process_command_line_mapping = {'type': 'text', 'object_relation': 'command-line'}
process_creation_time_mapping = {'type': 'datetime', 'object_relation': 'creation-time'}
process_image_mapping = {'type': 'filename', 'object_relation': 'image'}
process_name_mapping = {'type': 'text', 'object_relation': 'name'}
regkey_name_attribute_mapping = {'type': 'text', 'object_relation': 'name'}
references_attribute_mapping = {'type': 'link', 'object_relation': 'references'}
reply_to_attribute_mapping = {'type': 'email-reply-to', 'object_relation': 'reply-to'}
screenshot_attribute_mapping = {'type': 'attachment', 'object_relation': 'screenshot'}
section_name_mapping = {'type': 'text', 'object_relation': 'name'}
serial_number_attribute_mapping = {'type': 'text', 'object_relation': 'serial-number'}
size_attribute_mapping = {'type': 'size-in-bytes', 'object_relation': 'size-in-bytes'}
src_port_attribute_mapping = {'type': 'port', 'object_relation': 'src-port'}
start_datetime_attribute_mapping = {'type': 'datetime', 'object_relation': 'first-seen'}
state_attribute_mapping = {'type': 'text', 'object_relation': 'state'}
summary_attribute_mapping = {'type': 'text', 'object_relation': 'summary'}
to_attribute_mapping = {'type': 'email-dst', 'object_relation': 'to'}
url_attribute_mapping = {'type': 'url', 'object_relation': 'url'}
url_port_attribute_mapping = {'type': 'port', 'object_relation': 'port'}
user_id_mapping = {'type': 'text', 'object_relation': 'username'}
x_mailer_attribute_mapping = {'type': 'email-x-mailer', 'object_relation': 'x-mailer'}
x509_md5_attribute_mapping = {'type': 'x509-fingerprint-md5', 'object_relation': 'x509-fingerprint-md5'}
x509_sha1_attribute_mapping = {'type': 'x509-fingerprint-sha1', 'object_relation': 'x509-fingerprint-sha1'}
x509_sha256_attribute_mapping = {'type': 'x509-fingerprint-sha256', 'object_relation': 'x509-fingerprint-sha256'}
x509_spka_attribute_mapping = {'type': 'text', 'object_relation': 'pubkey-info-algorithm'} # x509 subject public key algorithm
x509_spke_attribute_mapping = {'type': 'text', 'object_relation': 'pubkey-info-exponent'} # x509 subject public key exponent
x509_spkm_attribute_mapping = {'type': 'text', 'object_relation': 'pubkey-info-modulus'} # x509 subject public key modulus
x509_subject_attribute_mapping = {'type': 'text', 'object_relation': 'subject'}
x509_version_attribute_mapping = {'type': 'text', 'object_relation': 'version'}
x509_vna_attribute_mapping = {'type': 'datetime', 'object_relation': 'validity-not-after'} # x509 validity not after
x509_vnb_attribute_mapping = {'type': 'datetime', 'object_relation': 'validity-not-before'} # x509 validity not before
asn_mapping = {'number': as_number_attribute_mapping,
'autonomous-system:number': as_number_attribute_mapping,
'name': description_attribute_mapping,
'autonomous-system:name': description_attribute_mapping,
'ipv4-addr': asn_subnet_attribute_mapping,
'ipv6-addr': asn_subnet_attribute_mapping,
'ipv4-addr:value': asn_subnet_attribute_mapping,
'ipv6-addr:value': asn_subnet_attribute_mapping}
attack_pattern_mapping = {'name': name_attribute_mapping,
'description': summary_attribute_mapping}
attack_pattern_references_mapping = {'mitre-attack': references_attribute_mapping,
'capec': id_attribute_mapping}
course_of_action_mapping = {'description': description_attribute_mapping,
'name': name_attribute_mapping}
credential_mapping = {'credential': credential_attribute_mapping,
'user-account:credential': credential_attribute_mapping,
'user_id': user_id_mapping,
'user-account:user_id': user_id_mapping}
domain_ip_mapping = {'domain-name': domain_attribute_mapping,
'domain-name:value': domain_attribute_mapping,
'ipv4-addr': ip_attribute_mapping,
'ipv6-addr': ip_attribute_mapping,
'ipv4-addr:value': ip_attribute_mapping,
'ipv6-addr:value': ip_attribute_mapping,
'domain-name:resolves_to_refs[*].value': ip_attribute_mapping,
'network-traffic:dst_port': dst_port_attribute_mapping,
'network-traffic:src_port': src_port_attribute_mapping}
email_mapping = {'date': email_date_attribute_mapping,
'email-message:date': email_date_attribute_mapping,
'email-message:to_refs[*].value': to_attribute_mapping,
'email-message:cc_refs[*].value': cc_attribute_mapping,
'subject': email_subject_attribute_mapping,
'email-message:subject': email_subject_attribute_mapping,
'X-Mailer': x_mailer_attribute_mapping,
'email-message:additional_header_fields.x_mailer': x_mailer_attribute_mapping,
'Reply-To': reply_to_attribute_mapping,
'email-message:additional_header_fields.reply_to': reply_to_attribute_mapping,
'email-message:from_ref.value': from_attribute_mapping,
'email-addr:value': to_attribute_mapping}
email_references_mapping = {'attachment': email_attachment_attribute_mapping,
'cc_refs': cc_attribute_mapping,
'from_ref': from_attribute_mapping,
'screenshot': screenshot_attribute_mapping,
'to_refs': to_attribute_mapping}
file_mapping = {'artifact:mime_type': mime_type_attribute_mapping,
'file:content_ref.mime_type': mime_type_attribute_mapping,
'mime_type': mime_type_attribute_mapping,
'file:mime_type': mime_type_attribute_mapping,
'name': filename_attribute_mapping,
'file:name': filename_attribute_mapping,
'name_enc': encoding_attribute_mapping,
'file:name_enc': encoding_attribute_mapping,
'file:parent_directory_ref.path': path_attribute_mapping,
'directory:path': path_attribute_mapping,
'size': size_attribute_mapping,
'file:size': size_attribute_mapping}
network_traffic_mapping = {'dst_port':dst_port_attribute_mapping,
'src_port': src_port_attribute_mapping,
'network-traffic:dst_port': dst_port_attribute_mapping,
'network-traffic:src_port': src_port_attribute_mapping}
ip_port_mapping = {'value': domain_attribute_mapping,
'domain-name:value': domain_attribute_mapping,
'network-traffic:dst_ref.value': {'type': 'ip-dst', 'object_relation': 'ip-dst'},
'network-traffic:src_ref.value': {'type': 'ip-src', 'object_relation': 'ip-src'}}
ip_port_mapping.update(network_traffic_mapping)
ip_port_references_mapping = {'domain-name': domain_attribute_mapping,
'ipv4-addr': network_traffic_ip,
'ipv6-addr': network_traffic_ip}
network_socket_extension_mapping = {'address_family': address_family_attribute_mapping,
"network-traffic:extensions.'socket-ext'.address_family": address_family_attribute_mapping,
'protocol_family': domain_family_attribute_mapping,
"network-traffic:extensions.'socket-ext'.protocol_family": domain_family_attribute_mapping,
'is_blocking': state_attribute_mapping,
"network-traffic:extensions.'socket-ext'.is_blocking": state_attribute_mapping,
'is_listening': state_attribute_mapping,
"network-traffic:extensions.'socket-ext'.is_listening": state_attribute_mapping}
network_traffic_references_mapping = {'domain-name': {'type': 'hostname', 'object_relation': 'hostname-{}'},
'ipv4-addr': network_traffic_ip,
'ipv6-addr': network_traffic_ip}
pe_mapping = {'pe_type': pe_type_mapping, 'number_of_sections': number_sections_mapping, 'imphash': imphash_mapping}
pe_section_mapping = {'name': section_name_mapping, 'size': size_attribute_mapping, 'entropy': entropy_mapping}
hash_types = ('MD5', 'SHA-1', 'SHA-256', 'SHA-224', 'SHA-384', 'SHA-512', 'ssdeep', 'tlsh')
for hash_type in hash_types:
misp_hash_type = hash_type.replace('-', '').lower()
attribute = {'type': misp_hash_type, 'object_relation': misp_hash_type}
file_mapping[hash_type] = attribute
file_mapping.update({f"file:hashes.'{feature}'": attribute for feature in (hash_type, misp_hash_type)})
file_mapping.update({f"file:hashes.{feature}": attribute for feature in (hash_type, misp_hash_type)})
pe_section_mapping[hash_type] = attribute
pe_section_mapping[misp_hash_type] = attribute
process_mapping = {'name': process_name_mapping,
'process:name': process_name_mapping,
'pid': pid_attribute_mapping,
'process:pid': pid_attribute_mapping,
'created': process_creation_time_mapping,
'process:created': process_creation_time_mapping,
'command_line': process_command_line_mapping,
'process:command_line': process_command_line_mapping,
'process:parent_ref.pid': {'type': 'text', 'object_relation': 'parent-pid'},
'process:child_refs[*].pid': {'type': 'text', 'object_relation': 'child-pid'},
'process:binary_ref.name': process_image_mapping}
child_process_reference_mapping = {'pid': {'type': 'text', 'object_relation': 'child-pid'}}
parent_process_reference_mapping = {'command_line': {'type': 'text', 'object_relation': 'parent-command-line'},
'pid': {'type': 'text', 'object_relation': 'parent-pid'},
'process-name': {'type': 'text', 'object_relation': 'parent-process-name'}}
regkey_mapping = {'data': data_attribute_mapping,
'windows-registry-key:values.data': data_attribute_mapping,
'data_type': data_type_attribute_mapping,
'windows-registry-key:values.data_type': data_type_attribute_mapping,
'modified': modified_attribute_mapping,
'windows-registry-key:modified': modified_attribute_mapping,
'name': regkey_name_attribute_mapping,
'windows-registry-key:values.name': regkey_name_attribute_mapping,
'key': key_attribute_mapping,
'windows-registry-key:key': key_attribute_mapping,
'windows-registry-key:value': {'type': 'text', 'object_relation': 'hive'}
}
url_mapping = {'url': url_attribute_mapping,
'url:value': url_attribute_mapping,
'domain-name': domain_attribute_mapping,
'domain-name:value': domain_attribute_mapping,
'network-traffic': url_port_attribute_mapping,
'network-traffic:dst_port': url_port_attribute_mapping,
'ipv4-addr:value': ip_attribute_mapping,
'ipv6-addr:value': ip_attribute_mapping
}
user_account_mapping = {'account_created': {'type': 'datetime', 'object_relation': 'created'},
'account_expires': {'type': 'datetime', 'object_relation': 'expires'},
'account_first_login': {'type': 'datetime', 'object_relation': 'first_login'},
'account_last_login': {'type': 'datetime', 'object_relation': 'last_login'},
'account_login': user_id_mapping,
'account_type': {'type': 'text', 'object_relation': 'account-type'},
'can_escalate_privs': {'type': 'boolean', 'object_relation': 'can_escalate_privs'},
'credential': credential_attribute_mapping,
'credential_last_changed': {'type': 'datetime', 'object_relation': 'password_last_changed'},
'display_name': {'type': 'text', 'object_relation': 'display-name'},
'gid': {'type': 'text', 'object_relation': 'group-id'},
'home_dir': {'type': 'text', 'object_relation': 'home_dir'},
'is_disabled': {'type': 'boolean', 'object_relation': 'disabled'},
'is_privileged': {'type': 'boolean', 'object_relation': 'privileged'},
'is_service_account': {'type': 'boolean', 'object_relation': 'is_service_account'},
'shell': {'type': 'text', 'object_relation': 'shell'},
'user_id': {'type': 'text', 'object_relation': 'user-id'}}
vulnerability_mapping = {'name': id_attribute_mapping,
'description': summary_attribute_mapping}
x509_mapping = {'issuer': issuer_attribute_mapping,
'x509-certificate:issuer': issuer_attribute_mapping,
'serial_number': serial_number_attribute_mapping,
'x509-certificate:serial_number': serial_number_attribute_mapping,
'subject': x509_subject_attribute_mapping,
'x509-certificate:subject': x509_subject_attribute_mapping,
'subject_public_key_algorithm': x509_spka_attribute_mapping,
'x509-certificate:subject_public_key_algorithm': x509_spka_attribute_mapping,
'subject_public_key_exponent': x509_spke_attribute_mapping,
'x509-certificate:subject_public_key_exponent': x509_spke_attribute_mapping,
'subject_public_key_modulus': x509_spkm_attribute_mapping,
'x509-certificate:subject_public_key_modulus': x509_spkm_attribute_mapping,
'validity_not_before': x509_vnb_attribute_mapping,
'x509-certificate:validity_not_before': x509_vnb_attribute_mapping,
'validity_not_after': x509_vna_attribute_mapping,
'x509-certificate:validity_not_after': x509_vna_attribute_mapping,
'version': x509_version_attribute_mapping,
'x509-certificate:version': x509_version_attribute_mapping,
'SHA-1': x509_sha1_attribute_mapping,
"x509-certificate:hashes.'sha1'": x509_sha1_attribute_mapping,
'SHA-256': x509_sha256_attribute_mapping,
"x509-certificate:hashes.'sha256'": x509_sha256_attribute_mapping,
'MD5': x509_md5_attribute_mapping,
"x509-certificate:hashes.'md5'": x509_md5_attribute_mapping,
}
attachment_types = ('file:content_ref.name', 'file:content_ref.payload_bin',
'artifact:x_misp_text_name', 'artifact:payload_bin',
"file:hashes.'MD5'", "file:content_ref.hashes.'MD5'",
'file:name')
connection_protocols = {"IP": "3", "ICMP": "3", "ARP": "3",
"TCP": "4", "UDP": "4",
"HTTP": "7", "HTTPS": "7", "FTP": "7"}

1
misp_modules/lib/synonymsToTagNames.json Normal file
View File

File diff suppressed because one or more lines are too long

1
misp_modules/modules/import_mod/__init__.py
View File

@ -15,4 +15,5 @@ __all__ = [
'csvimport',
'cof2misp',
'joe_import',
'taxii21'
]

373
misp_modules/modules/import_mod/taxii21.py Normal file
View File

@ -0,0 +1,373 @@
"""
Import content from a TAXII 2.1 server.
"""
import collections
import itertools
import json
import misp_modules.lib.stix2misp
from pathlib import Path
import re
import stix2.v20
import taxii2client
import taxii2client.exceptions
import requests
class ConfigError(Exception):
"""
Represents an error in the config settings for one invocation of this
module.
"""
pass
misperrors = {'error': 'Error'}
moduleinfo = {'version': '0.1', 'author': 'Abc',
'description': 'Import content from a TAXII 2.1 server',
'module-type': ['import']}
mispattributes = {
'inputSource': [],
'output': ['MISP objects'],
'format': 'misp_standard',
}
userConfig = {
"url": {
"type": "String",
"message": "A TAXII 2.1 collection URL",
},
"added_after": {
"type": "String",
"message": "Lower bound on time the object was uploaded to the TAXII server"
},
"stix_id": {
"type": "String",
"message": "STIX ID(s) of objects"
},
"spec_version": { # TAXII 2.1 specific
"type": "String",
"message": "STIX version(s) of objects"
},
"type": {
"type": "String",
"message": "STIX type(s) of objects"
},
"version": {
"type": "String",
"message": 'Version timestamp(s), or "first"/"last"/"all"'
},
# Should we give some user control over this? It will not be allowed to
# exceed the admin setting.
"STIX object limit": {
"type": "Integer",
"message": "Maximum number of STIX objects to process"
},
"username": {
"type": "String",
"message": "Username for TAXII server authentication, if necessary"
},
"password": {
"type": "String",
"message": "Password for TAXII server authentication, if necessary"
}
}
# Paging will be handled transparently by this module, so user-defined
# paging-related filtering parameters will not be supported.
# This module will not process more than this number of STIX objects in total
# from a TAXII server in one module invocation (across all pages), to limit
# resource consumption.
moduleconfig = [
"stix_object_limit"
]
# In case there is neither an admin nor user setting given.
_DEFAULT_STIX_OBJECT_LIMIT = 1000
# Page size to use when paging TAXII results. Trades off the amount of
# hammering on TAXII servers and overhead of repeated requests, with the
# resource consumption of a single page. (Should be an admin setting too?)
_PAGE_SIZE = 100
_synonymsToTagNames_path = Path(__file__).parent / "../../lib/synonymsToTagNames.json"
# Collects module config information necessary to perform the TAXII query.
Config = collections.namedtuple("Config", [
"url",
"added_after",
"id",
"spec_version",
"type",
"version",
"stix_object_limit",
"username",
"password"
])
def _pymisp_to_json_serializable(obj):
"""
Work around a possible bug with PyMISP's
AbstractMisp.to_dict(json_format=True) method, which doesn't always produce
a JSON-serializable value (i.e. a value which is serializable with the
default JSON encoder).
:param obj: A PyMISP object
:return: A JSON-serializable version of the object
"""
# The workaround creates a JSON string and then parses it back to a
# JSON-serializable value.
json_ = obj.to_json()
json_serializable = json.loads(json_)
return json_serializable
def _normalize_multi_values(value):
"""
Some TAXII filters may contain multiple values separated by commas,
without spaces around the commas. Maybe give MISP users a little more
flexibility? This function normalizes a possible multi-valued value
(e.g. multiple values delimited by commas or spaces, all in the same
string) to TAXII-required format.
:param value: A MISP config value
:return: A normalized value
"""
if "," in value:
value = re.sub(r"\s*,\s*", ",", value)
else:
# Assume space delimiting; replace spaces with commas.
# I don't think we need to worry about spaces embedded in values.
value = re.sub(r"\s+", ",", value)
value = value.strip(",")
return value
def _get_config(config):
"""
Combine user, admin, and default config settings to produce a config
object with all settings together.
:param config: The misp-modules request's "config" value.
:return: A Config object
:raises ConfigError: if any config errors are detected
"""
# Strip whitespace from all config settings... except for password?
for key, val in config.items():
if isinstance(val, str) and key != "password":
config[key] = val.strip()
url = config.get("url")
added_after = config.get("added_after")
id_ = config.get("stix_id")
spec_version = config.get("spec_version")
type_ = config.get("type")
version_ = config.get("version")
username = config.get("username")
password = config.get("password")
admin_stix_object_limit = config.get("stix_object_limit")
user_stix_object_limit = config.get("STIX object limit")
if admin_stix_object_limit:
admin_stix_object_limit = int(admin_stix_object_limit)
else:
admin_stix_object_limit = _DEFAULT_STIX_OBJECT_LIMIT
if user_stix_object_limit:
user_stix_object_limit = int(user_stix_object_limit)
stix_object_limit = min(user_stix_object_limit, admin_stix_object_limit)
else:
stix_object_limit = admin_stix_object_limit
# How much of this should we sanity-check here before passing it off to the
# TAXII client (and thence, to the TAXII server)?
if not url:
raise ConfigError("A TAXII 2.1 collection URL is required.")
if admin_stix_object_limit < 1:
raise ConfigError(
"Invalid admin object limit: must be positive: "
+ str(admin_stix_object_limit)
)
if stix_object_limit < 1:
raise ConfigError(
"Invalid object limit: must be positive: "
+ str(stix_object_limit)
)
if id_:
id_ = _normalize_multi_values(id_)
if spec_version:
spec_version = _normalize_multi_values(spec_version)
if type_:
type_ = _normalize_multi_values(type_)
if version_:
version_ = _normalize_multi_values(version_)
# STIX->MISP converter currently only supports STIX 2.0, so let's force
# spec_version="2.0".
if not spec_version:
spec_version = "2.0"
elif spec_version != "2.0":
raise ConfigError('Only spec_version="2.0" is supported for now.')
if (username and not password) or (not username and password):
raise ConfigError(
'Both or neither of "username" and "password" are required.'
)
config_obj = Config(
url, added_after, id_, spec_version, type_, version_, stix_object_limit,
username, password
)
return config_obj
def _query_taxii(config):
"""
Query the TAXII server according to the given config, convert the STIX
results to MISP, and return a standard misp-modules response.
:param config: Module config information as a Config object
:return: A dict containing a misp-modules response
"""
collection = taxii2client.Collection(
config.url, user=config.username, password=config.password
)
# No point in asking for more than our overall limit.
page_size = min(_PAGE_SIZE, config.stix_object_limit)
kwargs = {
"per_request": page_size
}
if config.spec_version:
kwargs["spec_version"] = config.spec_version
if config.version:
kwargs["version"] = config.version
if config.id:
kwargs["id"] = config.id
if config.type:
kwargs["type"] = config.type
if config.added_after:
kwargs["added_after"] = config.added_after
pages = taxii2client.as_pages(
collection.get_objects,
**kwargs
)
# Chain all the objects from all pages together...
all_stix_objects = itertools.chain.from_iterable(
taxii_envelope.get("objects", [])
for taxii_envelope in pages
)
# And only take the first N objects from that.
limited_stix_objects = itertools.islice(
all_stix_objects, 0, config.stix_object_limit
)
# Collect into a list. This is... unfortunate, but I don't think the
# converter will work incrementally (will it?). It expects all objects to
# be given at once.
#
# It may also be desirable to have all objects available at once so that
# cross-references can be made where possible, but it results in increased
# memory usage.
stix_objects = list(limited_stix_objects)
# The STIX 2.0 converter wants a 2.0 bundle. (Hope the TAXII server isn't
# returning 2.1 objects!)
bundle20 = stix2.v20.Bundle(stix_objects, allow_custom=True)
converter = misp_modules.lib.stix2misp.ExternalStixParser()
converter.handler(
bundle20, None, [0, "event", str(_synonymsToTagNames_path)]
)
attributes = [
_pymisp_to_json_serializable(attr)
for attr in converter.misp_event.attributes
]
objects = [
_pymisp_to_json_serializable(obj)
for obj in converter.misp_event.objects
]
tags = [
_pymisp_to_json_serializable(tag)
for tag in converter.misp_event.tags
]
result = {
"results": {
"Attribute": attributes,
"Object": objects,
"Tag": tags
}
}
return result
def handler(q=False):
if q is False:
return False
request = json.loads(q)
result = None
config = None
try:
config = _get_config(request["config"])
except ConfigError as e:
result = misperrors
result["error"] = e.args[0]
if not result:
try:
result = _query_taxii(config)
except taxii2client.exceptions.TAXIIServiceException as e:
result = misperrors
result["error"] = str(e)
except requests.HTTPError as e:
# Let's give a better error message for auth issues.
if e.response.status_code in (401, 403):
result = misperrors
result["error"] = "Access was denied."
else:
raise
return result
def introspection():
mispattributes["userConfig"] = userConfig
return mispattributes
def version():
moduleinfo['config'] = moduleconfig
return moduleinfo