mirror of https://github.com/MISP/misp-modules
Merge github.com:MISP/misp-modules
chrisr3d
commit
dcab9aa150
|
|
@ -18,8 +18,9 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/
|
||||||
### Expansion modules
|
### Expansion modules
|
||||||
|
|
||||||
* [ASN History](misp_modules/modules/expansion/asn_history.py) - a hover and expansion module to expand an AS number with the ASN description and its history.
|
* [ASN History](misp_modules/modules/expansion/asn_history.py) - a hover and expansion module to expand an AS number with the ASN description and its history.
|
||||||
* [CIRCL Passive SSL](misp_modules/modules/expansion/circl_passivessl.py) - a hover and expansion module to expand IP addresses with the X.509 certificate seen.
|
|
||||||
* [CIRCL Passive DNS](misp_modules/modules/expansion/circl_passivedns.py) - a hover and expansion module to expand hostname and IP addresses with passive DNS information.
|
* [CIRCL Passive DNS](misp_modules/modules/expansion/circl_passivedns.py) - a hover and expansion module to expand hostname and IP addresses with passive DNS information.
|
||||||
|
* [CIRCL Passive SSL](misp_modules/modules/expansion/circl_passivessl.py) - a hover and expansion module to expand IP addresses with the X.509 certificate seen.
|
||||||
|
* [countrycode](misp_modules/modules/expansion/countrycode.py) - a hover module to tell you what country a URL belongs to.
|
||||||
* [CVE](misp_modules/modules/expansion/cve.py) - a hover module to give more information about a vulnerability (CVE).
|
* [CVE](misp_modules/modules/expansion/cve.py) - a hover module to give more information about a vulnerability (CVE).
|
||||||
* [DNS](misp_modules/modules/expansion/dns.py) - a simple module to resolve MISP attributes like hostname and domain to expand IP addresses attributes.
|
* [DNS](misp_modules/modules/expansion/dns.py) - a simple module to resolve MISP attributes like hostname and domain to expand IP addresses attributes.
|
||||||
* [DomainTools](misp_modules/modules/expansion/domaintools.py) - a hover and expansion module to get information from [DomainTools](http://www.domaintools.com/) whois.
|
* [DomainTools](misp_modules/modules/expansion/domaintools.py) - a hover and expansion module to get information from [DomainTools](http://www.domaintools.com/) whois.
|
||||||
|
|
@ -28,13 +29,12 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/
|
||||||
* [GeoIP](misp_modules/modules/expansion/geoip_country.py) - a hover and expansion module to get GeoIP information from geolite/maxmind.
|
* [GeoIP](misp_modules/modules/expansion/geoip_country.py) - a hover and expansion module to get GeoIP information from geolite/maxmind.
|
||||||
* [IPASN](misp_modules/modules/expansion/ipasn.py) - a hover and expansion to get the BGP ASN of an IP address.
|
* [IPASN](misp_modules/modules/expansion/ipasn.py) - a hover and expansion to get the BGP ASN of an IP address.
|
||||||
* [iprep](misp-modules/modules/expansion/iprep.py) - an expansion module to get IP reputation from packetmail.net.
|
* [iprep](misp-modules/modules/expansion/iprep.py) - an expansion module to get IP reputation from packetmail.net.
|
||||||
|
* [OTX](misp_modules/modules/expansion/otx.py) - an expansion module for [OTX](https://otx.alienvault.com/).
|
||||||
* [passivetotal](misp_modules/modules/expansion/passivetotal.py) - a [passivetotal](https://www.passivetotal.org/) module that queries a number of different PassiveTotal datasets.
|
* [passivetotal](misp_modules/modules/expansion/passivetotal.py) - a [passivetotal](https://www.passivetotal.org/) module that queries a number of different PassiveTotal datasets.
|
||||||
* [shodan](misp_modules/modules/expansion/shodan.py) - a minimal [shodan](https://www.shodan.io/) expansion module.
|
* [shodan](misp_modules/modules/expansion/shodan.py) - a minimal [shodan](https://www.shodan.io/) expansion module.
|
||||||
* [sourcecache](misp_modules/modules/expansion/sourcecache.py) - a module to cache a specific link from a MISP instance.
|
* [sourcecache](misp_modules/modules/expansion/sourcecache.py) - a module to cache a specific link from a MISP instance.
|
||||||
* [ThreatCrowd](misp_modules/modules/expansion/threatcrowd.py) - an expansion module for [ThreatCrowd](https://www.threatcrowd.org/).
|
* [ThreatCrowd](misp_modules/modules/expansion/threatcrowd.py) - an expansion module for [ThreatCrowd](https://www.threatcrowd.org/).
|
||||||
* [OTX](misp_modules/modules/expansion/otx.py) - an expansion module for [OTX](https://otx.alienvault.com/).
|
|
||||||
* [threatminer](misp_modules/modules/expansion/threatminer.py) - an expansion module to expand from [ThreatMiner](https://www.threatminer.org/).
|
* [threatminer](misp_modules/modules/expansion/threatminer.py) - an expansion module to expand from [ThreatMiner](https://www.threatminer.org/).
|
||||||
* [countrycode](misp_modules/modules/expansion/countrycode.py) - a hover module to tell you what country a URL belongs to.
|
|
||||||
* [virustotal](misp_modules/modules/expansion/virustotal.py) - an expansion module to pull known resolutions and malware samples related with an IP/Domain from virusTotal (this modules require a VirusTotal private API key)
|
* [virustotal](misp_modules/modules/expansion/virustotal.py) - an expansion module to pull known resolutions and malware samples related with an IP/Domain from virusTotal (this modules require a VirusTotal private API key)
|
||||||
* [wikidata](misp_modules/modules/expansion/wiki.py) - a [wikidata](https://www.wikidata.org) expansion module.
|
* [wikidata](misp_modules/modules/expansion/wiki.py) - a [wikidata](https://www.wikidata.org) expansion module.
|
||||||
* [xforce](misp_modules/modules/expansion/xforceexchange.py) - an IBM X-Force Exchange expansion module.
|
* [xforce](misp_modules/modules/expansion/xforceexchange.py) - an IBM X-Force Exchange expansion module.
|
||||||
|
|
@ -50,10 +50,11 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/
|
||||||
### Import modules
|
### Import modules
|
||||||
|
|
||||||
* [Cuckoo JSON](misp_modules/modules/import_mod/cuckooimport.py) Cuckoo JSON import.
|
* [Cuckoo JSON](misp_modules/modules/import_mod/cuckooimport.py) Cuckoo JSON import.
|
||||||
|
* [Email Import](misp_modules/modules/import_mod/email_import.py) Email import module for MISP to import basic metadata.
|
||||||
* [OCR](misp_modules/modules/import_mod/ocr.py) Optical Character Recognition (OCR) module for MISP to import attributes from images, scan or faxes.
|
* [OCR](misp_modules/modules/import_mod/ocr.py) Optical Character Recognition (OCR) module for MISP to import attributes from images, scan or faxes.
|
||||||
* [OpenIOC](misp_modules/modules/import_mod/openiocimport.py) OpenIOC import based on PyMISP library.
|
* [OpenIOC](misp_modules/modules/import_mod/openiocimport.py) OpenIOC import based on PyMISP library.
|
||||||
* [stiximport](misp_modules/modules/import_mod/stiximport.py) - An import module to process STIX xml/json.
|
* [stiximport](misp_modules/modules/import_mod/stiximport.py) - An import module to process STIX xml/json.
|
||||||
* [Email Import](misp_modules/modules/import_mod/email_import.py) Email import module for MISP to import basic metadata.
|
* [ThreatAnalyzer](misp_modules/modules/import_mod/threatanalyzer_import.py) - An import module to process ThreatAnalyzer archive.zip/analysis.json sandbox exports.
|
||||||
* [VMRay](misp_modules/modules/import_mod/vmray_import.py) - An import module to process VMRay export.
|
* [VMRay](misp_modules/modules/import_mod/vmray_import.py) - An import module to process VMRay export.
|
||||||
|
|
||||||
## How to install and start MISP modules?
|
## How to install and start MISP modules?
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
from . import _vmray
|
from . import _vmray
|
||||||
|
|
||||||
__all__ = ['vmray_import', 'testimport', 'ocr', 'stiximport', 'cuckooimport',
|
__all__ = ['vmray_import', 'testimport', 'ocr', 'stiximport', 'cuckooimport',
|
||||||
'email_import', 'mispjson', 'openiocimport']
|
'email_import', 'mispjson', 'openiocimport', 'threatanalyzer_import']
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,507 @@
|
||||||
|
'''
|
||||||
|
import
|
||||||
|
define mandatory
|
||||||
|
|
||||||
|
'''
|
||||||
|
import json
|
||||||
|
import base64
|
||||||
|
import re
|
||||||
|
import zipfile
|
||||||
|
import ipaddress
|
||||||
|
import io
|
||||||
|
import logging
|
||||||
|
|
||||||
|
misperrors = {'error': 'Error'}
|
||||||
|
userConfig = {}
|
||||||
|
inputSource = ['file']
|
||||||
|
|
||||||
|
moduleinfo = {'version': '0.6', 'author': 'Christophe Vandeplas',
|
||||||
|
'description': 'Import for ThreatAnalyzer archive.zip/analysis.json files',
|
||||||
|
'module-type': ['import']}
|
||||||
|
|
||||||
|
moduleconfig = []
|
||||||
|
log = logging.getLogger('misp-modules')
|
||||||
|
|
||||||
|
# FIXME - many hardcoded filters should be migrated to import regexes. See also https://github.com/MISP/MISP/issues/2712
|
||||||
|
# DISCLAIMER - This module is to be considered as experimental and needs much fine-tuning.
|
||||||
|
# more can be done with what's in the ThreatAnalyzer archive.zip
|
||||||
|
|
||||||
|
|
||||||
|
def handler(q=False):
|
||||||
|
if q is False:
|
||||||
|
return False
|
||||||
|
results = []
|
||||||
|
zip_starts = 'PK'
|
||||||
|
request = json.loads(q)
|
||||||
|
data = base64.b64decode(request['data'])
|
||||||
|
|
||||||
|
if data[:len(zip_starts)].decode() == zip_starts:
|
||||||
|
with zipfile.ZipFile(io.BytesIO(data), 'r') as zf:
|
||||||
|
# unzipped_files = []
|
||||||
|
modified_files_mapping = {}
|
||||||
|
# pre-process some of the files in the zip
|
||||||
|
for zip_file_name in zf.namelist(): # Get all files in the zip file
|
||||||
|
# find the filenames of the modified_files
|
||||||
|
if re.match(r"Analysis/proc_\d+/modified_files/mapping\.log", zip_file_name):
|
||||||
|
with zf.open(zip_file_name, mode='r', pwd=None) as fp:
|
||||||
|
file_data = fp.read()
|
||||||
|
for line in file_data.decode().split('\n'):
|
||||||
|
if line:
|
||||||
|
l_fname, l_size, l_md5, l_created = line.split('|')
|
||||||
|
l_fname = cleanup_filepath(l_fname)
|
||||||
|
if l_fname:
|
||||||
|
if l_size == 0:
|
||||||
|
pass # FIXME create an attribute for the filename/path
|
||||||
|
else:
|
||||||
|
# file is a non empty sample, upload the sample later
|
||||||
|
modified_files_mapping[l_md5] = l_fname
|
||||||
|
|
||||||
|
# now really process the data
|
||||||
|
for zip_file_name in zf.namelist(): # Get all files in the zip file
|
||||||
|
# print('Processing file: {}'.format(zip_file_name))
|
||||||
|
if re.match(r"Analysis/proc_\d+/modified_files/.+\.", zip_file_name) and "mapping.log" not in zip_file_name:
|
||||||
|
sample_md5 = zip_file_name.split('/')[-1].split('.')[0]
|
||||||
|
if sample_md5 in modified_files_mapping:
|
||||||
|
sample_filename = modified_files_mapping[sample_md5]
|
||||||
|
# print("{} maps to {}".format(sample_md5, sample_filename))
|
||||||
|
with zf.open(zip_file_name, mode='r', pwd=None) as fp:
|
||||||
|
file_data = fp.read()
|
||||||
|
results.append({
|
||||||
|
'values': sample_filename,
|
||||||
|
'data': base64.b64encode(file_data).decode(),
|
||||||
|
'type': 'malware-sample', 'categories': ['Artifacts dropped', 'Payload delivery'], 'to_ids': True, 'comment': ''})
|
||||||
|
|
||||||
|
if 'Analysis/analysis.json' in zip_file_name:
|
||||||
|
with zf.open(zip_file_name, mode='r', pwd=None) as fp:
|
||||||
|
file_data = fp.read()
|
||||||
|
analysis_json = json.loads(file_data.decode('utf-8'))
|
||||||
|
results += process_analysis_json(analysis_json)
|
||||||
|
# if 'sample' in zip_file_name:
|
||||||
|
# sample['data'] = base64.b64encode(file_data).decode()
|
||||||
|
|
||||||
|
else:
|
||||||
|
try:
|
||||||
|
results = process_analysis_json(json.loads(data.decode('utf-8')))
|
||||||
|
except ValueError:
|
||||||
|
log.warning('MISP modules {0} failed: uploaded file is not a zip or json file.'.format(request['module']))
|
||||||
|
return {'error': 'Uploaded file is not a zip or json file.'.format(request['module'])}
|
||||||
|
pass
|
||||||
|
# keep only unique entries based on the value field
|
||||||
|
results = list({v['values']: v for v in results}.values())
|
||||||
|
r = {'results': results}
|
||||||
|
return r
|
||||||
|
|
||||||
|
|
||||||
|
def process_analysis_json(analysis_json):
|
||||||
|
if 'analysis' in analysis_json and 'processes' in analysis_json['analysis'] and 'process' in analysis_json['analysis']['processes']:
|
||||||
|
# if 'analysis' in analysis_json and '@filename' in analysis_json['analysis']:
|
||||||
|
# sample['values'] = analysis_json['analysis']['@filename']
|
||||||
|
for process in analysis_json['analysis']['processes']['process']:
|
||||||
|
# print_json(process)
|
||||||
|
if 'connection_section' in process and 'connection' in process['connection_section']:
|
||||||
|
for connection_section_connection in process['connection_section']['connection']:
|
||||||
|
|
||||||
|
connection_section_connection['@remote_ip'] = cleanup_ip(connection_section_connection['@remote_ip'])
|
||||||
|
connection_section_connection['@remote_hostname'] = cleanup_hostname(connection_section_connection['@remote_hostname'])
|
||||||
|
if connection_section_connection['@remote_ip'] and connection_section_connection['@remote_hostname']:
|
||||||
|
val = '{}|{}'.format(connection_section_connection['@remote_hostname'],
|
||||||
|
connection_section_connection['@remote_ip'])
|
||||||
|
# print("connection_section_connection hostname|ip: {}|{} IDS:yes".format(
|
||||||
|
# connection_section_connection['@remote_hostname'],
|
||||||
|
# connection_section_connection['@remote_ip'])
|
||||||
|
# )
|
||||||
|
yield({'values': val, 'type': 'domain|ip', 'categories': 'Network activity', 'to_ids': True, 'comment': ''})
|
||||||
|
elif connection_section_connection['@remote_ip']:
|
||||||
|
# print("connection_section_connection ip-dst: {} IDS:yes".format(
|
||||||
|
# connection_section_connection['@remote_ip'])
|
||||||
|
# )
|
||||||
|
yield({'values': connection_section_connection['@remote_ip'], 'type': 'ip-dst', 'to_ids': True, 'comment': ''})
|
||||||
|
elif connection_section_connection['@remote_hostname']:
|
||||||
|
# print("connection_section_connection hostname: {} IDS:yes".format(
|
||||||
|
# connection_section_connection['@remote_hostname'])
|
||||||
|
# )
|
||||||
|
yield({'values': connection_section_connection['@remote_hostname'], 'type': 'hostname', 'to_ids': True, 'comment': ''})
|
||||||
|
if 'http_command' in connection_section_connection:
|
||||||
|
for http_command in connection_section_connection['http_command']:
|
||||||
|
# print('connection_section_connection HTTP COMMAND: {}\t{}'.format(
|
||||||
|
# http_command['@method'], # comment
|
||||||
|
# http_command['@url']) # url
|
||||||
|
# )
|
||||||
|
val = cleanup_url(http_command['@url'])
|
||||||
|
if val:
|
||||||
|
yield({'values': val, 'type': 'url', 'categories': 'Network activity', 'to_ids': True, 'comment': http_command['@method']})
|
||||||
|
|
||||||
|
if 'http_header' in connection_section_connection:
|
||||||
|
for http_header in connection_section_connection['http_header']:
|
||||||
|
if 'User-Agent:' in http_header['@header']:
|
||||||
|
val = http_header['@header'][len('User-Agent: '):]
|
||||||
|
yield({'values': val, 'type': 'user-agent', 'categories': 'Network activity', 'to_ids': False, 'comment': ''})
|
||||||
|
elif 'Host:' in http_header['@header']:
|
||||||
|
val = http_header['@header'][len('Host: '):]
|
||||||
|
if ':' in val:
|
||||||
|
try:
|
||||||
|
val_port = int(val.split(':')[1])
|
||||||
|
except ValueError as e:
|
||||||
|
val_port = False
|
||||||
|
val_hostname = cleanup_hostname(val.split(':')[0])
|
||||||
|
val_ip = cleanup_ip(val.split(':')[0])
|
||||||
|
if val_hostname and val_port:
|
||||||
|
val_combined = '{}|{}'.format(val_hostname, val_port)
|
||||||
|
# print({'values': val_combined, 'type': 'hostname|port', 'to_ids': True, 'comment': ''})
|
||||||
|
yield({'values': val_combined, 'type': 'hostname|port', 'to_ids': True, 'comment': ''})
|
||||||
|
elif val_ip and val_port:
|
||||||
|
val_combined = '{}|{}'.format(val_ip, val_port)
|
||||||
|
# print({'values': val_combined, 'type': 'ip-dst|port', 'to_ids': True, 'comment': ''})
|
||||||
|
yield({'values': val_combined, 'type': 'ip-dst|port', 'to_ids': True, 'comment': ''})
|
||||||
|
else:
|
||||||
|
continue
|
||||||
|
val_hostname = cleanup_hostname(val)
|
||||||
|
if val_hostname:
|
||||||
|
# print({'values': val_hostname, 'type': 'hostname', 'to_ids': True, 'comment': ''})
|
||||||
|
yield({'values': val_hostname, 'type': 'hostname', 'to_ids': True, 'comment': ''})
|
||||||
|
else:
|
||||||
|
# LATER header not processed
|
||||||
|
pass
|
||||||
|
if 'filesystem_section' in process and 'create_file' in process['filesystem_section']:
|
||||||
|
for filesystem_section_create_file in process['filesystem_section']['create_file']:
|
||||||
|
# first skip some items
|
||||||
|
if filesystem_section_create_file['@create_disposition'] in {'FILE_OPEN_IF'}:
|
||||||
|
continue
|
||||||
|
# FIXME - this section is probably not needed considering the 'stored_files stored_created_file' section we process later.
|
||||||
|
# print('CREATE FILE: {}\t{}'.format(
|
||||||
|
# filesystem_section_create_file['@srcfile'], # filename
|
||||||
|
# filesystem_section_create_file['@create_disposition']) # comment - use this to filter out cases
|
||||||
|
# )
|
||||||
|
|
||||||
|
if 'networkoperation_section' in process and 'dns_request_by_addr' in process['networkoperation_section']:
|
||||||
|
for networkoperation_section_dns_request_by_addr in process['networkoperation_section']['dns_request_by_addr']:
|
||||||
|
# FIXME - it's unclear what this section is for.
|
||||||
|
# TODO filter this
|
||||||
|
# print('DNS REQUEST: {}\t{}'.format(
|
||||||
|
# networkoperation_section_dns_request_by_addr['@request_address'], # ip-dst
|
||||||
|
# networkoperation_section_dns_request_by_addr['@result_name']) # hostname
|
||||||
|
# ) # => NOT hostname|ip
|
||||||
|
pass
|
||||||
|
if 'networkoperation_section' in process and 'dns_request_by_name' in process['networkoperation_section']:
|
||||||
|
for networkoperation_section_dns_request_by_name in process['networkoperation_section']['dns_request_by_name']:
|
||||||
|
networkoperation_section_dns_request_by_name['@request_name'] = cleanup_hostname(networkoperation_section_dns_request_by_name['@request_name'].rstrip('.'))
|
||||||
|
networkoperation_section_dns_request_by_name['@result_addresses'] = cleanup_ip(networkoperation_section_dns_request_by_name['@result_addresses'])
|
||||||
|
if networkoperation_section_dns_request_by_name['@request_name'] and networkoperation_section_dns_request_by_name['@result_addresses']:
|
||||||
|
val = '{}|{}'.format(networkoperation_section_dns_request_by_name['@request_name'],
|
||||||
|
networkoperation_section_dns_request_by_name['@result_addresses'])
|
||||||
|
# print("networkoperation_section_dns_request_by_name hostname|ip: {}|{} IDS:yes".format(
|
||||||
|
# networkoperation_section_dns_request_by_name['@request_name'],
|
||||||
|
# networkoperation_section_dns_request_by_name['@result_addresses'])
|
||||||
|
# )
|
||||||
|
yield({'values': val, 'type': 'domain|ip', 'categories': 'Network activity', 'to_ids': True, 'comment': ''})
|
||||||
|
elif networkoperation_section_dns_request_by_name['@request_name']:
|
||||||
|
# print("networkoperation_section_dns_request_by_name hostname: {} IDS:yes".format(
|
||||||
|
# networkoperation_section_dns_request_by_name['@request_name'])
|
||||||
|
# )
|
||||||
|
yield({'values': networkoperation_section_dns_request_by_name['@request_name'], 'type': 'hostname', 'to_ids': True, 'comment': ''})
|
||||||
|
elif networkoperation_section_dns_request_by_name['@result_addresses']:
|
||||||
|
# this happens when the IP is both in the request_name and result_address.
|
||||||
|
# print("networkoperation_section_dns_request_by_name hostname: {} IDS:yes".format(
|
||||||
|
# networkoperation_section_dns_request_by_name['@result_addresses'])
|
||||||
|
# )
|
||||||
|
yield({'values': networkoperation_section_dns_request_by_name['@result_addresses'], 'type': 'ip-dst', 'to_ids': True, 'comment': ''})
|
||||||
|
|
||||||
|
if 'networkpacket_section' in process and 'connect_to_computer' in process['networkpacket_section']:
|
||||||
|
for networkpacket_section_connect_to_computer in process['networkpacket_section']['connect_to_computer']:
|
||||||
|
networkpacket_section_connect_to_computer['@remote_hostname'] = cleanup_hostname(networkpacket_section_connect_to_computer['@remote_hostname'])
|
||||||
|
networkpacket_section_connect_to_computer['@remote_ip'] = cleanup_ip(networkpacket_section_connect_to_computer['@remote_ip'])
|
||||||
|
if networkpacket_section_connect_to_computer['@remote_hostname'] and networkpacket_section_connect_to_computer['@remote_ip']:
|
||||||
|
# print("networkpacket_section_connect_to_computer hostname|ip: {}|{} IDS:yes COMMENT:port {}".format(
|
||||||
|
# networkpacket_section_connect_to_computer['@remote_hostname'],
|
||||||
|
# networkpacket_section_connect_to_computer['@remote_ip'],
|
||||||
|
# networkpacket_section_connect_to_computer['@remote_port'])
|
||||||
|
# )
|
||||||
|
val_combined = "{}|{}".format(networkpacket_section_connect_to_computer['@remote_hostname'], networkpacket_section_connect_to_computer['@remote_ip'])
|
||||||
|
yield({'values': val_combined, 'type': 'hostname|ip', 'to_ids': True, 'comment': ''})
|
||||||
|
elif networkpacket_section_connect_to_computer['@remote_hostname']:
|
||||||
|
# print("networkpacket_section_connect_to_computer hostname: {} IDS:yes COMMENT:port {}".format(
|
||||||
|
# networkpacket_section_connect_to_computer['@remote_hostname'],
|
||||||
|
# networkpacket_section_connect_to_computer['@remote_port'])
|
||||||
|
# )
|
||||||
|
val_combined = "{}|{}".format(networkpacket_section_connect_to_computer['@remote_hostname'], networkpacket_section_connect_to_computer['@remote_port'])
|
||||||
|
yield({'values': val_combined, 'type': 'hostname|port', 'to_ids': True, 'comment': ''})
|
||||||
|
elif networkpacket_section_connect_to_computer['@remote_ip']:
|
||||||
|
# print("networkpacket_section_connect_to_computer ip-dst: {} IDS:yes COMMENT:port {}".format(
|
||||||
|
# networkpacket_section_connect_to_computer['@remote_ip'],
|
||||||
|
# networkpacket_section_connect_to_computer['@remote_port'])
|
||||||
|
# )
|
||||||
|
val_combined = "{}|{}".format(networkpacket_section_connect_to_computer['@remote_ip'], networkpacket_section_connect_to_computer['@remote_port'])
|
||||||
|
yield({'values': val_combined, 'type': 'ip-dst|port', 'to_ids': True, 'comment': ''})
|
||||||
|
|
||||||
|
if 'registry_section' in process and 'create_key' in process['registry_section']:
|
||||||
|
# FIXME this is a complicated section, together with the 'set_value'.
|
||||||
|
# it looks like this section is not ONLY about creating registry keys,
|
||||||
|
# more about accessing a handle to keys (with specific permissions)
|
||||||
|
# maybe we don't want to keep this, in favor of 'set_value'
|
||||||
|
for create_key in process['registry_section']['create_key']:
|
||||||
|
# print('REG CREATE: {}\t{}'.format(
|
||||||
|
# create_key['@desired_access'],
|
||||||
|
# create_key['@key_name']))
|
||||||
|
pass
|
||||||
|
if 'registry_section' in process and 'delete_key' in process['registry_section']:
|
||||||
|
# LATER we probably don't want to keep this. Much pollution.
|
||||||
|
# Maybe for later once we have filtered out this.
|
||||||
|
for delete_key in process['registry_section']['delete_key']:
|
||||||
|
# print('REG DELETE: {}'.format(
|
||||||
|
# delete_key['@key_name'])
|
||||||
|
# )
|
||||||
|
pass
|
||||||
|
if 'registry_section' in process and 'set_value' in process['registry_section']:
|
||||||
|
# FIXME this is a complicated section, together with the 'create_key'.
|
||||||
|
for set_value in process['registry_section']['set_value']:
|
||||||
|
# '@data_type' == 'REG_BINARY',
|
||||||
|
# '@data_type' == 'REG_DWORD',
|
||||||
|
# '@data_type' == 'REG_EXPAND_SZ',
|
||||||
|
# '@data_type' == 'REG_MULTI_SZ',
|
||||||
|
# '@data_type' == 'REG_NONE',
|
||||||
|
# '@data_type' == 'REG_QWORD',
|
||||||
|
# '@data_type' == 'REG_SZ',
|
||||||
|
regkey = cleanup_regkey("{}\\{}".format(set_value['@key_name'], set_value['@value_name']))
|
||||||
|
regdata = cleanup_regdata(set_value.get('@data'))
|
||||||
|
if not regkey:
|
||||||
|
continue
|
||||||
|
if set_value['@data_size'] == '0' or not regdata:
|
||||||
|
# print('registry_section set_value REG SET: {}\t{}\t{}'.format(
|
||||||
|
# set_value['@data_type'],
|
||||||
|
# set_value['@key_name'],
|
||||||
|
# set_value['@value_name'])
|
||||||
|
# )
|
||||||
|
yield({'values': regkey, 'type': 'regkey', 'to_ids': True,
|
||||||
|
'categories': ['External analysis', 'Persistence mechanism', 'Artifacts dropped'], 'comment': set_value['@data_type']})
|
||||||
|
else:
|
||||||
|
try:
|
||||||
|
# unicode fun...
|
||||||
|
# print('registry_section set_value REG SET: {}\t{}\t{}\t{}'.format(
|
||||||
|
# set_value['@data_type'],
|
||||||
|
# set_value['@key_name'],
|
||||||
|
# set_value['@value_name'],
|
||||||
|
# set_value['@data'])
|
||||||
|
# )
|
||||||
|
val = "{}|{}".format(regkey, regdata)
|
||||||
|
yield({'values': val, 'type': 'regkey|value', 'to_ids': True,
|
||||||
|
'categories': ['External analysis', 'Persistence mechanism', 'Artifacts dropped'], 'comment': set_value['@data_type']})
|
||||||
|
except Exception as e:
|
||||||
|
print("EXCEPTION registry_section {}".format(e))
|
||||||
|
# TODO - maybe we want to handle these later, or not...
|
||||||
|
pass
|
||||||
|
pass
|
||||||
|
|
||||||
|
if 'stored_files' in process and 'stored_created_file' in process['stored_files']:
|
||||||
|
for stored_created_file in process['stored_files']['stored_created_file']:
|
||||||
|
stored_created_file['@filename'] = cleanup_filepath(stored_created_file['@filename'])
|
||||||
|
if stored_created_file['@filename']:
|
||||||
|
if stored_created_file['@filesize'] is not '0':
|
||||||
|
val = '{}|{}'.format(stored_created_file['@filename'], stored_created_file['@md5'])
|
||||||
|
# print("stored_created_file filename|md5: {}|{} IDS:yes".format(
|
||||||
|
# stored_created_file['@filename'], # filename
|
||||||
|
# stored_created_file['@md5']) # md5
|
||||||
|
# ) # => filename|md5
|
||||||
|
yield({'values': val, 'type': 'filename|md5', 'to_ids': True,
|
||||||
|
'categories': ['Artifacts dropped', 'Payload delivery'], 'comment': ''})
|
||||||
|
|
||||||
|
else:
|
||||||
|
# print("stored_created_file filename: {} IDS:yes".format(
|
||||||
|
# stored_created_file['@filename']) # filename
|
||||||
|
# ) # => filename
|
||||||
|
yield({'values': stored_created_file['@filename'],
|
||||||
|
'type': 'filename', 'to_ids': True,
|
||||||
|
'categories': ['Artifacts dropped', 'Payload delivery'], 'comment': ''})
|
||||||
|
|
||||||
|
if 'stored_files' in process and 'stored_modified_file' in process['stored_files']:
|
||||||
|
for stored_modified_file in process['stored_files']['stored_modified_file']:
|
||||||
|
stored_modified_file['@filename'] = cleanup_filepath(stored_modified_file['@filename'])
|
||||||
|
if stored_modified_file['@filename']:
|
||||||
|
if stored_modified_file['@filesize'] is not '0':
|
||||||
|
val = '{}|{}'.format(stored_modified_file['@filename'], stored_modified_file['@md5'])
|
||||||
|
# print("stored_modified_file MODIFY FILE: {}\t{}".format(
|
||||||
|
# stored_modified_file['@filename'], # filename
|
||||||
|
# stored_modified_file['@md5']) # md5
|
||||||
|
# ) # => filename|md5
|
||||||
|
yield({'values': val, 'type': 'filename|md5', 'to_ids': True,
|
||||||
|
'categories': ['Artifacts dropped', 'Payload delivery'],
|
||||||
|
'comment': 'modified'})
|
||||||
|
else:
|
||||||
|
# print("stored_modified_file MODIFY FILE: {}\t{}".format(
|
||||||
|
# stored_modified_file['@filename']) # filename
|
||||||
|
# ) # => filename
|
||||||
|
yield({'values': stored_modified_file['@filename'], 'type': 'filename', 'to_ids': True,
|
||||||
|
'categories': ['Artifacts dropped', 'Payload delivery'],
|
||||||
|
'comment': 'modified'})
|
||||||
|
|
||||||
|
|
||||||
|
def add_file(filename, results, hash, index, filedata=None):
|
||||||
|
pass
|
||||||
|
# results.append({'values': filename, 'data': "{}|{}".format(filename, filedata.decode()), 'type': 'malware-sample',
|
||||||
|
# 'categories': ['Artifacts dropped', 'Payload delivery']})
|
||||||
|
|
||||||
|
|
||||||
|
def add_file_zip():
|
||||||
|
# if 'malware-sample' in request:
|
||||||
|
# sample_filename = request.get("malware-sample").split("|", 1)[0]
|
||||||
|
# data = base64.b64decode(data)
|
||||||
|
# fl = io.BytesIO(data)
|
||||||
|
# zf = zipfile.ZipFile(fl)
|
||||||
|
# sample_hashname = zf.namelist()[0]
|
||||||
|
# data = zf.read(sample_hashname, b"infected")
|
||||||
|
# zf.close()
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def print_json(data):
|
||||||
|
print(json.dumps(data, sort_keys=True, indent=4, separators=(',', ': ')))
|
||||||
|
|
||||||
|
|
||||||
|
def list_in_string(lst, data, regex=False):
|
||||||
|
for item in lst:
|
||||||
|
if regex:
|
||||||
|
if re.search(item, data, flags=re.IGNORECASE):
|
||||||
|
return True
|
||||||
|
else:
|
||||||
|
if item in data:
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def cleanup_ip(item):
|
||||||
|
# you should exclude private IP ranges via import regexes
|
||||||
|
noise_substrings = {
|
||||||
|
'224.0.0.',
|
||||||
|
'127.0.0.',
|
||||||
|
'8.8.8.8',
|
||||||
|
'8.8.4.4',
|
||||||
|
'0.0.0.0',
|
||||||
|
'NONE'
|
||||||
|
}
|
||||||
|
if list_in_string(noise_substrings, item):
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
ipaddress.ip_address(item)
|
||||||
|
return item
|
||||||
|
except ValueError:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def cleanup_hostname(item):
|
||||||
|
noise_substrings = {
|
||||||
|
'wpad',
|
||||||
|
'teredo.ipv6.microsoft.com',
|
||||||
|
'WIN7SP1-x64-UNP'
|
||||||
|
}
|
||||||
|
# take away common known bad
|
||||||
|
if list_in_string(noise_substrings, item):
|
||||||
|
return None
|
||||||
|
# eliminate IP addresses
|
||||||
|
try:
|
||||||
|
ipaddress.ip_address(item)
|
||||||
|
except ValueError:
|
||||||
|
# this is not an IP, so continue
|
||||||
|
return item
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def cleanup_url(item):
|
||||||
|
if item in ['/']:
|
||||||
|
return None
|
||||||
|
return item
|
||||||
|
|
||||||
|
|
||||||
|
def cleanup_filepath(item):
|
||||||
|
noise_substrings = {
|
||||||
|
'C:\\Windows\\Prefetch\\',
|
||||||
|
'\\AppData\\Roaming\\Microsoft\\Windows\\Recent\\',
|
||||||
|
'\\AppData\\Roaming\\Microsoft\\Office\\Recent\\',
|
||||||
|
'C:\\ProgramData\\Microsoft\\OfficeSoftwareProtectionPlatform\\Cache\\cache.dat',
|
||||||
|
'\\AppData\\Local\\Microsoft\\Windows\\Temporary Internet Files\\Content.',
|
||||||
|
'\\AppData\\Local\\Microsoft\\Internet Explorer\\Recovery\\High\\',
|
||||||
|
'\\AppData\\Local\\Microsoft\\Internet Explorer\\DOMStore\\',
|
||||||
|
'\\AppData\\LocalLow\\Microsoft\\Internet Explorer\\Services\\search_',
|
||||||
|
'\\AppData\\Local\\Microsoft\\Windows\\History\\History.',
|
||||||
|
'\\AppData\\Roaming\\Microsoft\\Windows\\Cookies\\',
|
||||||
|
'\\AppData\\LocalLow\\Microsoft\\CryptnetUrlCache\\',
|
||||||
|
'\\AppData\\Local\\Microsoft\\Windows\\Caches\\',
|
||||||
|
'\\AppData\\Local\\Microsoft\\Windows\WebCache\\',
|
||||||
|
'\\AppData\\Local\\Microsoft\\Windows\\Explorer\\thumbcache',
|
||||||
|
|
||||||
|
'\\AppData\\Roaming\\Adobe\\Acrobat\\9.0\\SharedDataEvents-journal',
|
||||||
|
'\\AppData\\Roaming\\Adobe\\Acrobat\\9.0\\UserCache.bin',
|
||||||
|
|
||||||
|
'\\AppData\\Roaming\\Macromedia\\Flash Player\\macromedia.com\\support\\flashplayer\\sys\\settings.sol',
|
||||||
|
'\\AppData\\Roaming\Adobe\\Flash Player\\NativeCache\\',
|
||||||
|
'C:\\Windows\\AppCompat\\Programs\\',
|
||||||
|
'C:\~' # caused by temp file created by MS Office when opening malicious doc/xls/...
|
||||||
|
}
|
||||||
|
if list_in_string(noise_substrings, item):
|
||||||
|
return None
|
||||||
|
return item
|
||||||
|
|
||||||
|
|
||||||
|
def cleanup_regkey(item):
|
||||||
|
noise_substrings = {
|
||||||
|
r'\\Software\\Microsoft\\Windows\\CurrentVersion\\Installer\\UserData\\',
|
||||||
|
r'\\Software\\Microsoft\\Windows\\CurrentVersion\\Internet Settings\\',
|
||||||
|
r'\\CurrentVersion\\Explorer\\RecentDocs\\',
|
||||||
|
r'\\CurrentVersion\\Explorer\\UserAssist\\',
|
||||||
|
r'\\CurrentVersion\\Explorer\\FileExts\\[a-z\.]+\\OpenWith',
|
||||||
|
r'\\Software\\Microsoft\\Internet Explorer\\Main\\WindowsSearch',
|
||||||
|
r'\\Software\\Microsoft\\Office\\[0-9\.]+\\',
|
||||||
|
r'\\SOFTWARE\\Microsoft\\OfficeSoftwareProtectionPlatform\\',
|
||||||
|
r'\\Software\\Microsoft\\Office\\Common\\Smart Tag\\',
|
||||||
|
r'\\Usage\\SpellingAndGrammarFiles',
|
||||||
|
r'^HKLM\\Software\\Microsoft\\Tracing\\',
|
||||||
|
r'\\Software\\Classes\\CLSID\\',
|
||||||
|
r'\\Software\\Classes\\Local Settings\\MuiCache\\',
|
||||||
|
r'\\Local Settings\\Software\\Microsoft\\Windows\\Shell\\Bag',
|
||||||
|
r'\\Software\\Microsoft\\Windows\\CurrentVersion\\Explorer\\RunMRU\\'
|
||||||
|
}
|
||||||
|
item = item.replace('\\REGISTRY\\MACHINE\\', 'HKLM\\')
|
||||||
|
item = item.replace('\\REGISTRY\\USER\\', 'HKCU\\')
|
||||||
|
if list_in_string(noise_substrings, item, regex=True):
|
||||||
|
return None
|
||||||
|
return item
|
||||||
|
|
||||||
|
|
||||||
|
def cleanup_regdata(item):
|
||||||
|
if not item:
|
||||||
|
return None
|
||||||
|
item = item.replace('(UNICODE_0x00000000)', '')
|
||||||
|
return item
|
||||||
|
|
||||||
|
|
||||||
|
def get_zipped_contents(filename, data, password=None):
|
||||||
|
with zipfile.ZipFile(io.BytesIO(data), 'r') as zf:
|
||||||
|
unzipped_files = []
|
||||||
|
if password is not None:
|
||||||
|
password = str.encode(password) # Byte encoded password required
|
||||||
|
for zip_file_name in zf.namelist(): # Get all files in the zip file
|
||||||
|
# print(zip_file_name)
|
||||||
|
with zf.open(zip_file_name, mode='r', pwd=password) as fp:
|
||||||
|
file_data = fp.read()
|
||||||
|
unzipped_files.append({'values': zip_file_name,
|
||||||
|
'data': file_data,
|
||||||
|
'comment': 'Extracted from {0}'.format(filename)})
|
||||||
|
# print("{} : {}".format(zip_file_name, len(file_data)))
|
||||||
|
return unzipped_files
|
||||||
|
|
||||||
|
|
||||||
|
def introspection():
|
||||||
|
modulesetup = {}
|
||||||
|
try:
|
||||||
|
userConfig
|
||||||
|
modulesetup['userConfig'] = userConfig
|
||||||
|
except NameError:
|
||||||
|
pass
|
||||||
|
try:
|
||||||
|
inputSource
|
||||||
|
modulesetup['inputSource'] = inputSource
|
||||||
|
except NameError:
|
||||||
|
pass
|
||||||
|
return modulesetup
|
||||||
|
|
||||||
|
|
||||||
|
def version():
|
||||||
|
moduleinfo['config'] = moduleconfig
|
||||||
|
return moduleinfo
|
||||||
Loading…
Reference in New Issue