Merge branch 'master' of github.com:MISP/misp-modules into features_csvimport

pull/304/head
chrisr3d 2018-09-03 15:40:19 +02:00
commit fcc18cbd73
5 changed files with 352 additions and 17 deletions

View File

@ -23,6 +23,7 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/
* [countrycode](misp_modules/modules/expansion/countrycode.py) - a hover module to tell you what country a URL belongs to.
* [CrowdStrike Falcon](misp_modules/modules/expansion/crowdstrike_falcon.py) - an expansion module to expand using CrowdStrike Falcon Intel Indicator API.
* [CVE](misp_modules/modules/expansion/cve.py) - a hover module to give more information about a vulnerability (CVE).
* [DBL Spamhaus](misp_modules/modules/expansion/dbl_spamhaus.py) - a hover module to check Spamhaus DBL for a domain name.
* [DNS](misp_modules/modules/expansion/dns.py) - a simple module to resolve MISP attributes like hostname and domain to expand IP addresses attributes.
* [DomainTools](misp_modules/modules/expansion/domaintools.py) - a hover and expansion module to get information from [DomainTools](http://www.domaintools.com/) whois.
* [EUPI](misp_modules/modules/expansion/eupi.py) - a hover and expansion module to get information about an URL from the [Phishing Initiative project](https://phishing-initiative.eu/?lang=en).

View File

@ -1,3 +1,3 @@
from . import _vmray
__all__ = ['vmray_submit', 'asn_history', 'circl_passivedns', 'circl_passivessl', 'countrycode', 'cve', 'dns', 'domaintools', 'eupi', 'farsight_passivedns', 'ipasn', 'passivetotal', 'sourcecache', 'virustotal', 'whois', 'shodan', 'reversedns', 'geoip_country', 'wiki', 'iprep', 'threatminer', 'otx', 'threatcrowd', 'vulndb', 'crowdstrike_falcon', 'yara_syntax_validator', 'hashdd', 'onyphe', 'onyphe_full', 'rbl', 'xforceexchange', 'sigma_syntax_validator', 'stix2_pattern_syntax_validator', 'sigma_queries']
__all__ = ['vmray_submit', 'asn_history', 'circl_passivedns', 'circl_passivessl', 'countrycode', 'cve', 'dns', 'domaintools', 'eupi', 'farsight_passivedns', 'ipasn', 'passivetotal', 'sourcecache', 'virustotal', 'whois', 'shodan', 'reversedns', 'geoip_country', 'wiki', 'iprep', 'threatminer', 'otx', 'threatcrowd', 'vulndb', 'crowdstrike_falcon', 'yara_syntax_validator', 'hashdd', 'onyphe', 'onyphe_full', 'rbl', 'xforceexchange', 'sigma_syntax_validator', 'stix2_pattern_syntax_validator', 'sigma_queries', 'dbl_spamhaus']

View File

@ -0,0 +1,60 @@
import json
import datetime
from collections import defaultdict
try:
import dns.resolver
resolver = dns.resolver.Resolver()
resolver.timeout = 0.2
resolver.lifetime = 0.2
except ModuleNotFoundError:
print("dnspython3 is missing, use 'pip install dnspython3' to install it.")
sys.exit(0)
misperrors = {'error': 'Error'}
mispattributes = {'input': ['domain', 'domain|ip', 'hostname', 'hostname|port'], 'output': ['text']}
moduleinfo = {'version': '0.1', 'author': 'Christian Studer',
'description': 'Checks Spamhaus DBL for a domain name.',
'module-type': ['expansion', 'hover']}
moduleconfig = []
dbl = 'dbl.spamhaus.org'
dbl_mapping = {'127.0.1.2': 'spam domain',
'127.0.1.4': 'phish domain',
'127.0.1.5': 'malware domain',
'127.0.1.6': 'botnet C&C domain',
'127.0.1.102': 'abused legit spam',
'127.0.1.103': 'abused spammed redirector domain',
'127.0.1.104': 'abused legit phish',
'127.0.1.105': 'abused legit malware',
'127.0.1.106': 'abused legit botnet C&C',
'127.0.1.255': 'IP queries prohibited!'}
def fetch_requested_value(request):
for attribute_type in mispattributes['input']:
if request.get(attribute_type):
return request[attribute_type].split('|')[0]
return None
def handler(q=False):
if q is False:
return False
request = json.loads(q)
requested_value = fetch_requested_value(request)
if requested_value is None:
misperrors['error'] = "Unsupported attributes type"
return misperrors
query = "{}.{}".format(requested_value, dbl)
try:
query_result = resolver.query(query, 'A')[0]
result = "{} - {}".format(requested_value, dbl_mapping[str(query_result)])
except Exception as e:
result = e
return {'results': [{'types': mispattributes.get('output'), 'values': result}]}
def introspection():
return mispattributes
def version():
moduleinfo['config'] = moduleconfig
return moduleinfo

View File

@ -0,0 +1,265 @@
import json
import requests
import logging
import sys
import time
log = logging.getLogger('urlscan')
log.setLevel(logging.DEBUG)
ch = logging.StreamHandler(sys.stdout)
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
log.addHandler(ch)
moduleinfo = {
'version': '0.1',
'author': 'Dave Johnson',
'description': 'Module to query urlscan.io',
'module-type': ['expansion']
}
moduleconfig = ['apikey']
misperrors = {'error': 'Error'}
mispattributes = {
'input': ['hostname', 'domain', 'url', 'hash'],
'output': ['hostname', 'domain', 'ip-src', 'ip-dst', 'url', 'text', 'link', 'hash']
}
def handler(q=False):
if q is False:
return False
request = json.loads(q)
if (request.get('config')):
if (request['config'].get('apikey') is None):
misperrors['error'] = 'urlscan apikey is missing'
return misperrors
client = urlscanAPI(request['config']['apikey'])
r = {'results': []}
if 'ip-src' in request:
r['results'] += lookup_indicator(client, request['ip-src'])
if 'ip-dst' in request:
r['results'] += lookup_indicator(client, request['ip-dst'])
if 'domain' in request:
r['results'] += lookup_indicator(client, request['domain'])
if 'hostname' in request:
r['results'] += lookup_indicator(client, request['hostname'])
if 'url' in request:
r['results'] += lookup_indicator(client, request['url'])
f 'hash' in request:
r['results'] += lookup_indicator(client, request['hash'])
# Return any errors generated from lookup to the UI and remove duplicates
uniq = []
log.debug(r['results'])
for item in r['results']:
log.debug(item)
if 'error' in item:
misperrors['error'] = item['error']
return misperrors
if item not in uniq:
uniq.append(item)
r['results'] = uniq
return r
def lookup_indicator(client, query):
result = client.search_url(query)
log.debug('RESULTS: ' + json.dumps(result))
r = []
misp_comment = "{}: Enriched via the urlscan module".format(query)
# Determine if the page is reachable
for request in result['data']['requests']:
if request['response'].get('failed'):
if request['response']['failed']['errorText']:
log.debug('The page could not load')
r.append(
{'error': 'Domain could not be resolved: {}'.format(request['response']['failed']['errorText'])})
if result.get('page'):
if result['page'].get('domain'):
misp_val = result['page']['domain']
r.append({'types': 'domain',
'categories': ['Network activity'],
'values': misp_val,
'comment': misp_comment})
if result['page'].get('ip'):
misp_val = result['page']['ip']
r.append({'types': 'ip-dst',
'categories': ['Network activity'],
'values': misp_val,
'comment': misp_comment})
if result['page'].get('country'):
misp_val = 'country: ' + result['page']['country']
if result['page'].get('city'):
misp_val += ', city: ' + result['page']['city']
r.append({'types': 'text',
'categories': ['External analysis'],
'values': misp_val,
'comment': misp_comment})
if result['page'].get('asn'):
misp_val = result['page']['asn']
r.append({'types': 'AS', 'categories': ['External analysis'], 'values': misp_val, 'comment': misp_comment})
if result['page'].get('asnname'):
misp_val = result['page']['asnname']
r.append({'types': 'text',
'categories': ['External analysis'],
'values': misp_val,
'comment': misp_comment})
if result.get('stats'):
if result['stats'].get('malicious'):
log.debug('There is something in results > stats > malicious')
threat_list = set()
if 'matches' in result['meta']['processors']['gsb']['data']:
for item in result['meta']['processors']['gsb']['data']['matches']:
if item['threatType']:
threat_list.add(item['threatType'])
threat_list = ', '.join(threat_list)
log.debug('threat_list values are: \'' + threat_list + '\'')
if threat_list:
misp_val = '{} threat(s) detected'.format(threat_list)
r.append({'types': 'text',
'categories': ['External analysis'],
'values': misp_val,
'comment': misp_comment})
if result.get('lists'):
if result['lists'].get('urls'):
for url in result['lists']['urls']:
url = url.lower()
if 'office' in url:
misp_val = "Possible Office-themed phishing"
elif 'o365' in url or '0365' in url:
misp_val = "Possible O365-themed phishing"
elif 'microsoft' in url:
misp_val = "Possible Microsoft-themed phishing"
elif 'paypal' in url:
misp_val = "Possible PayPal-themed phishing"
elif 'onedrive' in url:
misp_val = "Possible OneDrive-themed phishing"
elif 'docusign' in url:
misp_val = "Possible DocuSign-themed phishing"
r.append({'types': 'text',
'categories': ['External analysis'],
'values': misp_val,
'comment': misp_comment})
if result.get('task'):
if result['task'].get('reportURL'):
misp_val = result['task']['reportURL']
r.append({'types': 'link',
'categories': ['External analysis'],
'values': misp_val,
'comment': misp_comment})
if result['task'].get('screenshotURL'):
image_url = result['task']['screenshotURL']
r.append({'types': 'link',
'categories': ['External analysis'],
'values': image_url,
'comment': misp_comment})
### TO DO ###
### Add ability to add an in-line screenshot of the target website into an attribute
# screenshot = requests.get(image_url).content
# r.append({'types': ['attachment'],
# 'categories': ['External analysis'],
# 'values': image_url,
# 'image': str(base64.b64encode(screenshot), 'utf-8'),
# 'comment': 'Screenshot of website'})
return r
def introspection():
return mispattributes
def version():
moduleinfo['config'] = moduleconfig
return moduleinfo
class urlscanAPI():
def __init__(self, apikey=None, uuid=None):
self.key = apikey
self.uuid = uuid
def request(self, query):
log.debug('From request function with the parameter: ' + query)
payload = {'url': query}
headers = {'API-Key': self.key,
'Content-Type': "application/json",
'Cache-Control': "no-cache"}
# Troubleshooting problems with initial search request
log.debug('PAYLOAD: ' + json.dumps(payload))
log.debug('HEADERS: ' + json.dumps(headers))
search_url_string = "https://urlscan.io/api/v1/scan/"
response = requests.request("POST",
search_url_string,
data=json.dumps(payload),
headers=headers)
# HTTP 400 - Bad Request
if response.status_code == 400:
raise Exception('HTTP Error 400 - Bad Request')
# HTTP 404 - Not found
if response.status_code == 404:
raise Exception('HTTP Error 404 - These are not the droids you\'re looking for')
# Any other status code
if response.status_code != 200:
raise Exception('HTTP Error ' + str(response.status_code))
if response.text:
response = json.loads(response.content.decode("utf-8"))
time.sleep(3)
self.uuid = response['uuid']
# Strings for to check for errors on the results page
# Null response string for any unavailable resources
null_response_string = '"status": 404'
# Redirect string accounting for 301/302/303/307/308 status codes
redirect_string = '"status": 30'
# Normal response string with 200 status code
normal_response_string = '"status": 200'
results_url_string = "https://urlscan.io/api/v1/result/" + self.uuid
log.debug('Results URL: ' + results_url_string)
# Need to wait for results to process and check if they are valid
tries = 10
while tries >= 0:
results = requests.request("GET", results_url_string)
log.debug('Made a GET request')
results = results.content.decode("utf-8")
# checking if there is a 404 status code and no available resources
if null_response_string in results and \
redirect_string not in results and \
normal_response_string not in results:
log.debug('Results not processed. Please check again later.')
time.sleep(3)
tries -= 1
else:
return json.loads(results)
raise Exception('Results contained a 404 status error and could not be processed.')
def search_url(self, query):
log.debug('From search_url with parameter: ' + query)
return self.request(query)

View File

@ -15,7 +15,7 @@ misperrors = {'error': 'Error'}
userConfig = {}
inputSource = ['file']
moduleinfo = {'version': '0.7', 'author': 'Christophe Vandeplas',
moduleinfo = {'version': '0.9', 'author': 'Christophe Vandeplas',
'description': 'Import for ThreatAnalyzer archive.zip/analysis.json files',
'module-type': ['import']}
@ -45,7 +45,7 @@ def handler(q=False):
if re.match(r"Analysis/proc_\d+/modified_files/mapping\.log", zip_file_name):
with zf.open(zip_file_name, mode='r', pwd=None) as fp:
file_data = fp.read()
for line in file_data.decode().split('\n'):
for line in file_data.decode("utf-8", 'ignore').split('\n'):
if not line:
continue
if line.count('|') == 3:
@ -55,7 +55,8 @@ def handler(q=False):
l_fname = cleanup_filepath(l_fname)
if l_fname:
if l_size == 0:
pass # FIXME create an attribute for the filename/path
results.append({'values': l_fname, 'type': 'filename', 'to_ids': True,
'categories': ['Artifacts dropped', 'Payload delivery'], 'comment': ''})
else:
# file is a non empty sample, upload the sample later
modified_files_mapping[l_md5] = l_fname
@ -73,7 +74,7 @@ def handler(q=False):
results.append({
'values': current_sample_filename,
'data': base64.b64encode(file_data).decode(),
'type': 'malware-sample', 'categories': ['Payload delivery', 'Artifacts dropped'], 'to_ids': True, 'comment': ''})
'type': 'malware-sample', 'categories': ['Artifacts dropped', 'Payload delivery'], 'to_ids': True, 'comment': ''})
if 'Analysis/analysis.json' in zip_file_name:
with zf.open(zip_file_name, mode='r', pwd=None) as fp:
@ -88,7 +89,7 @@ def handler(q=False):
results.append({
'values': sample_filename,
'data': base64.b64encode(file_data).decode(),
'type': 'malware-sample', 'categories': ['Artifacts dropped', 'Payload delivery'], 'to_ids': True, 'comment': ''})
'type': 'malware-sample', 'categories': ['Payload delivery', 'Artifacts dropped'], 'to_ids': True, 'comment': ''})
except Exception as e:
# no 'sample' in archive, might be an url analysis, just ignore
pass
@ -113,7 +114,15 @@ def process_analysis_json(analysis_json):
for process in analysis_json['analysis']['processes']['process']:
# print_json(process)
if 'connection_section' in process and 'connection' in process['connection_section']:
# compensate for absurd behavior of the data format: if one entry = immediately the dict, if multiple entries = list containing dicts
# this will always create a list, even with only one item
if isinstance(process['connection_section']['connection'], dict):
process['connection_section']['connection'] = [process['connection_section']['connection']]
# iterate over each entry
for connection_section_connection in process['connection_section']['connection']:
if 'name_to_ip' in connection_section_connection: # TA 6.1 data format
connection_section_connection['@remote_ip'] = connection_section_connection['name_to_ip']['@result_addresses']
connection_section_connection['@remote_hostname'] = connection_section_connection['name_to_ip']['@request_name']
connection_section_connection['@remote_ip'] = cleanup_ip(connection_section_connection['@remote_ip'])
connection_section_connection['@remote_hostname'] = cleanup_hostname(connection_section_connection['@remote_hostname'])
@ -124,7 +133,7 @@ def process_analysis_json(analysis_json):
# connection_section_connection['@remote_hostname'],
# connection_section_connection['@remote_ip'])
# )
yield({'values': val, 'type': 'domain|ip', 'categories': 'Network activity', 'to_ids': True, 'comment': ''})
yield({'values': val, 'type': 'domain|ip', 'categories': ['Network activity'], 'to_ids': True, 'comment': ''})
elif connection_section_connection['@remote_ip']:
# print("connection_section_connection ip-dst: {} IDS:yes".format(
# connection_section_connection['@remote_ip'])
@ -138,18 +147,18 @@ def process_analysis_json(analysis_json):
if 'http_command' in connection_section_connection:
for http_command in connection_section_connection['http_command']:
# print('connection_section_connection HTTP COMMAND: {}\t{}'.format(
# http_command['@method'], # comment
# http_command['@url']) # url
# connection_section_connection['http_command']['@method'], # comment
# connection_section_connection['http_command']['@url']) # url
# )
val = cleanup_url(http_command['@url'])
if val:
yield({'values': val, 'type': 'url', 'categories': 'Network activity', 'to_ids': True, 'comment': http_command['@method']})
yield({'values': val, 'type': 'url', 'categories': ['Network activity'], 'to_ids': True, 'comment': http_command['@method']})
if 'http_header' in connection_section_connection:
for http_header in connection_section_connection['http_header']:
if 'User-Agent:' in http_header['@header']:
val = http_header['@header'][len('User-Agent: '):]
yield({'values': val, 'type': 'user-agent', 'categories': 'Network activity', 'to_ids': False, 'comment': ''})
yield({'values': val, 'type': 'user-agent', 'categories': ['Network activity'], 'to_ids': False, 'comment': ''})
elif 'Host:' in http_header['@header']:
val = http_header['@header'][len('Host: '):]
if ':' in val:
@ -162,7 +171,7 @@ def process_analysis_json(analysis_json):
if val_hostname and val_port:
val_combined = '{}|{}'.format(val_hostname, val_port)
# print({'values': val_combined, 'type': 'hostname|port', 'to_ids': True, 'comment': ''})
yield({'values': val_combined, 'type': 'hostname|port', 'to_ids': True, 'comment': ''})
yield({'values': val_combined, 'type': 'hostname|port', 'categories': ['Network activity'], 'to_ids': True, 'comment': ''})
elif val_ip and val_port:
val_combined = '{}|{}'.format(val_ip, val_port)
# print({'values': val_combined, 'type': 'ip-dst|port', 'to_ids': True, 'comment': ''})
@ -207,7 +216,7 @@ def process_analysis_json(analysis_json):
# networkoperation_section_dns_request_by_name['@request_name'],
# networkoperation_section_dns_request_by_name['@result_addresses'])
# )
yield({'values': val, 'type': 'domain|ip', 'categories': 'Network activity', 'to_ids': True, 'comment': ''})
yield({'values': val, 'type': 'domain|ip', 'categories': ['Network activity'], 'to_ids': True, 'comment': ''})
elif networkoperation_section_dns_request_by_name['@request_name']:
# print("networkoperation_section_dns_request_by_name hostname: {} IDS:yes".format(
# networkoperation_section_dns_request_by_name['@request_name'])
@ -231,14 +240,14 @@ def process_analysis_json(analysis_json):
# networkpacket_section_connect_to_computer['@remote_port'])
# )
val_combined = "{}|{}".format(networkpacket_section_connect_to_computer['@remote_hostname'], networkpacket_section_connect_to_computer['@remote_ip'])
yield({'values': val_combined, 'type': 'hostname|ip', 'to_ids': True, 'comment': ''})
yield({'values': val_combined, 'type': 'domain|ip', 'to_ids': True, 'comment': ''})
elif networkpacket_section_connect_to_computer['@remote_hostname']:
# print("networkpacket_section_connect_to_computer hostname: {} IDS:yes COMMENT:port {}".format(
# networkpacket_section_connect_to_computer['@remote_hostname'],
# networkpacket_section_connect_to_computer['@remote_port'])
# )
val_combined = "{}|{}".format(networkpacket_section_connect_to_computer['@remote_hostname'], networkpacket_section_connect_to_computer['@remote_port'])
yield({'values': val_combined, 'type': 'hostname|port', 'to_ids': True, 'comment': ''})
yield({'values': val_combined, 'type': 'hostname|port', 'categories': ['Network activity'], 'to_ids': True, 'comment': ''})
elif networkpacket_section_connect_to_computer['@remote_ip']:
# print("networkpacket_section_connect_to_computer ip-dst: {} IDS:yes COMMENT:port {}".format(
# networkpacket_section_connect_to_computer['@remote_ip'],
@ -446,9 +455,9 @@ def cleanup_filepath(item):
'\\AppData\\Roaming\\Adobe\\Acrobat\\9.0\\UserCache.bin',
'\\AppData\\Roaming\\Macromedia\\Flash Player\\macromedia.com\\support\\flashplayer\\sys\\settings.sol',
'\\AppData\\Roaming\Adobe\\Flash Player\\NativeCache\\',
'\\AppData\\Roaming\\Adobe\\Flash Player\\NativeCache\\',
'C:\\Windows\\AppCompat\\Programs\\',
'C:\~' # caused by temp file created by MS Office when opening malicious doc/xls/...
'C:\\~' # caused by temp file created by MS Office when opening malicious doc/xls/...
}
if list_in_string(noise_substrings, item):
return None