mirror of https://github.com/MISP/misp-modules
Merge branch 'main' of github.com:MISP/misp-modules into main
commit
eac43b0dc7
|
@ -505,13 +505,18 @@ A module to query the Phishing Initiative service (https://phishing-initiative.l
|
||||||
|
|
||||||
Module to access Farsight DNSDB Passive DNS.
|
Module to access Farsight DNSDB Passive DNS.
|
||||||
- **features**:
|
- **features**:
|
||||||
>This module takes a domain, hostname or IP address MISP attribute as input to query the Farsight Passive DNS API. The API returns then the result of the query with some information about the value queried.
|
>This module takes a domain, hostname or IP address MISP attribute as input to query the Farsight Passive DNS API.
|
||||||
|
> The results of rdata and rrset lookups are then returned and parsed into passive-dns objects.
|
||||||
|
>
|
||||||
|
>An API key is required to submit queries to the API.
|
||||||
|
> It is also possible to define a custom server URL, and to set a limit of results to get.
|
||||||
|
> This limit is set for each lookup, which means we can have an up to the limit number of passive-dns objects resulting from an rdata query about an IP address, but an up to the limit number of passive-dns objects for each lookup queries about a domain or a hostname (== twice the limit).
|
||||||
- **input**:
|
- **input**:
|
||||||
>A domain, hostname or IP address MISP attribute.
|
>A domain, hostname or IP address MISP attribute.
|
||||||
- **output**:
|
- **output**:
|
||||||
>Text containing information about the input, resulting from the query on the Farsight Passive DNS API.
|
>Passive-dns objects, resulting from the query on the Farsight Passive DNS API.
|
||||||
- **references**:
|
- **references**:
|
||||||
>https://www.farsightsecurity.com/
|
>https://www.farsightsecurity.com/, https://docs.dnsdb.info/dnsdb-api/
|
||||||
- **requirements**:
|
- **requirements**:
|
||||||
>An access to the Farsight Passive DNS API (apikey)
|
>An access to the Farsight Passive DNS API (apikey)
|
||||||
|
|
||||||
|
|
|
@ -3,7 +3,7 @@
|
||||||
"logo": "logos/farsight.png",
|
"logo": "logos/farsight.png",
|
||||||
"requirements": ["An access to the Farsight Passive DNS API (apikey)"],
|
"requirements": ["An access to the Farsight Passive DNS API (apikey)"],
|
||||||
"input": "A domain, hostname or IP address MISP attribute.",
|
"input": "A domain, hostname or IP address MISP attribute.",
|
||||||
"output": "Text containing information about the input, resulting from the query on the Farsight Passive DNS API.",
|
"output": "Passive-dns objects, resulting from the query on the Farsight Passive DNS API.",
|
||||||
"references": ["https://www.farsightsecurity.com/"],
|
"references": ["https://www.farsightsecurity.com/", "https://docs.dnsdb.info/dnsdb-api/"],
|
||||||
"features": "This module takes a domain, hostname or IP address MISP attribute as input to query the Farsight Passive DNS API. The API returns then the result of the query with some information about the value queried."
|
"features": "This module takes a domain, hostname or IP address MISP attribute as input to query the Farsight Passive DNS API.\n The results of rdata and rrset lookups are then returned and parsed into passive-dns objects.\n\nAn API key is required to submit queries to the API.\n It is also possible to define a custom server URL, and to set a limit of results to get.\n This limit is set for each lookup, which means we can have an up to the limit number of passive-dns objects resulting from an rdata query about an IP address, but an up to the limit number of passive-dns objects for each lookup queries about a domain or a hostname (== twice the limit)."
|
||||||
}
|
}
|
||||||
|
|
|
@ -119,7 +119,10 @@ class DnsdbClient(object):
|
||||||
break
|
break
|
||||||
yield json.loads(line.decode('ascii'))
|
yield json.loads(line.decode('ascii'))
|
||||||
except (HTTPError, URLError) as e:
|
except (HTTPError, URLError) as e:
|
||||||
raise QueryError(str(e), sys.exc_traceback)
|
try:
|
||||||
|
raise QueryError(str(e), sys.exc_traceback)
|
||||||
|
except AttributeError:
|
||||||
|
raise QueryError(str(e), sys.exc_info)
|
||||||
|
|
||||||
|
|
||||||
def quote(path):
|
def quote(path):
|
||||||
|
|
|
@ -6,19 +6,19 @@ from pymisp import MISPEvent, MISPObject
|
||||||
misperrors = {'error': 'Error'}
|
misperrors = {'error': 'Error'}
|
||||||
mispattributes = {'input': ['cpe'], 'format': 'misp_standard'}
|
mispattributes = {'input': ['cpe'], 'format': 'misp_standard'}
|
||||||
moduleinfo = {
|
moduleinfo = {
|
||||||
'version': '1',
|
'version': '2',
|
||||||
'author': 'Christian Studer',
|
'author': 'Christian Studer',
|
||||||
'description': 'An expansion module to enrich a CPE attribute with its related vulnerabilities.',
|
'description': 'An expansion module to enrich a CPE attribute with its related vulnerabilities.',
|
||||||
'module-type': ['expansion', 'hover']
|
'module-type': ['expansion', 'hover']
|
||||||
}
|
}
|
||||||
moduleconfig = ["custom_API_URL", "limit"]
|
moduleconfig = ["custom_API_URL", "limit"]
|
||||||
cveapi_url = 'https://cvepremium.circl.lu/api/cvefor/'
|
cveapi_url = 'https://cvepremium.circl.lu/api/query'
|
||||||
|
DEFAULT_LIMIT = 10
|
||||||
|
|
||||||
|
|
||||||
class VulnerabilitiesParser():
|
class VulnerabilitiesParser():
|
||||||
def __init__(self, attribute, api_url):
|
def __init__(self, attribute):
|
||||||
self.attribute = attribute
|
self.attribute = attribute
|
||||||
self.api_url = api_url
|
|
||||||
self.misp_event = MISPEvent()
|
self.misp_event = MISPEvent()
|
||||||
self.misp_event.add_attribute(**attribute)
|
self.misp_event.add_attribute(**attribute)
|
||||||
self.vulnerability_mapping = {
|
self.vulnerability_mapping = {
|
||||||
|
@ -32,11 +32,11 @@ class VulnerabilitiesParser():
|
||||||
},
|
},
|
||||||
'vulnerable_configuration': {
|
'vulnerable_configuration': {
|
||||||
'type': 'cpe',
|
'type': 'cpe',
|
||||||
'object_relation': 'vulnerable_configuration'
|
'object_relation': 'vulnerable-configuration'
|
||||||
},
|
},
|
||||||
'vulnerable_configuration_cpe_2_2': {
|
'vulnerable_configuration_cpe_2_2': {
|
||||||
'type': 'cpe',
|
'type': 'cpe',
|
||||||
'object_relation': 'vulnerable_configuration'
|
'object_relation': 'vulnerable-configuration'
|
||||||
},
|
},
|
||||||
'Modified': {
|
'Modified': {
|
||||||
'type': 'datetime',
|
'type': 'datetime',
|
||||||
|
@ -100,18 +100,26 @@ def handler(q=False):
|
||||||
attribute = request['attribute']
|
attribute = request['attribute']
|
||||||
if attribute.get('type') != 'cpe':
|
if attribute.get('type') != 'cpe':
|
||||||
return {'error': 'Wrong input attribute type.'}
|
return {'error': 'Wrong input attribute type.'}
|
||||||
api_url = check_url(request['config']['custom_API_URL']) if request['config'].get('custom_API_URL') else cveapi_url
|
config = request['config']
|
||||||
url = f"{api_url}{attribute['value']}"
|
url = check_url(config['custom_API_URL']) if config.get('custom_API_URL') else cveapi_url
|
||||||
if request['config'].get('limit'):
|
limit = int(config['limit']) if config.get('limit') else DEFAULT_LIMIT
|
||||||
url = f"{url}/{request['config']['limit']}"
|
params = {
|
||||||
response = requests.get(url)
|
"retrieve": "cves",
|
||||||
|
"dict_filter": {
|
||||||
|
"vulnerable_configuration": attribute['value']
|
||||||
|
},
|
||||||
|
"limit": limit,
|
||||||
|
"sort": "cvss",
|
||||||
|
"sort_dir": "DESC"
|
||||||
|
}
|
||||||
|
response = requests.post(url, json=params)
|
||||||
if response.status_code == 200:
|
if response.status_code == 200:
|
||||||
vulnerabilities = response.json()
|
vulnerabilities = response.json()['data']
|
||||||
if not vulnerabilities:
|
if not vulnerabilities:
|
||||||
return {'error': 'No related vulnerability for this CPE.'}
|
return {'error': 'No related vulnerability for this CPE.'}
|
||||||
else:
|
else:
|
||||||
return {'error': 'API not accessible.'}
|
return {'error': 'API not accessible.'}
|
||||||
parser = VulnerabilitiesParser(attribute, api_url)
|
parser = VulnerabilitiesParser(attribute)
|
||||||
parser.parse_vulnerabilities(vulnerabilities)
|
parser.parse_vulnerabilities(vulnerabilities)
|
||||||
return parser.get_result()
|
return parser.get_result()
|
||||||
|
|
||||||
|
|
|
@ -1,15 +1,83 @@
|
||||||
import json
|
import json
|
||||||
from ._dnsdb_query.dnsdb_query import DnsdbClient, QueryError
|
from ._dnsdb_query.dnsdb_query import DEFAULT_DNSDB_SERVER, DnsdbClient, QueryError
|
||||||
|
from . import check_input_attribute, standard_error_message
|
||||||
|
from pymisp import MISPEvent, MISPObject
|
||||||
|
|
||||||
misperrors = {'error': 'Error'}
|
misperrors = {'error': 'Error'}
|
||||||
mispattributes = {'input': ['hostname', 'domain', 'ip-src', 'ip-dst'], 'output': ['freetext']}
|
mispattributes = {
|
||||||
moduleinfo = {'version': '0.1', 'author': 'Christophe Vandeplas', 'description': 'Module to access Farsight DNSDB Passive DNS', 'module-type': ['expansion', 'hover']}
|
'input': ['hostname', 'domain', 'ip-src', 'ip-dst'],
|
||||||
moduleconfig = ['apikey']
|
'format': 'misp_standard'
|
||||||
|
}
|
||||||
|
moduleinfo = {
|
||||||
|
'version': '0.2',
|
||||||
|
'author': 'Christophe Vandeplas',
|
||||||
|
'description': 'Module to access Farsight DNSDB Passive DNS',
|
||||||
|
'module-type': ['expansion', 'hover']
|
||||||
|
}
|
||||||
|
moduleconfig = ['apikey', 'server', 'limit']
|
||||||
|
|
||||||
server = 'https://api.dnsdb.info'
|
DEFAULT_LIMIT = 10
|
||||||
|
|
||||||
# TODO return a MISP object with the different attributes
|
|
||||||
|
class FarsightDnsdbParser():
|
||||||
|
def __init__(self, attribute):
|
||||||
|
self.attribute = attribute
|
||||||
|
self.misp_event = MISPEvent()
|
||||||
|
self.misp_event.add_attribute(**attribute)
|
||||||
|
self.passivedns_mapping = {
|
||||||
|
'bailiwick': {'type': 'text', 'object_relation': 'bailiwick'},
|
||||||
|
'count': {'type': 'counter', 'object_relation': 'count'},
|
||||||
|
'rdata': {'type': 'text', 'object_relation': 'rdata'},
|
||||||
|
'rrname': {'type': 'text', 'object_relation': 'rrname'},
|
||||||
|
'rrtype': {'type': 'text', 'object_relation': 'rrtype'},
|
||||||
|
'time_first': {'type': 'datetime', 'object_relation': 'time_first'},
|
||||||
|
'time_last': {'type': 'datetime', 'object_relation': 'time_last'},
|
||||||
|
'zone_time_first': {'type': 'datetime', 'object_relation': 'zone_time_first'},
|
||||||
|
'zone_time_last': {'type': 'datetime', 'object_relation': 'zone_time_last'}
|
||||||
|
}
|
||||||
|
self.type_to_feature = {
|
||||||
|
'domain': 'domain name',
|
||||||
|
'hostname': 'hostname',
|
||||||
|
'ip-src': 'IP address',
|
||||||
|
'ip-dst': 'IP address'
|
||||||
|
}
|
||||||
|
self.comment = 'Result from an %s lookup on DNSDB about the %s: %s'
|
||||||
|
|
||||||
|
def parse_passivedns_results(self, query_response):
|
||||||
|
default_fields = ('count', 'rrname', 'rrname')
|
||||||
|
optional_fields = (
|
||||||
|
'bailiwick',
|
||||||
|
'time_first',
|
||||||
|
'time_last',
|
||||||
|
'zone_time_first',
|
||||||
|
'zone_time_last'
|
||||||
|
)
|
||||||
|
for query_type, results in query_response.items():
|
||||||
|
comment = self.comment % (query_type, self.type_to_feature[self.attribute['type']], self.attribute['value'])
|
||||||
|
for result in results:
|
||||||
|
passivedns_object = MISPObject('passive-dns')
|
||||||
|
for feature in default_fields:
|
||||||
|
passivedns_object.add_attribute(**self._parse_attribute(comment, feature, result[feature]))
|
||||||
|
for feature in optional_fields:
|
||||||
|
if result.get(feature):
|
||||||
|
passivedns_object.add_attribute(**self._parse_attribute(comment, feature, result[feature]))
|
||||||
|
if isinstance(result['rdata'], list):
|
||||||
|
for rdata in result['rdata']:
|
||||||
|
passivedns_object.add_attribute(**self._parse_attribute(comment, 'rdata', rdata))
|
||||||
|
else:
|
||||||
|
passivedns_object.add_attribute(**self._parse_attribute(comment, 'rdata', result['rdata']))
|
||||||
|
passivedns_object.add_reference(self.attribute['uuid'], 'related-to')
|
||||||
|
self.misp_event.add_object(passivedns_object)
|
||||||
|
|
||||||
|
def get_results(self):
|
||||||
|
event = json.loads(self.misp_event.to_json())
|
||||||
|
results = {key: event[key] for key in ('Attribute', 'Object')}
|
||||||
|
return {'results': results}
|
||||||
|
|
||||||
|
def _parse_attribute(self, comment, feature, value):
|
||||||
|
attribute = {'value': value, 'comment': comment}
|
||||||
|
attribute.update(self.passivedns_mapping[feature])
|
||||||
|
return attribute
|
||||||
|
|
||||||
|
|
||||||
def handler(q=False):
|
def handler(q=False):
|
||||||
|
@ -19,56 +87,47 @@ def handler(q=False):
|
||||||
if not request.get('config') or not request['config'].get('apikey'):
|
if not request.get('config') or not request['config'].get('apikey'):
|
||||||
misperrors['error'] = 'Farsight DNSDB apikey is missing'
|
misperrors['error'] = 'Farsight DNSDB apikey is missing'
|
||||||
return misperrors
|
return misperrors
|
||||||
client = DnsdbClient(server, request['config']['apikey'])
|
if not request.get('attribute') or not check_input_attribute(request['attribute']):
|
||||||
if request.get('hostname'):
|
return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'}
|
||||||
res = lookup_name(client, request['hostname'])
|
attribute = request['attribute']
|
||||||
elif request.get('domain'):
|
if attribute['type'] not in mispattributes['input']:
|
||||||
res = lookup_name(client, request['domain'])
|
return {'error': 'Unsupported attributes type'}
|
||||||
elif request.get('ip-src'):
|
config = request['config']
|
||||||
res = lookup_ip(client, request['ip-src'])
|
args = {'apikey': config['apikey']}
|
||||||
elif request.get('ip-dst'):
|
for feature, default in zip(('server', 'limit'), (DEFAULT_DNSDB_SERVER, DEFAULT_LIMIT)):
|
||||||
res = lookup_ip(client, request['ip-dst'])
|
args[feature] = config[feature] if config.get(feature) else default
|
||||||
else:
|
client = DnsdbClient(**args)
|
||||||
misperrors['error'] = "Unsupported attributes type"
|
to_query = lookup_ip if attribute['type'] in ('ip-src', 'ip-dst') else lookup_name
|
||||||
return misperrors
|
response = to_query(client, attribute['value'])
|
||||||
|
if not response:
|
||||||
out = ''
|
return {'error': f"Empty results on Farsight DNSDB for the queries {attribute['type']}: {attribute['value']}."}
|
||||||
for v in set(res): # uniquify entries
|
parser = FarsightDnsdbParser(attribute)
|
||||||
out = out + "{} ".format(v)
|
parser.parse_passivedns_results(response)
|
||||||
r = {'results': [{'types': mispattributes['output'], 'values': out}]}
|
return parser.get_results()
|
||||||
return r
|
|
||||||
|
|
||||||
|
|
||||||
def lookup_name(client, name):
|
def lookup_name(client, name):
|
||||||
|
response = {}
|
||||||
try:
|
try:
|
||||||
res = client.query_rrset(name) # RRSET = entries in the left-hand side of the domain name related labels
|
res = client.query_rrset(name) # RRSET = entries in the left-hand side of the domain name related labels
|
||||||
for item in res:
|
response['rrset'] = list(res)
|
||||||
if item.get('rrtype') in ['A', 'AAAA', 'CNAME']:
|
|
||||||
for i in item.get('rdata'):
|
|
||||||
yield(i.rstrip('.'))
|
|
||||||
if item.get('rrtype') in ['SOA']:
|
|
||||||
for i in item.get('rdata'):
|
|
||||||
# grab email field and replace first dot by @ to convert to an email address
|
|
||||||
yield(i.split(' ')[1].rstrip('.').replace('.', '@', 1))
|
|
||||||
except QueryError:
|
except QueryError:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
try:
|
try:
|
||||||
res = client.query_rdata_name(name) # RDATA = entries on the right-hand side of the domain name related labels
|
res = client.query_rdata_name(name) # RDATA = entries on the right-hand side of the domain name related labels
|
||||||
for item in res:
|
response['rdata'] = list(res)
|
||||||
if item.get('rrtype') in ['A', 'AAAA', 'CNAME']:
|
|
||||||
yield(item.get('rrname').rstrip('.'))
|
|
||||||
except QueryError:
|
except QueryError:
|
||||||
pass
|
pass
|
||||||
|
return response
|
||||||
|
|
||||||
|
|
||||||
def lookup_ip(client, ip):
|
def lookup_ip(client, ip):
|
||||||
try:
|
try:
|
||||||
res = client.query_rdata_ip(ip)
|
res = client.query_rdata_ip(ip)
|
||||||
for item in res:
|
response = {'rdata': list(res)}
|
||||||
yield(item['rrname'].rstrip('.'))
|
|
||||||
except QueryError:
|
except QueryError:
|
||||||
pass
|
response = {}
|
||||||
|
return response
|
||||||
|
|
||||||
|
|
||||||
def introspection():
|
def introspection():
|
||||||
|
|
|
@ -39,7 +39,7 @@ class TruSTARParser:
|
||||||
|
|
||||||
# Relevant fields from each TruSTAR endpoint
|
# Relevant fields from each TruSTAR endpoint
|
||||||
SUMMARY_FIELDS = ["severityLevel", "source", "score", "attributes"]
|
SUMMARY_FIELDS = ["severityLevel", "source", "score", "attributes"]
|
||||||
METADATA_FIELDS = ["sightings", "first_seen", "last_seen", "tags"]
|
METADATA_FIELDS = ["sightings", "firstSeen", "lastSeen", "tags"]
|
||||||
|
|
||||||
REPORT_BASE_URL = "https://station.trustar.co/constellation/reports/{}"
|
REPORT_BASE_URL = "https://station.trustar.co/constellation/reports/{}"
|
||||||
|
|
||||||
|
|
|
@ -221,7 +221,7 @@ class TestExpansions(unittest.TestCase):
|
||||||
try:
|
try:
|
||||||
self.assertIn(result, self.get_values(response))
|
self.assertIn(result, self.get_values(response))
|
||||||
except Exception:
|
except Exception:
|
||||||
self.assertTrue(self.get_errors(response).startwith('Something went wrong'))
|
self.assertTrue(self.get_errors(response).startswith('Something went wrong'))
|
||||||
else:
|
else:
|
||||||
query = {"module": module_name, "ip-src": "8.8.8.8"}
|
query = {"module": module_name, "ip-src": "8.8.8.8"}
|
||||||
response = self.misp_modules_post(query)
|
response = self.misp_modules_post(query)
|
||||||
|
@ -285,7 +285,7 @@ class TestExpansions(unittest.TestCase):
|
||||||
encoded = b64encode(f.read()).decode()
|
encoded = b64encode(f.read()).decode()
|
||||||
query = {"module": "ocr_enrich", "attachment": filename, "data": encoded}
|
query = {"module": "ocr_enrich", "attachment": filename, "data": encoded}
|
||||||
response = self.misp_modules_post(query)
|
response = self.misp_modules_post(query)
|
||||||
self.assertEqual(self.get_values(response), 'Threat Sharing')
|
self.assertEqual(self.get_values(response).strip('\n'), 'Threat Sharing')
|
||||||
|
|
||||||
def test_ods(self):
|
def test_ods(self):
|
||||||
filename = 'test.ods'
|
filename = 'test.ods'
|
||||||
|
|
Loading…
Reference in New Issue