MISP
/
misp-modules
mirror of https://github.com/MISP/misp-modules
2
0
Fork 0
Code Issues Releases Wiki Activity

fix: Fixed pep8 in the new module and related libraries

pull/363/head
chrisr3d 2020-01-09 16:01:18 +01:00
parent 7722e2cb93
commit 70b3079aa3
No known key found for this signature in database
GPG Key ID: 6BBED1B63A6D639F
7 changed files with 509 additions and 513 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
misp_modules/lib/vt_graph_parser/errors.py
View File

@ -5,16 +5,16 @@ This module provides custom errors for data importers.
class GraphImportError(Exception): class GraphImportError(Exception):
pass pass
class InvalidFileFormatError(Exception): class InvalidFileFormatError(Exception):
pass pass
class MispEventNotFoundError(Exception): class MispEventNotFoundError(Exception):
pass pass
class ServerError(Exception): class ServerError(Exception):
pass pass

97
misp_modules/lib/vt_graph_parser/helpers/parsers.py
View File

@ -26,64 +26,63 @@ VIRUSTOTAL_GRAPH_LINK_PREFIX = "https://www.virustotal.com/graph/"
def _parse_data(attributes, objects): def _parse_data(attributes, objects):
"""Parse MISP event attributes and objects data. """Parse MISP event attributes and objects data.
Args: Args:
attributes (dict): dictionary which contains the MISP event attributes data. attributes (dict): dictionary which contains the MISP event attributes data.
objects (dict): dictionary which contains the MISP event objects data. objects (dict): dictionary which contains the MISP event objects data.
Returns: Returns:
([MispAttribute], str): MISP attributes and VTGraph link if exists. ([MispAttribute], str): MISP attributes and VTGraph link if exists.
Link defaults to "". Link defaults to "".
""" """
attributes_data = [] attributes_data = []
vt_graph_link = "" vt_graph_link = ""
# Get simple MISP event attributes. # Get simple MISP event attributes.
attributes_data += ( attributes_data += (
[attr for attr in attributes [attr for attr in attributes
if attr.get("type") in MISP_INPUT_ATTR]) if attr.get("type") in MISP_INPUT_ATTR])
# Get attributes from MISP objects too. # Get attributes from MISP objects too.
if objects: if objects:
for object_ in objects: for object_ in objects:
object_attrs = object_.get("Attribute", []) object_attrs = object_.get("Attribute", [])
attributes_data += ( attributes_data += (
[attr for attr in object_attrs [attr for attr in object_attrs
if attr.get("type") in MISP_INPUT_ATTR]) if attr.get("type") in MISP_INPUT_ATTR])
# Check if there is any VirusTotal Graph computed in MISP event. # Check if there is any VirusTotal Graph computed in MISP event.
vt_graph_links = ( vt_graph_links = (
attr for attr in attributes if attr.get("type") == "link" attr for attr in attributes if attr.get("type") == "link"
and attr.get("value", "").startswith(VIRUSTOTAL_GRAPH_LINK_PREFIX)) and attr.get("value", "").startswith(VIRUSTOTAL_GRAPH_LINK_PREFIX))
# MISP could have more than one VirusTotal Graph, so we will take # MISP could have more than one VirusTotal Graph, so we will take
# the last one. # the last one.
current_id = 0 # MISP attribute id is the number of the attribute. current_id = 0 # MISP attribute id is the number of the attribute.
vt_graph_link = "" vt_graph_link = ""
for link in vt_graph_links: for link in vt_graph_links:
if int(link.get("id")) > current_id: if int(link.get("id")) > current_id:
current_id = int(link.get("id")) current_id = int(link.get("id"))
vt_graph_link = link.get("value") vt_graph_link = link.get("value")
attributes = [ attributes = [
MispAttribute(data["type"], data["category"], data["value"]) MispAttribute(data["type"], data["category"], data["value"])
for data in attributes_data] for data in attributes_data]
return (attributes, return (attributes,
vt_graph_link.replace(VIRUSTOTAL_GRAPH_LINK_PREFIX, "")) vt_graph_link.replace(VIRUSTOTAL_GRAPH_LINK_PREFIX, ""))
def parse_pymisp_response(payload): def parse_pymisp_response(payload):
"""Get event attributes and VirusTotal Graph id from pymisp response. """Get event attributes and VirusTotal Graph id from pymisp response.
Args: Args:
payload (dict): dictionary which contains pymisp response. payload (dict): dictionary which contains pymisp response.
Returns:
([MispAttribute], str): MISP attributes and VTGraph link if exists.
Link defaults to "".
"""
event_attrs = payload.get("Attribute", [])
objects = payload.get("Object")
return _parse_data(event_attrs, objects)
Returns:
([MispAttribute], str): MISP attributes and VTGraph link if exists.
Link defaults to "".
"""
event_attrs = payload.get("Attribute", [])
objects = payload.get("Object")
return _parse_data(event_attrs, objects)

442
misp_modules/lib/vt_graph_parser/helpers/rules.py
View File

@ -15,290 +15,290 @@ import abc
class MispEventRule(object): class MispEventRule(object):
"""Rules for MISP event nodes connection object wrapper.""" """Rules for MISP event nodes connection object wrapper."""
def __init__(self, last_rule=None, node=None): def __init__(self, last_rule=None, node=None):
"""Create a MispEventRule instance. """Create a MispEventRule instance.
MispEventRule is a collection of rules that can infer the relationships MispEventRule is a collection of rules that can infer the relationships
between nodes from MISP events. between nodes from MISP events.
Args: Args:
last_rule (MispEventRule): previous rule. last_rule (MispEventRule): previous rule.
node (Node): actual node. node (Node): actual node.
""" """
self.last_rule = last_rule self.last_rule = last_rule
self.node = node self.node = node
self.relation_event = { self.relation_event = {
"ip_address": self.__ip_transition, "ip_address": self.__ip_transition,
"url": self.__url_transition, "url": self.__url_transition,
"domain": self.__domain_transition, "domain": self.__domain_transition,
"file": self.__file_transition "file": self.__file_transition
} }
def get_last_different_rule(self): def get_last_different_rule(self):
"""Search the last rule whose event was different from actual. """Search the last rule whose event was different from actual.
Returns: Returns:
MispEventRule: the last different rule. MispEventRule: the last different rule.
""" """
if not isinstance(self, self.last_rule.__class__): if not isinstance(self, self.last_rule.__class__):
return self.last_rule return self.last_rule
else: else:
return self.last_rule.get_last_different_rule() return self.last_rule.get_last_different_rule()
def resolve_relation(self, graph, node, misp_category): def resolve_relation(self, graph, node, misp_category):
"""Try to infer a relationship between two nodes. """Try to infer a relationship between two nodes.
This method is based on a non-deterministic finite automaton for This method is based on a non-deterministic finite automaton for
this reason the future rule only depends on the actual rule and the input this reason the future rule only depends on the actual rule and the input
node. node.
For example if the actual rule is a MISPEventDomainRule and the given node For example if the actual rule is a MISPEventDomainRule and the given node
is an ip_address node, the connection type between them will be is an ip_address node, the connection type between them will be
`resolutions` and the this rule will transit to MISPEventIPRule. `resolutions` and the this rule will transit to MISPEventIPRule.
Args: Args:
graph (VTGraph): graph to be computed. graph (VTGraph): graph to be computed.
node (Node): the node to be linked. node (Node): the node to be linked.
misp_category: (str): MISP category of the given node. misp_category: (str): MISP category of the given node.
Returns: Returns:
MispEventRule: the transited rule. MispEventRule: the transited rule.
""" """
if node.node_type in self.relation_event: if node.node_type in self.relation_event:
return self.relation_event[node.node_type](graph, node, misp_category) return self.relation_event[node.node_type](graph, node, misp_category)
else: else:
return self.manual_link(graph, node) return self.manual_link(graph, node)
def manual_link(self, graph, node): def manual_link(self, graph, node):
"""Creates a manual link between self.node and the given node. """Creates a manual link between self.node and the given node.
We accept MISP types that VirusTotal does not know how to link, so we create We accept MISP types that VirusTotal does not know how to link, so we create
a end to end relationship instead of create an unknown relationship node. a end to end relationship instead of create an unknown relationship node.
Args: Args:
graph (VTGraph): graph to be computed. graph (VTGraph): graph to be computed.
node (Node): the node to be linked. node (Node): the node to be linked.
Returns: Returns:
MispEventRule: the transited rule. MispEventRule: the transited rule.
""" """
graph.add_link(self.node.node_id, node.node_id, "manual") graph.add_link(self.node.node_id, node.node_id, "manual")
return self return self
@abc.abstractmethod @abc.abstractmethod
def __file_transition(self, graph, node, misp_category): def __file_transition(self, graph, node, misp_category):
"""Make a new transition due to file attribute event. """Make a new transition due to file attribute event.
Args: Args:
graph (VTGraph): graph to be computed. graph (VTGraph): graph to be computed.
node (Node): the node to be linked. node (Node): the node to be linked.
misp_category: (str): MISP category of the given node. misp_category: (str): MISP category of the given node.
Returns: Returns:
MispEventRule: the transited rule. MispEventRule: the transited rule.
""" """
pass pass
@abc.abstractmethod @abc.abstractmethod
def __ip_transition(self, graph, node, misp_category): def __ip_transition(self, graph, node, misp_category):
"""Make a new transition due to ip attribute event. """Make a new transition due to ip attribute event.
Args: Args:
graph (VTGraph): graph to be computed. graph (VTGraph): graph to be computed.
node (Node): the node to be linked. node (Node): the node to be linked.
misp_category: (str): MISP category of the given node. misp_category: (str): MISP category of the given node.
Returns: Returns:
MispEventRule: the transited rule. MispEventRule: the transited rule.
""" """
pass pass
@abc.abstractmethod @abc.abstractmethod
def __url_transition(self, graph, node, misp_category): def __url_transition(self, graph, node, misp_category):
"""Make a new transition due to url attribute event. """Make a new transition due to url attribute event.
Args: Args:
graph (VTGraph): graph to be computed. graph (VTGraph): graph to be computed.
node (Node): the node to be linked. node (Node): the node to be linked.
misp_category: (str): MISP category of the given node. misp_category: (str): MISP category of the given node.
Returns: Returns:
MispEventRule: the transited rule. MispEventRule: the transited rule.
""" """
pass pass
@abc.abstractmethod @abc.abstractmethod
def __domain_transition(self, graph, node, misp_category): def __domain_transition(self, graph, node, misp_category):
"""Make a new transition due to domain attribute event. """Make a new transition due to domain attribute event.
Args: Args:
graph (VTGraph): graph to be computed. graph (VTGraph): graph to be computed.
node (Node): the node to be linked. node (Node): the node to be linked.
misp_category: (str): MISP category of the given node. misp_category: (str): MISP category of the given node.
Returns: Returns:
MispEventRule: the transited rule. MispEventRule: the transited rule.
""" """
pass pass
class MispEventURLRule(MispEventRule): class MispEventURLRule(MispEventRule):
"""Rule for URL event.""" """Rule for URL event."""
def __init__(self, last_rule=None, node=None): def __init__(self, last_rule=None, node=None):
super(MispEventURLRule, self).__init__(last_rule, node) super(MispEventURLRule, self).__init__(last_rule, node)
self.relation_event = { self.relation_event = {
"ip_address": self.__ip_transition, "ip_address": self.__ip_transition,
"url": self.__url_transition, "url": self.__url_transition,
"domain": self.__domain_transition, "domain": self.__domain_transition,
"file": self.__file_transition "file": self.__file_transition
} }
def __file_transition(self, graph, node, misp_category): def __file_transition(self, graph, node, misp_category):
graph.add_link(self.node.node_id, node.node_id, "downloaded_files") graph.add_link(self.node.node_id, node.node_id, "downloaded_files")
return MispEventFileRule(self, node) return MispEventFileRule(self, node)
def __ip_transition(self, graph, node, misp_category): def __ip_transition(self, graph, node, misp_category):
graph.add_link(self.node.node_id, node.node_id, "contacted_ips") graph.add_link(self.node.node_id, node.node_id, "contacted_ips")
return MispEventIPRule(self, node) return MispEventIPRule(self, node)
def __url_transition(self, graph, node, misp_category): def __url_transition(self, graph, node, misp_category):
suitable_rule = self.get_last_different_rule() suitable_rule = self.get_last_different_rule()
if not isinstance(suitable_rule, MispEventInitialRule): if not isinstance(suitable_rule, MispEventInitialRule):
return suitable_rule.resolve_relation(graph, node, misp_category) return suitable_rule.resolve_relation(graph, node, misp_category)
else: else:
return MispEventURLRule(self, node) return MispEventURLRule(self, node)
def __domain_transition(self, graph, node, misp_category): def __domain_transition(self, graph, node, misp_category):
graph.add_link(self.node.node_id, node.node_id, "contacted_domains") graph.add_link(self.node.node_id, node.node_id, "contacted_domains")
return MispEventDomainRule(self, node) return MispEventDomainRule(self, node)
class MispEventIPRule(MispEventRule): class MispEventIPRule(MispEventRule):
"""Rule for IP event.""" """Rule for IP event."""
def __init__(self, last_rule=None, node=None): def __init__(self, last_rule=None, node=None):
super(MispEventIPRule, self).__init__(last_rule, node) super(MispEventIPRule, self).__init__(last_rule, node)
self.relation_event = { self.relation_event = {
"ip_address": self.__ip_transition, "ip_address": self.__ip_transition,
"url": self.__url_transition, "url": self.__url_transition,
"domain": self.__domain_transition, "domain": self.__domain_transition,
"file": self.__file_transition "file": self.__file_transition
} }
def __file_transition(self, graph, node, misp_category): def __file_transition(self, graph, node, misp_category):
connection_type = "communicating_files" connection_type = "communicating_files"
if misp_category == "Artifacts dropped": if misp_category == "Artifacts dropped":
connection_type = "downloaded_files" connection_type = "downloaded_files"
graph.add_link(self.node.node_id, node.node_id, connection_type) graph.add_link(self.node.node_id, node.node_id, connection_type)
return MispEventFileRule(self, node) return MispEventFileRule(self, node)
def __ip_transition(self, graph, node, misp_category): def __ip_transition(self, graph, node, misp_category):
suitable_rule = self.get_last_different_rule() suitable_rule = self.get_last_different_rule()
if not isinstance(suitable_rule, MispEventInitialRule): if not isinstance(suitable_rule, MispEventInitialRule):
return suitable_rule.resolve_relation(graph, node, misp_category) return suitable_rule.resolve_relation(graph, node, misp_category)
else: else:
return MispEventIPRule(self, node) return MispEventIPRule(self, node)
def __url_transition(self, graph, node, misp_category): def __url_transition(self, graph, node, misp_category):
graph.add_link(self.node.node_id, node.node_id, "urls") graph.add_link(self.node.node_id, node.node_id, "urls")
return MispEventURLRule(self, node) return MispEventURLRule(self, node)
def __domain_transition(self, graph, node, misp_category): def __domain_transition(self, graph, node, misp_category):
graph.add_link(self.node.node_id, node.node_id, "resolutions") graph.add_link(self.node.node_id, node.node_id, "resolutions")
return MispEventDomainRule(self, node) return MispEventDomainRule(self, node)
class MispEventDomainRule(MispEventRule): class MispEventDomainRule(MispEventRule):
"""Rule for domain event.""" """Rule for domain event."""
def __init__(self, last_rule=None, node=None): def __init__(self, last_rule=None, node=None):
super(MispEventDomainRule, self).__init__(last_rule, node) super(MispEventDomainRule, self).__init__(last_rule, node)
self.relation_event = { self.relation_event = {
"ip_address": self.__ip_transition, "ip_address": self.__ip_transition,
"url": self.__url_transition, "url": self.__url_transition,
"domain": self.__domain_transition, "domain": self.__domain_transition,
"file": self.__file_transition "file": self.__file_transition
} }
def __file_transition(self, graph, node, misp_category): def __file_transition(self, graph, node, misp_category):
connection_type = "communicating_files" connection_type = "communicating_files"
if misp_category == "Artifacts dropped": if misp_category == "Artifacts dropped":
connection_type = "downloaded_files" connection_type = "downloaded_files"
graph.add_link(self.node.node_id, node.node_id, connection_type) graph.add_link(self.node.node_id, node.node_id, connection_type)
return MispEventFileRule(self, node) return MispEventFileRule(self, node)
def __ip_transition(self, graph, node, misp_category): def __ip_transition(self, graph, node, misp_category):
graph.add_link(self.node.node_id, node.node_id, "resolutions") graph.add_link(self.node.node_id, node.node_id, "resolutions")
return MispEventIPRule(self, node) return MispEventIPRule(self, node)
def __url_transition(self, graph, node, misp_category): def __url_transition(self, graph, node, misp_category):
graph.add_link(self.node.node_id, node.node_id, "urls") graph.add_link(self.node.node_id, node.node_id, "urls")
return MispEventURLRule(self, node) return MispEventURLRule(self, node)
def __domain_transition(self, graph, node, misp_category): def __domain_transition(self, graph, node, misp_category):
suitable_rule = self.get_last_different_rule() suitable_rule = self.get_last_different_rule()
if not isinstance(suitable_rule, MispEventInitialRule): if not isinstance(suitable_rule, MispEventInitialRule):
return suitable_rule.resolve_relation(graph, node, misp_category) return suitable_rule.resolve_relation(graph, node, misp_category)
else: else:
graph.add_link(self.node.node_id, node.node_id, "siblings") graph.add_link(self.node.node_id, node.node_id, "siblings")
return MispEventDomainRule(self, node) return MispEventDomainRule(self, node)
class MispEventFileRule(MispEventRule): class MispEventFileRule(MispEventRule):
"""Rule for File event.""" """Rule for File event."""
def __init__(self, last_rule=None, node=None): def __init__(self, last_rule=None, node=None):
super(MispEventFileRule, self).__init__(last_rule, node) super(MispEventFileRule, self).__init__(last_rule, node)
self.relation_event = { self.relation_event = {
"ip_address": self.__ip_transition, "ip_address": self.__ip_transition,
"url": self.__url_transition, "url": self.__url_transition,
"domain": self.__domain_transition, "domain": self.__domain_transition,
"file": self.__file_transition "file": self.__file_transition
} }
def __file_transition(self, graph, node, misp_category): def __file_transition(self, graph, node, misp_category):
suitable_rule = self.get_last_different_rule() suitable_rule = self.get_last_different_rule()
if not isinstance(suitable_rule, MispEventInitialRule): if not isinstance(suitable_rule, MispEventInitialRule):
return suitable_rule.resolve_relation(graph, node, misp_category) return suitable_rule.resolve_relation(graph, node, misp_category)
else: else:
return MispEventFileRule(self, node) return MispEventFileRule(self, node)
def __ip_transition(self, graph, node, misp_category): def __ip_transition(self, graph, node, misp_category):
graph.add_link(self.node.node_id, node.node_id, "contacted_ips") graph.add_link(self.node.node_id, node.node_id, "contacted_ips")
return MispEventIPRule(self, node) return MispEventIPRule(self, node)
def __url_transition(self, graph, node, misp_category): def __url_transition(self, graph, node, misp_category):
graph.add_link(self.node.node_id, node.node_id, "contacted_urls") graph.add_link(self.node.node_id, node.node_id, "contacted_urls")
return MispEventURLRule(self, node) return MispEventURLRule(self, node)
def __domain_transition(self, graph, node, misp_category): def __domain_transition(self, graph, node, misp_category):
graph.add_link(self.node.node_id, node.node_id, "contacted_domains") graph.add_link(self.node.node_id, node.node_id, "contacted_domains")
return MispEventDomainRule(self, node) return MispEventDomainRule(self, node)
class MispEventInitialRule(MispEventRule): class MispEventInitialRule(MispEventRule):
"""Initial rule.""" """Initial rule."""
def __init__(self, last_rule=None, node=None): def __init__(self, last_rule=None, node=None):
super(MispEventInitialRule, self).__init__(last_rule, node) super(MispEventInitialRule, self).__init__(last_rule, node)
self.relation_event = { self.relation_event = {
"ip_address": self.__ip_transition, "ip_address": self.__ip_transition,
"url": self.__url_transition, "url": self.__url_transition,
"domain": self.__domain_transition, "domain": self.__domain_transition,
"file": self.__file_transition "file": self.__file_transition
} }
def __file_transition(self, graph, node, misp_category): def __file_transition(self, graph, node, misp_category):
return MispEventFileRule(self, node) return MispEventFileRule(self, node)
def __ip_transition(self, graph, node, misp_category): def __ip_transition(self, graph, node, misp_category):
return MispEventIPRule(self, node) return MispEventIPRule(self, node)
def __url_transition(self, graph, node, misp_category): def __url_transition(self, graph, node, misp_category):
return MispEventURLRule(self, node) return MispEventURLRule(self, node)
def __domain_transition(self, graph, node, misp_category): def __domain_transition(self, graph, node, misp_category):
return MispEventDomainRule(self, node) return MispEventDomainRule(self, node)

93
misp_modules/lib/vt_graph_parser/helpers/wrappers.py
View File

@ -5,55 +5,54 @@ This module provides a Python object wrapper for MISP objects.
class MispAttribute(object): class MispAttribute(object):
"""Python object wrapper for MISP attribute. """Python object wrapper for MISP attribute.
Attributes: Attributes:
type (str): VirusTotal node type. type (str): VirusTotal node type.
category (str): MISP attribute category. category (str): MISP attribute category.
value (str): node id. value (str): node id.
label (str): node name. label (str): node name.
misp_type (str): MISP node type. misp_type (str): MISP node type.
"""
MISP_TYPES_REFERENCE = {
"hostname": "domain",
"domain": "domain",
"ip-src": "ip_address",
"ip-dst": "ip_address",
"url": "url",
"filename|X": "file",
"filename": "file",
"md5": "file",
"sha1": "file",
"sha256": "file",
"target-user": "victim",
"target-email": "email"
}
def __init__(self, misp_type, category, value, label=""):
"""Constructor for a MispAttribute.
Args:
misp_type (str): MISP type attribute.
category (str): MISP category attribute.
value (str): attribute value.
label (str): attribute label.
""" """
if misp_type.startswith("filename|"):
label, value = value.split("|")
misp_type = "filename|X"
if misp_type == "filename":
label = value
self.type = self.MISP_TYPES_REFERENCE.get(misp_type) MISP_TYPES_REFERENCE = {
self.category = category "hostname": "domain",
self.value = value "domain": "domain",
self.label = label "ip-src": "ip_address",
self.misp_type = misp_type "ip-dst": "ip_address",
"url": "url",
"filename|X": "file",
"filename": "file",
"md5": "file",
"sha1": "file",
"sha256": "file",
"target-user": "victim",
"target-email": "email"
}
def __eq__(self, other): def __init__(self, misp_type, category, value, label=""):
return (isinstance(other, self.__class__) and self.value == other.value and """Constructor for a MispAttribute.
self.type == other.type)
def __repr__(self): Args:
return 'MispAttribute("{type}", "{category}", "{value}")'.format(type=self.type, category=self.category, value=self.value) misp_type (str): MISP type attribute.
category (str): MISP category attribute.
value (str): attribute value.
label (str): attribute label.
"""
if misp_type.startswith("filename|"):
label, value = value.split("|")
misp_type = "filename|X"
if misp_type == "filename":
label = value
self.type = self.MISP_TYPES_REFERENCE.get(misp_type)
self.category = category
self.value = value
self.label = label
self.misp_type = misp_type
def __eq__(self, other):
return (isinstance(other, self.__class__) and self.value == other.value and self.type == other.type)
def __repr__(self):
return 'MispAttribute("{type}", "{category}", "{value}")'.format(type=self.type, category=self.category, value=self.value)

160
misp_modules/lib/vt_graph_parser/importers/base.py
View File

@ -9,90 +9,90 @@ from lib.vt_graph_parser.helpers.rules import MispEventInitialRule
def import_misp_graph( def import_misp_graph(
misp_attributes, graph_id, vt_api_key, fetch_information, name, misp_attributes, graph_id, vt_api_key, fetch_information, name,
private, fetch_vt_enterprise, user_editors, user_viewers, group_editors, private, fetch_vt_enterprise, user_editors, user_viewers, group_editors,
group_viewers, use_vt_to_connect_the_graph, max_api_quotas, group_viewers, use_vt_to_connect_the_graph, max_api_quotas,
max_search_depth): max_search_depth):
"""Import VirusTotal Graph from MISP. """Import VirusTotal Graph from MISP.
Args: Args:
misp_attributes ([MispAttribute]): list with the MISP attributes which misp_attributes ([MispAttribute]): list with the MISP attributes which
will be added to the returned graph. will be added to the returned graph.
graph_id: if supplied, the graph will be loaded instead of compute it again. graph_id: if supplied, the graph will be loaded instead of compute it again.
vt_api_key (str): VT API Key. vt_api_key (str): VT API Key.
fetch_information (bool): whether the script will fetch fetch_information (bool): whether the script will fetch
information for added nodes in VT. Defaults to True. information for added nodes in VT. Defaults to True.
name (str): graph title. Defaults to "". name (str): graph title. Defaults to "".
private (bool): True for private graphs. You need to have private (bool): True for private graphs. You need to have
Private Graph premium features enabled in your subscription. Defaults Private Graph premium features enabled in your subscription. Defaults
to False. to False.
fetch_vt_enterprise (bool, optional): if True, the graph will search any fetch_vt_enterprise (bool, optional): if True, the graph will search any
available information using VirusTotal Intelligence for the node if there available information using VirusTotal Intelligence for the node if there
is no normal information for it. Defaults to False. is no normal information for it. Defaults to False.
user_editors ([str]): usernames that can edit the graph. user_editors ([str]): usernames that can edit the graph.
Defaults to None. Defaults to None.
user_viewers ([str]): usernames that can view the graph. user_viewers ([str]): usernames that can view the graph.
Defaults to None. Defaults to None.
group_editors ([str]): groups that can edit the graph. group_editors ([str]): groups that can edit the graph.
Defaults to None. Defaults to None.
group_viewers ([str]): groups that can view the graph. group_viewers ([str]): groups that can view the graph.
Defaults to None. Defaults to None.
use_vt_to_connect_the_graph (bool): if True, graph nodes will use_vt_to_connect_the_graph (bool): if True, graph nodes will
be linked using VirusTotal API. Otherwise, the links will be generated be linked using VirusTotal API. Otherwise, the links will be generated
using production rules based on MISP attributes order. Defaults to using production rules based on MISP attributes order. Defaults to
False. False.
max_api_quotas (int): maximum number of api quotas that could max_api_quotas (int): maximum number of api quotas that could
be consumed to resolve graph using VirusTotal API. Defaults to 20000. be consumed to resolve graph using VirusTotal API. Defaults to 20000.
max_search_depth (int, optional): max search depth to explore max_search_depth (int, optional): max search depth to explore
relationship between nodes when use_vt_to_connect_the_graph is True. relationship between nodes when use_vt_to_connect_the_graph is True.
Defaults to 3. Defaults to 3.
If use_vt_to_connect_the_graph is True, it will take some time to compute If use_vt_to_connect_the_graph is True, it will take some time to compute
graph. graph.
Returns: Returns:
vt_graph_api.graph.VTGraph: the imported graph. vt_graph_api.graph.VTGraph: the imported graph.
""" """
rule = MispEventInitialRule() rule = MispEventInitialRule()
# Check if the event has been already computed in VirusTotal Graph. Otherwise # Check if the event has been already computed in VirusTotal Graph. Otherwise
# a new graph will be created. # a new graph will be created.
if not graph_id: if not graph_id:
graph = vt_graph_api.VTGraph( graph = vt_graph_api.VTGraph(
api_key=vt_api_key, name=name, private=private, api_key=vt_api_key, name=name, private=private,
user_editors=user_editors, user_viewers=user_viewers, user_editors=user_editors, user_viewers=user_viewers,
group_editors=group_editors, group_viewers=group_viewers) group_editors=group_editors, group_viewers=group_viewers)
else:
graph = vt_graph_api.VTGraph.load_graph(graph_id, vt_api_key)
attributes_to_add = [attr for attr in misp_attributes
if not graph.has_node(attr.value)]
total_expandable_attrs = max(sum(
1 for attr in attributes_to_add
if attr.type in vt_graph_api.Node.SUPPORTED_NODE_TYPES),
1)
max_quotas_per_search = max(int(max_api_quotas / total_expandable_attrs), 1)
previous_node_id = ""
for attr in attributes_to_add:
# Add the current attr as node to the graph.
added_node = graph.add_node(
attr.value, attr.type, fetch_information, fetch_vt_enterprise,
attr.label)
# If use_vt_to_connect_the_grap is True the nodes will be connected using
# VT API.
if use_vt_to_connect_the_graph:
if (attr.type not in vt_graph_api.Node.SUPPORTED_NODE_TYPES and
previous_node_id):
graph.add_link(previous_node_id, attr.value, "manual")
else:
graph.connect_with_graph(
attr.value, max_quotas_per_search, max_search_depth,
fetch_info_collected_nodes=fetch_information)
else: else:
rule = rule.resolve_relation(graph, added_node, attr.category) graph = vt_graph_api.VTGraph.load_graph(graph_id, vt_api_key)
return graph attributes_to_add = [attr for attr in misp_attributes
if not graph.has_node(attr.value)]
total_expandable_attrs = max(sum(
1 for attr in attributes_to_add
if attr.type in vt_graph_api.Node.SUPPORTED_NODE_TYPES),
1)
max_quotas_per_search = max(
int(max_api_quotas / total_expandable_attrs), 1)
previous_node_id = ""
for attr in attributes_to_add:
# Add the current attr as node to the graph.
added_node = graph.add_node(
attr.value, attr.type, fetch_information, fetch_vt_enterprise,
attr.label)
# If use_vt_to_connect_the_grap is True the nodes will be connected using
# VT API.
if use_vt_to_connect_the_graph:
if (attr.type not in vt_graph_api.Node.SUPPORTED_NODE_TYPES and previous_node_id):
graph.add_link(previous_node_id, attr.value, "manual")
else:
graph.connect_with_graph(
attr.value, max_quotas_per_search, max_search_depth,
fetch_info_collected_nodes=fetch_information)
else:
rule = rule.resolve_relation(graph, added_node, attr.category)
return graph

116
misp_modules/lib/vt_graph_parser/importers/pymisp_response.py
View File

@ -5,71 +5,69 @@ response payload giving by MISP API directly.
""" """
import json
from lib.vt_graph_parser import errors
from lib.vt_graph_parser.helpers.parsers import parse_pymisp_response from lib.vt_graph_parser.helpers.parsers import parse_pymisp_response
from lib.vt_graph_parser.importers.base import import_misp_graph from lib.vt_graph_parser.importers.base import import_misp_graph
def from_pymisp_response( def from_pymisp_response(
payload, vt_api_key, fetch_information=True, payload, vt_api_key, fetch_information=True,
private=False, fetch_vt_enterprise=False, user_editors=None, private=False, fetch_vt_enterprise=False, user_editors=None,
user_viewers=None, group_editors=None, group_viewers=None, user_viewers=None, group_editors=None, group_viewers=None,
use_vt_to_connect_the_graph=False, max_api_quotas=1000, use_vt_to_connect_the_graph=False, max_api_quotas=1000,
max_search_depth=3, expand_node_one_level=False): max_search_depth=3, expand_node_one_level=False):
"""Import VirusTotal Graph from MISP JSON file. """Import VirusTotal Graph from MISP JSON file.
Args: Args:
payload (dict): dictionary which contains the request payload. payload (dict): dictionary which contains the request payload.
vt_api_key (str): VT API Key. vt_api_key (str): VT API Key.
fetch_information (bool, optional): whether the script will fetch fetch_information (bool, optional): whether the script will fetch
information for added nodes in VT. Defaults to True. information for added nodes in VT. Defaults to True.
name (str, optional): graph title. Defaults to "". name (str, optional): graph title. Defaults to "".
private (bool, optional): True for private graphs. You need to have private (bool, optional): True for private graphs. You need to have
Private Graph premium features enabled in your subscription. Defaults Private Graph premium features enabled in your subscription. Defaults
to False. to False.
fetch_vt_enterprise (bool, optional): if True, the graph will search any fetch_vt_enterprise (bool, optional): if True, the graph will search any
available information using VirusTotal Intelligence for the node if there available information using VirusTotal Intelligence for the node if there
is no normal information for it. Defaults to False. is no normal information for it. Defaults to False.
user_editors ([str], optional): usernames that can edit the graph. user_editors ([str], optional): usernames that can edit the graph.
Defaults to None. Defaults to None.
user_viewers ([str], optional): usernames that can view the graph. user_viewers ([str], optional): usernames that can view the graph.
Defaults to None. Defaults to None.
group_editors ([str], optional): groups that can edit the graph. group_editors ([str], optional): groups that can edit the graph.
Defaults to None. Defaults to None.
group_viewers ([str], optional): groups that can view the graph. group_viewers ([str], optional): groups that can view the graph.
Defaults to None. Defaults to None.
use_vt_to_connect_the_graph (bool, optional): if True, graph nodes will use_vt_to_connect_the_graph (bool, optional): if True, graph nodes will
be linked using VirusTotal API. Otherwise, the links will be generated be linked using VirusTotal API. Otherwise, the links will be generated
using production rules based on MISP attributes order. Defaults to using production rules based on MISP attributes order. Defaults to
False. False.
max_api_quotas (int, optional): maximum number of api quotas that could max_api_quotas (int, optional): maximum number of api quotas that could
be consumed to resolve graph using VirusTotal API. Defaults to 20000. be consumed to resolve graph using VirusTotal API. Defaults to 20000.
max_search_depth (int, optional): max search depth to explore max_search_depth (int, optional): max search depth to explore
relationship between nodes when use_vt_to_connect_the_graph is True. relationship between nodes when use_vt_to_connect_the_graph is True.
Defaults to 3. Defaults to 3.
expand_one_level (bool, optional): expand entire graph one level. expand_one_level (bool, optional): expand entire graph one level.
Defaults to False. Defaults to False.
If use_vt_to_connect_the_graph is True, it will take some time to compute If use_vt_to_connect_the_graph is True, it will take some time to compute
graph. graph.
Raises: Raises:
LoaderError: if JSON file is invalid. LoaderError: if JSON file is invalid.
Returns: Returns:
[vt_graph_api.graph.VTGraph: the imported graph]. [vt_graph_api.graph.VTGraph: the imported graph].
""" """
graphs = [] graphs = []
for event_payload in payload['data']: for event_payload in payload['data']:
misp_attrs, graph_id = parse_pymisp_response(event_payload) misp_attrs, graph_id = parse_pymisp_response(event_payload)
name = "Graph created from MISP event" name = "Graph created from MISP event"
graph = import_misp_graph( graph = import_misp_graph(
misp_attrs, graph_id, vt_api_key, fetch_information, name, misp_attrs, graph_id, vt_api_key, fetch_information, name,
private, fetch_vt_enterprise, user_editors, user_viewers, group_editors, private, fetch_vt_enterprise, user_editors, user_viewers, group_editors,
group_viewers, use_vt_to_connect_the_graph, max_api_quotas, group_viewers, use_vt_to_connect_the_graph, max_api_quotas,
max_search_depth) max_search_depth)
if expand_node_one_level: if expand_node_one_level:
graph.expand_n_level(1) graph.expand_n_level(1)
graphs.append(graph) graphs.append(graph)
return graphs return graphs

106
misp_modules/modules/export_mod/vt_graph.py
View File

@ -43,71 +43,71 @@ moduleconfig = [
def handler(q=False): def handler(q=False):
"""Expansion handler. """Expansion handler.
Args: Args:
q (bool, optional): module data. Defaults to False. q (bool, optional): module data. Defaults to False.
Returns: Returns:
[str]: VirusTotal graph links [str]: VirusTotal graph links
""" """
if not q: if not q:
return False return False
request = json.loads(q) request = json.loads(q)
if not request.get('config') or not request['config'].get('vt_api_key'): if not request.get('config') or not request['config'].get('vt_api_key'):
misperrors['error'] = 'A VirusTotal api key is required for this module.' misperrors['error'] = 'A VirusTotal api key is required for this module.'
return misperrors return misperrors
config = request['config'] config = request['config']
api_key = config.get('vt_api_key') api_key = config.get('vt_api_key')
fetch_information = config.get('fetch_information') or False fetch_information = config.get('fetch_information') or False
private = config.get('private') or False private = config.get('private') or False
fetch_vt_enterprise = config.get('fetch_vt_enterprise') or False fetch_vt_enterprise = config.get('fetch_vt_enterprise') or False
expand_one_level = config.get('expand_one_level') or False expand_one_level = config.get('expand_one_level') or False
user_editors = config.get('user_editors') user_editors = config.get('user_editors')
if user_editors: if user_editors:
user_editors = user_editors.split(',') user_editors = user_editors.split(',')
user_viewers = config.get('user_viewers') user_viewers = config.get('user_viewers')
if user_viewers: if user_viewers:
user_viewers = user_viewers.split(',') user_viewers = user_viewers.split(',')
group_editors = config.get('group_editors') group_editors = config.get('group_editors')
if group_editors: if group_editors:
group_editors = group_editors.split(',') group_editors = group_editors.split(',')
group_viewers = config.get('group_viewers') group_viewers = config.get('group_viewers')
if group_viewers: if group_viewers:
group_viewers = group_viewers.split(',') group_viewers = group_viewers.split(',')
graphs = from_pymisp_response( graphs = from_pymisp_response(
request, api_key, fetch_information=fetch_information, request, api_key, fetch_information=fetch_information,
private=private, fetch_vt_enterprise=fetch_vt_enterprise, private=private, fetch_vt_enterprise=fetch_vt_enterprise,
user_editors=user_editors, user_viewers=user_viewers, user_editors=user_editors, user_viewers=user_viewers,
group_editors=group_editors, group_viewers=group_viewers, group_editors=group_editors, group_viewers=group_viewers,
expand_node_one_level=expand_one_level) expand_node_one_level=expand_one_level)
links = [] links = []
for graph in graphs: for graph in graphs:
graph.save_graph() graph.save_graph()
links.append(graph.get_ui_link()) links.append(graph.get_ui_link())
# This file will contains one VirusTotal graph link for each exported event # This file will contains one VirusTotal graph link for each exported event
file_data = str(base64.b64encode(bytes('\n'.join(links), 'utf-8')), 'utf-8') file_data = str(base64.b64encode(
return {'response': [], 'data': file_data} bytes('\n'.join(links), 'utf-8')), 'utf-8')
return {'response': [], 'data': file_data}
def introspection(): def introspection():
modulesetup = { modulesetup = {
'responseType': 'application/txt', 'responseType': 'application/txt',
'outputFileExtension': 'txt', 'outputFileExtension': 'txt',
'userConfig': {}, 'userConfig': {},
'inputSource': [] 'inputSource': []
} }
return modulesetup return modulesetup
def version(): def version():
moduleinfo['config'] = moduleconfig moduleinfo['config'] = moduleconfig
return moduleinfo return moduleinfo