MISP
/
misp-modules
mirror of https://github.com/MISP/misp-modules
2
0
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #528 from rderkachrf/rf_release_2_0 

Release 2.0: Update Recorded future expansion module with the new data
pull/532/head
Alexandre Dulaunoy 2021-10-25 21:14:24 +02:00 committed by GitHub
parent e802679f25 4fd3323220
commit 66ac7fefad
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 316 additions and 179 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

495
misp_modules/modules/expansion/recordedfuture.py
View File

@ -1,8 +1,14 @@
import json import json
import logging import logging
import requests import requests
from requests.exceptions import HTTPError, ProxyError,\ from requests.exceptions import (
InvalidURL, ConnectTimeout, ConnectionError HTTPError,
ProxyError,
InvalidURL,
ConnectTimeout,
ConnectionError,
)
from typing import Optional, List, Tuple, Dict
from . import check_input_attribute, checking_error, standard_error_message from . import check_input_attribute, checking_error, standard_error_message
import platform import platform
import os import os
@ -10,47 +16,63 @@ from urllib.parse import quote, urlparse
from pymisp import MISPAttribute, MISPEvent, MISPTag, MISPObject from pymisp import MISPAttribute, MISPEvent, MISPTag, MISPObject
moduleinfo = { moduleinfo = {
'version': '1.0.1', "version": "2.0.0",
'author': 'Recorded Future', "author": "Recorded Future",
'description': 'Module to retrieve data from Recorded Future', "description": "Module to retrieve data from Recorded Future",
'module-type': ['expansion', 'hover'] "module-type": ["expansion", "hover"],
} }
moduleconfig = ['token', 'proxy_host', 'proxy_port', 'proxy_username', 'proxy_password'] moduleconfig = ["token", "proxy_host", "proxy_port", "proxy_username", "proxy_password"]
misperrors = {'error': 'Error'} misperrors = {"error": "Error"}
ATTRIBUTES = [ GALAXY_FILE_PATH = "https://raw.githubusercontent.com/MISP/misp-galaxy/main/clusters/"
'ip',
'ip-src', ATTRIBUTESTYPES = [
'ip-dst', "ip",
'domain', "ip-src",
'hostname', "ip-dst",
'md5', "ip-src|port",
'sha1', "ip-dst|port",
'sha256', "domain",
'uri', "hostname",
'url', "md5",
'vulnerability', "sha1",
'weakness' "sha256",
"uri",
"url",
"vulnerability",
"weakness",
]
OUTPUTATTRIBUTESTYPES = ATTRIBUTESTYPES + [
"email-src",
"malware-sample",
"text",
"target-org",
"threat-actor",
"target-user",
] ]
mispattributes = { mispattributes = {
'input': ATTRIBUTES, "input": ATTRIBUTESTYPES,
'output': ATTRIBUTES + ['email-src', 'text'], "output": OUTPUTATTRIBUTESTYPES,
'format': 'misp_standard' "format": "misp_standard",
} }
LOGGER = logging.getLogger('recorded_future') LOGGER = logging.getLogger("recorded_future")
LOGGER.setLevel(logging.INFO) LOGGER.setLevel(logging.INFO)
class RequestHandler: class RequestHandler:
"""A class for handling any outbound requests from this module.""" """A class for handling any outbound requests from this module."""
def __init__(self): def __init__(self):
self.session = requests.Session() self.session = requests.Session()
self.app_id = f'{os.path.basename(__file__)}/{moduleinfo["version"]} ({platform.platform()}) ' \ self.app_id = (
f'misp_enrichment/{moduleinfo["version"]} python-requests/{requests.__version__}' f'{os.path.basename(__file__)}/{moduleinfo["version"]} ({platform.platform()}) '
f'misp_enrichment/{moduleinfo["version"]} python-requests/{requests.__version__}'
)
self.proxies = None self.proxies = None
self.rf_token = None self.rf_token = None
@ -58,27 +80,28 @@ class RequestHandler:
"""General get method with proxy error handling.""" """General get method with proxy error handling."""
try: try:
timeout = 7 if self.proxies else None timeout = 7 if self.proxies else None
response = self.session.get(url, headers=headers, proxies=self.proxies, timeout=timeout) response = self.session.get(
url, headers=headers, proxies=self.proxies, timeout=timeout
)
response.raise_for_status() response.raise_for_status()
return response return response
except (ConnectTimeout, ProxyError, InvalidURL) as error: except (ConnectTimeout, ProxyError, InvalidURL) as error:
msg = 'Error connecting with proxy, please check the Recorded Future app proxy settings.' msg = "Error connecting with proxy, please check the Recorded Future app proxy settings."
LOGGER.error(f'{msg} Error: {error}') LOGGER.error(f"{msg} Error: {error}")
misperrors['error'] = msg misperrors["error"] = msg
raise raise
def rf_lookup(self, category: str, ioc: str) -> requests.Response: def rf_lookup(self, category: str, ioc: str) -> requests.Response:
"""Do a lookup call using Recorded Future's ConnectAPI.""" """Do a lookup call using Recorded Future's ConnectAPI."""
parsed_ioc = quote(ioc, safe='') parsed_ioc = quote(ioc, safe="")
url = f'https://api.recordedfuture.com/v2/{category}/{parsed_ioc}?fields=risk%2CrelatedEntities' url = f"https://api.recordedfuture.com/gw/misp/lookup/{category}/{parsed_ioc}"
headers = {'X-RFToken': self.rf_token, headers = {"X-RFToken": self.rf_token, "User-Agent": self.app_id}
'User-Agent': self.app_id}
try: try:
response = self.get(url, headers) response = self.get(url, headers)
except HTTPError as error: except HTTPError as error:
msg = f'Error when requesting data from Recorded Future. {error.response}: {error.response.reason}' msg = f"Error when requesting data from Recorded Future. {error.response}: {error.response.reason}"
LOGGER.error(msg) LOGGER.error(msg)
misperrors['error'] = msg misperrors["error"] = msg
raise raise
return response return response
@ -88,20 +111,49 @@ GLOBAL_REQUEST_HANDLER = RequestHandler()
class GalaxyFinder: class GalaxyFinder:
"""A class for finding MISP galaxy matches to Recorded Future data.""" """A class for finding MISP galaxy matches to Recorded Future data."""
def __init__(self): def __init__(self):
self.session = requests.Session() self.session = requests.Session()
# There are duplicates values for different keys because Links entities and Related entities
# have have different naming for the same types
self.sources = { self.sources = {
'RelatedThreatActor': [ "RelatedThreatActor": [f"{GALAXY_FILE_PATH}threat-actor.json"],
'https://raw.githubusercontent.com/MISP/misp-galaxy/main/clusters/threat-actor.json' "Threat Actor": [f"{GALAXY_FILE_PATH}threat-actor.json"],
"RelatedMalware": [
f"{GALAXY_FILE_PATH}banker.json",
f"{GALAXY_FILE_PATH}botnet.json",
f"{GALAXY_FILE_PATH}exploit-kit.json",
f"{GALAXY_FILE_PATH}rat.json",
f"{GALAXY_FILE_PATH}ransomware.json",
f"{GALAXY_FILE_PATH}malpedia.json",
],
"Malware": [
f"{GALAXY_FILE_PATH}banker.json",
f"{GALAXY_FILE_PATH}botnet.json",
f"{GALAXY_FILE_PATH}exploit-kit.json",
f"{GALAXY_FILE_PATH}rat.json",
f"{GALAXY_FILE_PATH}ransomware.json",
f"{GALAXY_FILE_PATH}malpedia.json",
],
"MitreAttackIdentifier": [
f"{GALAXY_FILE_PATH}mitre-attack-pattern.json",
f"{GALAXY_FILE_PATH}mitre-course-of-action.json",
f"{GALAXY_FILE_PATH}mitre-enterprise-attack-attack-pattern.json",
f"{GALAXY_FILE_PATH}mitre-enterprise-attack-course-of-action.json",
f"{GALAXY_FILE_PATH}mitre-enterprise-attack-intrusion-set.json",
f"{GALAXY_FILE_PATH}mitre-enterprise-attack-malware.json",
f"{GALAXY_FILE_PATH}mitre-enterprise-attack-tool.json",
f"{GALAXY_FILE_PATH}mitre-intrusion-set.json",
f"{GALAXY_FILE_PATH}mitre-malware.json",
f"{GALAXY_FILE_PATH}mitre-mobile-attack-attack-pattern.json",
f"{GALAXY_FILE_PATH}mitre-mobile-attack-course-of-action.json",
f"{GALAXY_FILE_PATH}mitre-mobile-attack-intrusion-set.json",
f"{GALAXY_FILE_PATH}mitre-mobile-attack-malware.json",
f"{GALAXY_FILE_PATH}mitre-mobile-attack-tool.json",
f"{GALAXY_FILE_PATH}mitre-pre-attack-attack-pattern.json",
f"{GALAXY_FILE_PATH}mitre-pre-attack-intrusion-set.json",
f"{GALAXY_FILE_PATH}mitre-tool.json",
], ],
'RelatedMalware': [
'https://raw.githubusercontent.com/MISP/misp-galaxy/main/clusters/banker.json',
'https://raw.githubusercontent.com/MISP/misp-galaxy/main/clusters/botnet.json',
'https://raw.githubusercontent.com/MISP/misp-galaxy/main/clusters/exploit-kit.json',
'https://raw.githubusercontent.com/MISP/misp-galaxy/main/clusters/rat.json',
'https://raw.githubusercontent.com/MISP/misp-galaxy/main/clusters/ransomware.json',
'https://raw.githubusercontent.com/MISP/misp-galaxy/main/clusters/malpedia.json'
]
} }
self.galaxy_clusters = {} self.galaxy_clusters = {}
@ -112,33 +164,38 @@ class GalaxyFinder:
for source in self.sources.get(related_type): for source in self.sources.get(related_type):
try: try:
response = GLOBAL_REQUEST_HANDLER.get(source) response = GLOBAL_REQUEST_HANDLER.get(source)
name = source.split('/')[-1].split('.')[0] name = source.split("/")[-1].split(".")[0]
self.galaxy_clusters[related_type] = {name: response.json()} self.galaxy_clusters.setdefault(related_type, {}).update(
{name: response.json()}
)
except ConnectionError as error: except ConnectionError as error:
LOGGER.warning(f'pull_galaxy_cluster failed for source: {source}, with error: {error}.') LOGGER.warning(
f"pull_galaxy_cluster failed for source: {source}, with error: {error}."
)
def find_galaxy_match(self, indicator: str, related_type: str) -> str: def find_galaxy_match(self, indicator: str, related_type: str) -> str:
"""Searches the clusters of the related_type for a match with the indicator. """Searches the clusters of the related_type for a match with the indicator.
:returns the first matching galaxy string or an empty string if no galaxy match is found. :returns the first matching galaxy string or an empty string if no galaxy match is found.
""" """
self.pull_galaxy_cluster(related_type) self.pull_galaxy_cluster(related_type)
for cluster_name, cluster in self.galaxy_clusters.get(related_type, {}).items(): for cluster_name, cluster in self.galaxy_clusters.get(related_type, {}).items():
for value in cluster['values']: for value in cluster["values"]:
try: if indicator in value.get("meta", {}).get(
if indicator in value['meta']['synonyms'] or indicator in value['value']: "synonyms", ""
value = value['value'] ) or indicator in value.get("value", ""):
return f'misp-galaxy:{cluster_name}="{value}"' value = value["value"]
except KeyError: return f'misp-galaxy:{cluster_name}="{value}"'
pass return ""
return ''
class RFColors: class RFColors:
"""Class for setting signature RF-colors.""" """Class for setting signature RF-colors."""
def __init__(self): def __init__(self):
self.rf_white = '#CCCCCC' self.rf_white = "#CCCCCC"
self.rf_yellow = '#FFCE00' self.rf_grey = " #CDCDCD"
self.rf_red = '#CF0A2C' self.rf_yellow = "#FFCF00"
self.rf_red = "#D10028"
def riskscore_color(self, risk_score: int) -> str: def riskscore_color(self, risk_score: int) -> str:
"""Returns appropriate hex-colors according to risk score.""" """Returns appropriate hex-colors according to risk score."""
@ -160,194 +217,272 @@ class RFColors:
else: # risk_rule_criticality == 3 or 4 else: # risk_rule_criticality == 3 or 4
return self.rf_red return self.rf_red
def criticality_color(self, criticality) -> str:
mapper = {
"None": self.rf_grey,
"Low": self.rf_grey,
"Unusual": self.rf_grey,
"Informational": self.rf_grey,
"Medium": self.rf_yellow,
"Suspicious": self.rf_yellow,
"High": self.rf_red,
"Critical": self.rf_red,
"Very Critical": self.rf_red,
"Malicious": self.rf_red,
"Very Malicious": self.rf_red,
}
return mapper.get(criticality, self.rf_white)
class RFEnricher: class RFEnricher:
"""Class for enriching an attribute with data from Recorded Future. """Class for enriching an attribute with data from Recorded Future.
The enrichment data is returned as a custom MISP object. The enrichment data is returned as a custom MISP object.
""" """
def __init__(self, attribute_props: dict): def __init__(self, attribute_props: dict):
self.event = MISPEvent() self.event = MISPEvent()
self.enrichment_object = MISPObject('Recorded Future Enrichment') self.enrichment_object = MISPObject("Recorded Future Enrichment")
description = ( description = (
'An object containing the enriched attribute and ' "An object containing the enriched attribute and "
'related entities from Recorded Future.' "related entities from Recorded Future."
)
self.enrichment_object.from_dict(
**{"meta-category": "misc", "description": description, "distribution": 0}
) )
self.enrichment_object.from_dict(**{
'meta-category': 'misc',
'description': description,
'distribution': 0
})
# Create a copy of enriched attribute to add tags to # Create a copy of enriched attribute to add tags to
temp_attr = MISPAttribute() temp_attr = MISPAttribute()
temp_attr.from_dict(**attribute_props) temp_attr.from_dict(**attribute_props)
self.enriched_attribute = MISPAttribute() self.enriched_attribute = MISPAttribute()
self.enriched_attribute.from_dict(**{ self.enriched_attribute.from_dict(
'value': temp_attr.value, **{"value": temp_attr.value, "type": temp_attr.type, "distribution": 0}
'type': temp_attr.type, )
'distribution': 0
})
self.related_attributes = [] self.related_attributes: List[Tuple[str, MISPAttribute]] = []
self.color_picker = RFColors() self.color_picker = RFColors()
self.galaxy_finder = GalaxyFinder() self.galaxy_finder = GalaxyFinder()
# Mapping from MISP-type to RF-type # Mapping from MISP-type to RF-type
self.type_to_rf_category = { self.type_to_rf_category = {
'ip': 'ip', "ip": "ip",
'ip-src': 'ip', "ip-src": "ip",
'ip-dst': 'ip', "ip-dst": "ip",
'domain': 'domain', "ip-src|port": "ip",
'hostname': 'domain', "ip-dst|port": "ip",
'md5': 'hash', "domain": "domain",
'sha1': 'hash', "hostname": "domain",
'sha256': 'hash', "md5": "hash",
'uri': 'url', "sha1": "hash",
'url': 'url', "sha256": "hash",
'vulnerability': 'vulnerability', "uri": "url",
'weakness': 'vulnerability' "url": "url",
"vulnerability": "vulnerability",
"weakness": "vulnerability",
} }
# Related entities from RF portrayed as related attributes in MISP # Related entities have 'Related' as part of the word and Links entities from RF
# portrayed as related attributes in MISP
self.related_attribute_types = [ self.related_attribute_types = [
'RelatedIpAddress', 'RelatedInternetDomainName', 'RelatedHash', "RelatedIpAddress",
'RelatedEmailAddress', 'RelatedCyberVulnerability' "RelatedInternetDomainName",
"RelatedHash",
"RelatedEmailAddress",
"RelatedCyberVulnerability",
"IpAddress",
"InternetDomainName",
"Hash",
"EmailAddress",
"CyberVulnerability",
]
# Related entities have 'Related' as part of the word and and Links entities from RF portrayed as tags in MISP
self.galaxy_tag_types = [
"RelatedMalware",
"RelatedThreatActor",
"Threat Actor",
"MitreAttackIdentifier",
"Malware",
] ]
# Related entities from RF portrayed as tags in MISP
self.galaxy_tag_types = ['RelatedMalware', 'RelatedThreatActor']
def enrich(self) -> None: def enrich(self) -> None:
"""Run the enrichment.""" """Run the enrichment."""
category = self.type_to_rf_category.get(self.enriched_attribute.type) category = self.type_to_rf_category.get(self.enriched_attribute.type, "")
json_response = GLOBAL_REQUEST_HANDLER.rf_lookup(category, self.enriched_attribute.value) enriched_attribute_value = self.enriched_attribute.value
# If enriched attribute has a port we need to remove that port
# since RF do not support enriching ip addresses with port
if self.enriched_attribute.type in ["ip-src|port", "ip-dst|port"]:
enriched_attribute_value = enriched_attribute_value.split("|")[0]
json_response = GLOBAL_REQUEST_HANDLER.rf_lookup(
category, enriched_attribute_value
)
response = json.loads(json_response.content) response = json.loads(json_response.content)
try: try:
# Add risk score and risk rules as tags to the enriched attribute # Add risk score and risk rules as tags to the enriched attribute
risk_score = response['data']['risk']['score'] risk_score = response["data"]["risk"]["score"]
hex_color = self.color_picker.riskscore_color(risk_score) hex_color = self.color_picker.riskscore_color(risk_score)
tag_name = f'recorded-future:risk-score="{risk_score}"' tag_name = f'recorded-future:risk-score="{risk_score}"'
self.add_tag(tag_name, hex_color) self.add_tag(tag_name, hex_color)
for evidence in response['data']['risk']['evidenceDetails']: risk_criticality = response["data"]["risk"]["criticalityLabel"]
risk_rule = evidence['rule'] hex_color = self.color_picker.criticality_color(risk_criticality)
criticality = evidence['criticality'] tag_name = f'recorded-future:criticality="{risk_criticality}"'
self.add_tag(tag_name, hex_color)
for evidence in response["data"]["risk"]["evidenceDetails"]:
risk_rule = evidence["rule"]
criticality = evidence["criticality"]
hex_color = self.color_picker.riskrule_color(criticality) hex_color = self.color_picker.riskrule_color(criticality)
tag_name = f'recorded-future:risk-rule="{risk_rule}"' tag_name = f'recorded-future:risk-rule="{risk_rule}"'
self.add_tag(tag_name, hex_color) self.add_tag(tag_name, hex_color)
# Retrieve related entities links_data = response["data"].get("links", {}).get("hits")
for related_entity in response['data']['relatedEntities']: # Check if we have error in links response. If yes, then user do not have right module enabled in token
related_type = related_entity['type'] links_access_error = response["data"].get("links", {}).get("error")
if related_type in self.related_attribute_types: galaxy_tags = []
# Related entities returned as additional attributes if not links_access_error:
for related in related_entity['entities']: for hit in links_data:
if int(related["count"]) > 4: for section in hit["sections"]:
indicator = related['entity']['name'] for sec_list in section["lists"]:
self.add_related_attribute(indicator, related_type) entity_type = sec_list["type"]["name"]
elif related_type in self.galaxy_tag_types: for entity in sec_list["entities"]:
# Related entities added as galaxy-tags to the enriched attribute if entity_type in self.galaxy_tag_types:
galaxy_tags = [] galaxy = self.galaxy_finder.find_galaxy_match(
for related in related_entity['entities']: entity["name"], entity_type
if int(related["count"]) > 4: )
indicator = related['entity']['name'] if galaxy and galaxy not in galaxy_tags:
galaxy = self.galaxy_finder.find_galaxy_match(indicator, related_type) galaxy_tags.append(galaxy)
# Handle deduplication of galaxy tags else:
if galaxy and galaxy not in galaxy_tags: self.add_attribute(entity["name"], entity_type)
galaxy_tags.append(galaxy)
for galaxy in galaxy_tags: else:
self.add_tag(galaxy) # Retrieve related entities
for related_entity in response["data"]["relatedEntities"]:
related_type = related_entity["type"]
if related_type in self.related_attribute_types:
# Related entities returned as additional attributes
for related in related_entity["entities"]:
# filter those entities that have count bigger than 4, to reduce noise
# because there can be a huge list of related entities
if int(related["count"]) > 4:
indicator = related["entity"]["name"]
self.add_attribute(indicator, related_type)
elif related_type in self.galaxy_tag_types:
# Related entities added as galaxy-tags to the enriched attribute
galaxy_tags = []
for related in related_entity["entities"]:
# filter those entities that have count bigger than 4, to reduce noise
# because there can be a huge list of related entities
if int(related["count"]) > 4:
indicator = related["entity"]["name"]
galaxy = self.galaxy_finder.find_galaxy_match(
indicator, related_type
)
# Handle deduplication of galaxy tags
if galaxy and galaxy not in galaxy_tags:
galaxy_tags.append(galaxy)
for galaxy in galaxy_tags:
self.add_tag(galaxy)
except KeyError: except KeyError:
misperrors['error'] = 'Unexpected format in Recorded Future api response.' misperrors["error"] = "Unexpected format in Recorded Future api response."
raise raise
def add_related_attribute(self, indicator: str, related_type: str) -> None: def add_attribute(self, indicator: str, indicator_type: str) -> None:
"""Helper method for adding an indicator to the related attribute list.""" """Helper method for adding an indicator to the attribute list."""
out_type = self.get_output_type(related_type, indicator) out_type = self.get_output_type(indicator_type, indicator)
attribute = MISPAttribute() attribute = MISPAttribute()
attribute.from_dict(**{'value': indicator, 'type': out_type, 'distribution': 0}) attribute.from_dict(**{"value": indicator, "type": out_type, "distribution": 0})
self.related_attributes.append((related_type, attribute)) self.related_attributes.append((indicator_type, attribute))
def add_tag(self, tag_name: str, hex_color: str = None) -> None: def add_tag(self, tag_name: str, hex_color: str = None) -> None:
"""Helper method for adding a tag to the enriched attribute.""" """Helper method for adding a tag to the enriched attribute."""
tag = MISPTag() tag = MISPTag()
tag_properties = {'name': tag_name} tag_properties = {"name": tag_name}
if hex_color: if hex_color:
tag_properties['colour'] = hex_color tag_properties["colour"] = hex_color
tag.from_dict(**tag_properties) tag.from_dict(**tag_properties)
self.enriched_attribute.add_tag(tag) self.enriched_attribute.add_tag(tag)
def get_output_type(self, related_type: str, indicator: str) -> str: def get_output_type(self, related_type: str, indicator: str) -> str:
"""Helper method for translating a Recorded Future related type to a MISP output type.""" """Helper method for translating a Recorded Future related type to a MISP output type."""
output_type = 'text' output_type = "text"
if related_type == 'RelatedIpAddress': if related_type in ["RelatedIpAddress", "IpAddress"]:
output_type = 'ip-dst' output_type = "ip-dst"
elif related_type == 'RelatedInternetDomainName': elif related_type in ["RelatedInternetDomainName", "InternetDomainName"]:
output_type = 'domain' output_type = "domain"
elif related_type == 'RelatedHash': elif related_type in ["RelatedHash", "Hash"]:
hash_len = len(indicator) hash_len = len(indicator)
if hash_len == 64: if hash_len == 64:
output_type = 'sha256' output_type = "sha256"
elif hash_len == 40: elif hash_len == 40:
output_type = 'sha1' output_type = "sha1"
elif hash_len == 32: elif hash_len == 32:
output_type = 'md5' output_type = "md5"
elif related_type == 'RelatedEmailAddress': elif related_type in ["RelatedEmailAddress", "EmailAddress"]:
output_type = 'email-src' output_type = "email-src"
elif related_type == 'RelatedCyberVulnerability': elif related_type in ["RelatedCyberVulnerability", "CyberVulnerability"]:
signature = indicator.split('-')[0] signature = indicator.split("-")[0]
if signature == 'CVE': if signature == "CVE":
output_type = 'vulnerability' output_type = "vulnerability"
elif signature == 'CWE': elif signature == "CWE":
output_type = 'weakness' output_type = "weakness"
elif related_type == "MalwareSignature":
output_type = "malware-sample"
elif related_type == "Organization":
output_type = "target-org"
elif related_type == "Username":
output_type = "target-user"
return output_type return output_type
def get_results(self) -> dict: def get_results(self) -> dict:
"""Build and return the enrichment results.""" """Build and return the enrichment results."""
self.enrichment_object.add_attribute('Enriched attribute', **self.enriched_attribute) self.enrichment_object.add_attribute(
"Enriched attribute", **self.enriched_attribute
)
for related_type, attribute in self.related_attributes: for related_type, attribute in self.related_attributes:
self.enrichment_object.add_attribute(related_type, **attribute) self.enrichment_object.add_attribute(related_type, **attribute)
self.event.add_object(**self.enrichment_object) self.event.add_object(**self.enrichment_object)
event = json.loads(self.event.to_json()) event = json.loads(self.event.to_json())
result = {key: event[key] for key in ['Object'] if key in event} result = {key: event[key] for key in ["Object"] if key in event}
return {'results': result} return {"results": result}
def get_proxy_settings(config: dict) -> dict: def get_proxy_settings(config: dict) -> Optional[Dict[str, str]]:
"""Returns proxy settings in the requests format. """Returns proxy settings in the requests format.
If no proxy settings are set, return None.""" If no proxy settings are set, return None."""
proxies = None proxies = None
host = config.get('proxy_host') host = config.get("proxy_host")
port = config.get('proxy_port') port = config.get("proxy_port")
username = config.get('proxy_username') username = config.get("proxy_username")
password = config.get('proxy_password') password = config.get("proxy_password")
if host: if host:
if not port: if not port:
misperrors['error'] = 'The recordedfuture_proxy_host config is set, ' \ misperrors["error"] = (
'please also set the recordedfuture_proxy_port.' "The recordedfuture_proxy_host config is set, "
"please also set the recordedfuture_proxy_port."
)
raise KeyError raise KeyError
parsed = urlparse(host) parsed = urlparse(host)
if 'http' in parsed.scheme: if "http" in parsed.scheme:
scheme = 'http' scheme = "http"
else: else:
scheme = parsed.scheme scheme = parsed.scheme
netloc = parsed.netloc netloc = parsed.netloc
host = f'{netloc}:{port}' host = f"{netloc}:{port}"
if username: if username:
if not password: if not password:
misperrors['error'] = 'The recordedfuture_proxy_username config is set, ' \ misperrors["error"] = (
'please also set the recordedfuture_proxy_password.' "The recordedfuture_proxy_username config is set, "
"please also set the recordedfuture_proxy_password."
)
raise KeyError raise KeyError
auth = f'{username}:{password}' auth = f"{username}:{password}"
host = auth + '@' + host host = auth + "@" + host
proxies = { proxies = {"http": f"{scheme}://{host}", "https": f"{scheme}://{host}"}
'http': f'{scheme}://{host}',
'https': f'{scheme}://{host}'
}
LOGGER.info(f'Proxy settings: {proxies}') LOGGER.info(f"Proxy settings: {proxies}")
return proxies return proxies
@ -357,23 +492,25 @@ def handler(q=False):
return False return False
request = json.loads(q) request = json.loads(q)
config = request.get('config') config = request.get("config")
if config and config.get('token'): if config and config.get("token"):
GLOBAL_REQUEST_HANDLER.rf_token = config.get('token') GLOBAL_REQUEST_HANDLER.rf_token = config.get("token")
else: else:
misperrors['error'] = 'Missing Recorded Future token.' misperrors["error"] = "Missing Recorded Future token."
return misperrors return misperrors
if not request.get('attribute') or not check_input_attribute(request['attribute'], requirements=('type', 'value')): if not request.get("attribute") or not check_input_attribute(
return {'error': f'{standard_error_message}, {checking_error}.'} request["attribute"], requirements=("type", "value")
if request['attribute']['type'] not in mispattributes['input']: ):
return {'error': 'Unsupported attribute type.'} return {"error": f"{standard_error_message}, {checking_error}."}
if request["attribute"]["type"] not in mispattributes["input"]:
return {"error": "Unsupported attribute type."}
try: try:
GLOBAL_REQUEST_HANDLER.proxies = get_proxy_settings(config) GLOBAL_REQUEST_HANDLER.proxies = get_proxy_settings(config)
except KeyError: except KeyError:
return misperrors return misperrors
input_attribute = request.get('attribute') input_attribute = request.get("attribute")
rf_enricher = RFEnricher(input_attribute) rf_enricher = RFEnricher(input_attribute)
try: try:
@ -392,5 +529,5 @@ def introspection():
def version(): def version():
"""Returns a dict with the version and the associated meta-data """Returns a dict with the version and the associated meta-data
including potential configurations required of the module.""" including potential configurations required of the module."""
moduleinfo['config'] = moduleconfig moduleinfo["config"] = moduleconfig
return moduleinfo return moduleinfo