From ee21a88127abc91332abfff10dabd04c1afca422 Mon Sep 17 00:00:00 2001 From: Jesse Hedden Date: Thu, 6 Aug 2020 21:59:13 -0700 Subject: [PATCH 1/5] updating to include metadata and alter type of trustar link generated --- .../modules/expansion/trustar_enrich.py | 70 +++++++++++++------ 1 file changed, 48 insertions(+), 22 deletions(-) diff --git a/misp_modules/modules/expansion/trustar_enrich.py b/misp_modules/modules/expansion/trustar_enrich.py index 48b4895..f4df990 100644 --- a/misp_modules/modules/expansion/trustar_enrich.py +++ b/misp_modules/modules/expansion/trustar_enrich.py @@ -1,7 +1,10 @@ import json import pymisp +from base64 import b64encode +from collections import OrderedDict from pymisp import MISPAttribute, MISPEvent, MISPObject from trustar import TruStar +from urllib.parse import quote misperrors = {'error': "Error"} mispattributes = { @@ -33,9 +36,12 @@ class TruSTARParser: 'SHA256': "sha256" } + SUMMARY_FIELDS = ["source", "score", "attributes"] + METADATA_FIELDS = ["sightings", "first_seen", "last_seen", "tags"] + REPORT_BASE_URL = "https://station.trustar.co/constellation/reports/{}" - CLIENT_METATAG = "MISP-{}".format(pymisp.__version__) + CLIENT_METATAG = f"MISP-{pymisp.__version__}" def __init__(self, attribute, config): config['enclave_ids'] = config.get('enclave_ids', "").strip().split(',') @@ -55,20 +61,36 @@ class TruSTARParser: results = {key: event[key] for key in ('Attribute', 'Object') if (key in event and event[key])} return {'results': results} - def generate_trustar_links(self, entity_value): + def generate_trustar_link(self, entity_type, entity_value): """ - Generates links to TruSTAR reports if they exist. + Generates link to TruSTAR report of entity. :param entity_value: Value of entity. """ - report_links = list() - trustar_reports = self.ts_client.search_reports(entity_value) - for report in trustar_reports: - report_links.append(self.REPORT_BASE_URL.format(report.id)) + report_id = b64encode(quote(f"{entity_type}|{entity_value}").encode()).decode() - return report_links + return self.REPORT_BASE_URL.format(report_id) - def parse_indicator_summary(self, summaries): + def generate_enrichment_report(self, summary, metadata): + """ + Extracts desired fields from summary and metadata reports and + generates an enrichment report. + + :param summary: Indicator summary report. + :param metadata: Indicator metadata report. + :return: Enrichment report. + """ + enrichment_report = OrderedDict() + + for field in self.SUMMARY_FIELDS: + enrichment_report[field] = summary.get(field) + + for field in self.METADATA_FIELDS: + enrichment_report[field] = metadata.get(field) + + return enrichment_report + + def parse_indicator_summary(self, summaries, metadata): """ Converts a response from the TruSTAR /1.3/indicators/summaries endpoint a MISP trustar_report object and adds the summary data and links as attributes. @@ -78,18 +100,21 @@ class TruSTARParser: """ for summary in summaries: - trustar_obj = MISPObject('trustar_report') - indicator_type = summary.indicator_type - indicator_value = summary.value - if summary_type in self.ENTITY_TYPE_MAPPINGS: - trustar_obj.add_attribute(indicator_type, attribute_type=self.ENTITY_TYPE_MAPPINGS[indicator_type], - value=indicator_value) - trustar_obj.add_attribute("INDICATOR_SUMMARY", attribute_type="text", - value=json.dumps(summary.to_dict(), sort_keys=True, indent=4)) - report_links = self.generate_trustar_links(indicator_value) - for link in report_links: - trustar_obj.add_attribute("REPORT_LINK", attribute_type="link", value=link) - self.misp_event.add_object(**trustar_obj) + if summary.indicator_type in self.ENTITY_TYPE_MAPPINGS: + indicator_type = summary.indicator_type + indicator_value = summary.indicator_value + try: + enrichment_report = self.generate_enrichment_report(summary.to_dict(), metadata.to_dict()) + trustar_obj = MISPObject('trustar_report') + trustar_obj.add_attribute(indicator_type, attribute_type=self.ENTITY_TYPE_MAPPINGS[indicator_type], + value=indicator_value) + trustar_obj.add_attribute("INDICATOR_SUMMARY", attribute_type="text", + value=json.dumps(enrichment_report, indent=4)) + report_link = self.generate_trustar_link(indicator_type, indicator_value) + trustar_obj.add_attribute("REPORT_LINK", attribute_type="link", value=report_link) + self.misp_event.add_object(**trustar_obj) + except Exception as e: + misperrors['error'] = f"Error enriching data with TruSTAR -- {e}" def handler(q=False): @@ -114,10 +139,11 @@ def handler(q=False): trustar_parser = TruSTARParser(attribute, config) try: + metadata = trustar_parser.ts_client.get_indicators_metadata([attribute['value']]) summaries = list( trustar_parser.ts_client.get_indicator_summaries([attribute['value']], page_size=MAX_PAGE_SIZE)) except Exception as e: - misperrors['error'] = "Unable to retrieve TruSTAR summary data: {}".format(e) + misperrors['error'] = f"Unable to retrieve TruSTAR summary data: {e}" return misperrors trustar_parser.parse_indicator_summary(summaries) From 2d464adfd635d89c9b9db402a17aece813a503ab Mon Sep 17 00:00:00 2001 From: Jesse Hedden Date: Sun, 9 Aug 2020 20:29:37 -0700 Subject: [PATCH 2/5] added error checking --- .../modules/expansion/trustar_enrich.py | 123 ++++++++++++------ 1 file changed, 86 insertions(+), 37 deletions(-) diff --git a/misp_modules/modules/expansion/trustar_enrich.py b/misp_modules/modules/expansion/trustar_enrich.py index f4df990..81238ad 100644 --- a/misp_modules/modules/expansion/trustar_enrich.py +++ b/misp_modules/modules/expansion/trustar_enrich.py @@ -3,7 +3,7 @@ import pymisp from base64 import b64encode from collections import OrderedDict from pymisp import MISPAttribute, MISPEvent, MISPObject -from trustar import TruStar +from trustar import TruStar, Indicator from urllib.parse import quote misperrors = {'error': "Error"} @@ -36,7 +36,7 @@ class TruSTARParser: 'SHA256': "sha256" } - SUMMARY_FIELDS = ["source", "score", "attributes"] + SUMMARY_FIELDS = ["severityLevel", "source", "score", "attributes"] METADATA_FIELDS = ["sightings", "first_seen", "last_seen", "tags"] REPORT_BASE_URL = "https://station.trustar.co/constellation/reports/{}" @@ -57,64 +57,104 @@ class TruSTARParser: """ Returns the MISP Event enriched with TruSTAR indicator summary data. """ - event = json.loads(self.misp_event.to_json()) - results = {key: event[key] for key in ('Attribute', 'Object') if (key in event and event[key])} - return {'results': results} + try: + event = json.loads(self.misp_event.to_json()) + results = {key: event[key] for key in ('Attribute', 'Object') if (key in event and event[key])} + return {'results': results} + except Exception as e: + misperrors['error'] += f" -- Encountered issue serializing enrichment data -- {e}" + return misperrors def generate_trustar_link(self, entity_type, entity_value): """ Generates link to TruSTAR report of entity. + :param entity_type: Type of entity. :param entity_value: Value of entity. + :return: Link to indicator report in TruSTAR platform. """ report_id = b64encode(quote(f"{entity_type}|{entity_value}").encode()).decode() return self.REPORT_BASE_URL.format(report_id) + @staticmethod + def extract_tags(enrichment_report): + """ + Extracts tags from the enrichment report in order to add them + to the TruSTAR MISP Object. Removes tags from report to avoid + redundancy. + + :param: Enrichment data. + """ + if enrichment_report and enrichment_report.get('tags'): + return [tag.get('name') for tag in enrichment_report.pop('tags')] + return None + def generate_enrichment_report(self, summary, metadata): """ Extracts desired fields from summary and metadata reports and generates an enrichment report. - :param summary: Indicator summary report. - :param metadata: Indicator metadata report. + :param summary: Indicator summary report. + :param metadata: Indicator metadata report. :return: Enrichment report. """ enrichment_report = OrderedDict() - for field in self.SUMMARY_FIELDS: - enrichment_report[field] = summary.get(field) + if summary: + summary_dict = summary.to_dict() + enrichment_report.update( + {field: summary_dict[field] for field in self.SUMMARY_FIELDS if summary_dict.get(field)}) - for field in self.METADATA_FIELDS: - enrichment_report[field] = metadata.get(field) + if metadata: + metadata_dict = metadata.to_dict() + enrichment_report.update( + {field: metadata_dict[field] for field in self.METADATA_FIELDS if metadata_dict.get(field)}) return enrichment_report - def parse_indicator_summary(self, summaries, metadata): + def parse_indicator_summary(self, indicator, summary, metadata): """ - Converts a response from the TruSTAR /1.3/indicators/summaries endpoint - a MISP trustar_report object and adds the summary data and links as attributes. + Pulls enrichment data from the TruSTAR /indicators/summaries and /indicators/metadata endpoints + and creates a MISP trustar_report. - :param summaries: A TruSTAR Python SDK Page.generator object for generating - indicator summaries pages. + :param indicator: Value of the attribute + :summary: Indicator summary response object. + :metadata: Indicator response object. """ - for summary in summaries: - if summary.indicator_type in self.ENTITY_TYPE_MAPPINGS: - indicator_type = summary.indicator_type - indicator_value = summary.indicator_value - try: - enrichment_report = self.generate_enrichment_report(summary.to_dict(), metadata.to_dict()) - trustar_obj = MISPObject('trustar_report') - trustar_obj.add_attribute(indicator_type, attribute_type=self.ENTITY_TYPE_MAPPINGS[indicator_type], - value=indicator_value) - trustar_obj.add_attribute("INDICATOR_SUMMARY", attribute_type="text", - value=json.dumps(enrichment_report, indent=4)) - report_link = self.generate_trustar_link(indicator_type, indicator_value) - trustar_obj.add_attribute("REPORT_LINK", attribute_type="link", value=report_link) - self.misp_event.add_object(**trustar_obj) - except Exception as e: - misperrors['error'] = f"Error enriching data with TruSTAR -- {e}" + # Verify that the indicator type is supported by TruSTAR + if summary and summary.indicator_type in self.ENTITY_TYPE_MAPPINGS: + indicator_type = summary.indicator_type + elif metadata and metadata.type in self.ENTITY_TYPE_MAPPINGS: + indicator_type = metadata.type + else: + misperrors['error'] += " -- Attribute not found or not supported" + raise Exception + + try: + # Extract most relevant fields from indicator summary and metadata responses + enrichment_report = self.generate_enrichment_report(summary, metadata) + tags = self.extract_tags(enrichment_report) + + if enrichment_report: + trustar_obj = MISPObject('trustar_report') + trustar_obj.add_attribute(indicator_type, attribute_type=self.ENTITY_TYPE_MAPPINGS[indicator_type], + value=indicator) + trustar_obj.add_attribute("INDICATOR_SUMMARY", attribute_type="text", + value=json.dumps(enrichment_report, indent=4)) + report_link = self.generate_trustar_link(indicator_type, indicator) + trustar_obj.add_attribute("REPORT_LINK", attribute_type="link", value=report_link) + self.misp_event.add_object(**trustar_obj) + elif not tags: + raise Exception("No relevant data found") + + if tags: + for tag in tags: + self.misp_event.add_attribute_tag(tag, indicator) + except Exception as e: + misperrors['error'] += f" -- Error enriching attribute {indicator} -- {e}" + raise e def handler(q=False): @@ -139,14 +179,23 @@ def handler(q=False): trustar_parser = TruSTARParser(attribute, config) try: - metadata = trustar_parser.ts_client.get_indicators_metadata([attribute['value']]) - summaries = list( - trustar_parser.ts_client.get_indicator_summaries([attribute['value']], page_size=MAX_PAGE_SIZE)) + metadata = trustar_parser.ts_client.get_indicators_metadata([Indicator(value=attribute['value'])])[0] except Exception as e: - misperrors['error'] = f"Unable to retrieve TruSTAR summary data: {e}" + metadata = None + misperrors['error'] += f" -- Could not retrieve indicator metadata from TruSTAR {e}" + + try: + summary = list( + trustar_parser.ts_client.get_indicator_summaries([attribute['value']], page_size=MAX_PAGE_SIZE))[0] + except Exception as e: + summary = None + misperrors['error'] += f" -- Unable to retrieve TruSTAR summary data: {e}" + + try: + trustar_parser.parse_indicator_summary(attribute['value'], summary, metadata) + except Exception: return misperrors - trustar_parser.parse_indicator_summary(summaries) return trustar_parser.get_results() From 0b576faa68c1067af8a520085c37798bba146361 Mon Sep 17 00:00:00 2001 From: Jesse Hedden Date: Sun, 9 Aug 2020 20:36:47 -0700 Subject: [PATCH 3/5] added comments --- misp_modules/modules/expansion/trustar_enrich.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/misp_modules/modules/expansion/trustar_enrich.py b/misp_modules/modules/expansion/trustar_enrich.py index 81238ad..6e615db 100644 --- a/misp_modules/modules/expansion/trustar_enrich.py +++ b/misp_modules/modules/expansion/trustar_enrich.py @@ -99,6 +99,7 @@ class TruSTARParser: :param metadata: Indicator metadata report. :return: Enrichment report. """ + # Preserve order of fields as they exist in SUMMARY_FIELDS and METADATA_FIELDS enrichment_report = OrderedDict() if summary: @@ -147,11 +148,13 @@ class TruSTARParser: trustar_obj.add_attribute("REPORT_LINK", attribute_type="link", value=report_link) self.misp_event.add_object(**trustar_obj) elif not tags: + # If enrichment report is empty and there are no tags, nothing to add to attribute raise Exception("No relevant data found") if tags: for tag in tags: self.misp_event.add_attribute_tag(tag, indicator) + except Exception as e: misperrors['error'] += f" -- Error enriching attribute {indicator} -- {e}" raise e @@ -177,18 +180,18 @@ def handler(q=False): attribute = request['attribute'] trustar_parser = TruSTARParser(attribute, config) + metadata = None + summary = None try: metadata = trustar_parser.ts_client.get_indicators_metadata([Indicator(value=attribute['value'])])[0] except Exception as e: - metadata = None misperrors['error'] += f" -- Could not retrieve indicator metadata from TruSTAR {e}" try: summary = list( trustar_parser.ts_client.get_indicator_summaries([attribute['value']], page_size=MAX_PAGE_SIZE))[0] except Exception as e: - summary = None misperrors['error'] += f" -- Unable to retrieve TruSTAR summary data: {e}" try: From 91417d390b85f141b6ae337f7050833b1d846208 Mon Sep 17 00:00:00 2001 From: Jesse Hedden Date: Sun, 9 Aug 2020 20:41:52 -0700 Subject: [PATCH 4/5] added comments --- misp_modules/modules/expansion/trustar_enrich.py | 1 + 1 file changed, 1 insertion(+) diff --git a/misp_modules/modules/expansion/trustar_enrich.py b/misp_modules/modules/expansion/trustar_enrich.py index 6e615db..7b6ff3c 100644 --- a/misp_modules/modules/expansion/trustar_enrich.py +++ b/misp_modules/modules/expansion/trustar_enrich.py @@ -85,6 +85,7 @@ class TruSTARParser: redundancy. :param: Enrichment data. + :return: List of tags. """ if enrichment_report and enrichment_report.get('tags'): return [tag.get('name') for tag in enrichment_report.pop('tags')] From a3c01fa318b8b1a008e302885ff9161731349589 Mon Sep 17 00:00:00 2001 From: Jesse Hedden Date: Mon, 10 Aug 2020 07:53:24 -0700 Subject: [PATCH 5/5] added comments --- misp_modules/modules/expansion/trustar_enrich.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/misp_modules/modules/expansion/trustar_enrich.py b/misp_modules/modules/expansion/trustar_enrich.py index 7b6ff3c..33dd814 100644 --- a/misp_modules/modules/expansion/trustar_enrich.py +++ b/misp_modules/modules/expansion/trustar_enrich.py @@ -36,6 +36,7 @@ class TruSTARParser: 'SHA256': "sha256" } + # Relevant fields from each TruSTAR endpoint SUMMARY_FIELDS = ["severityLevel", "source", "score", "attributes"] METADATA_FIELDS = ["sightings", "first_seen", "last_seen", "tags"] @@ -140,13 +141,16 @@ class TruSTARParser: tags = self.extract_tags(enrichment_report) if enrichment_report: + # Create MISP trustar_report object and populate it with enrichment data trustar_obj = MISPObject('trustar_report') trustar_obj.add_attribute(indicator_type, attribute_type=self.ENTITY_TYPE_MAPPINGS[indicator_type], value=indicator) trustar_obj.add_attribute("INDICATOR_SUMMARY", attribute_type="text", value=json.dumps(enrichment_report, indent=4)) + report_link = self.generate_trustar_link(indicator_type, indicator) trustar_obj.add_attribute("REPORT_LINK", attribute_type="link", value=report_link) + self.misp_event.add_object(**trustar_obj) elif not tags: # If enrichment report is empty and there are no tags, nothing to add to attribute