From 93cff2e50e7316d3b3efd858be579f06f54e7ef2 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Fri, 5 Nov 2021 11:37:10 +0100 Subject: [PATCH 1/3] chg: [feed-generator] Added exclude malware samples option --- examples/feed-generator/generate.py | 13 +++++++++++-- examples/feed-generator/settings.default.py | 6 ++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/examples/feed-generator/generate.py b/examples/feed-generator/generate.py index 2ff423d..9449a7a 100755 --- a/examples/feed-generator/generate.py +++ b/examples/feed-generator/generate.py @@ -12,6 +12,11 @@ try: except ImportError: include_deleted = False +try: + from settings import exclude_malware_samples +except ImportError: + exclude_malware_samples = False + valid_attribute_distributions = [] @@ -70,9 +75,13 @@ if __name__ == '__main__': for event in events: try: e = misp.get_event(event.uuid, deleted=include_deleted, pythonify=True) + if exclude_malware_samples: + for i, attribute in enumerate(e.attributes): + if attribute.type == 'malware-sample': + del e.attributes[i] e_feed = e.to_feed(valid_distributions=valid_attribute_distributions, with_meta=True) - except Exception as e: - print(e, event.uuid) + except Exception as err: + print(err, event.uuid) continue if not e_feed: print(f'Invalid distribution {e.distribution}, skipping') diff --git a/examples/feed-generator/settings.default.py b/examples/feed-generator/settings.default.py index 5df0130..e5de19d 100755 --- a/examples/feed-generator/settings.default.py +++ b/examples/feed-generator/settings.default.py @@ -42,3 +42,9 @@ include_deleted = False # 5: Inherit Event valid_attribute_distribution_levels = ['0', '1', '2', '3', '4', '5'] + +# By default, all attribute passing the filtering rules will be exported. +# This setting can be used to filter out attributes being of the type `malaware-sample`. +# Warning: Keep in mind that if you propagate data (via synchronisation/feeds/...), recipients +# will not be able to get the malware samples back. +exclude_malware_samples = False \ No newline at end of file From 820eb77cff07e4457da31b547ae616fa419d8c63 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Fri, 5 Nov 2021 11:37:48 +0100 Subject: [PATCH 2/3] fix: [feed-generator] Revert back the event initial search to use the index endpoint instead of RestSearch Relying on RestSearch was offering more flexibility than index in terms of filtering options, however, it might introduce a significant overhead potentially leading to timeout. --- examples/feed-generator/generate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/feed-generator/generate.py b/examples/feed-generator/generate.py index 9449a7a..1211b91 100755 --- a/examples/feed-generator/generate.py +++ b/examples/feed-generator/generate.py @@ -62,7 +62,7 @@ def saveManifest(manifest): if __name__ == '__main__': misp = init() try: - events = misp.search(metadata=True, limit=entries, **filters, pythonify=True) + events = misp.search_index(minimal=True, limit=entries, **filters, pythonify=False) except Exception as e: print(e) sys.exit("Invalid response received from MISP.") From 57de6de139aa7531f90253882eaee75537265ffc Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Wed, 17 Nov 2021 12:38:25 +0100 Subject: [PATCH 3/3] chg: [feed-generator] Make the feature to exlude attribute type more generic --- examples/feed-generator/generate.py | 18 +++++++++--------- examples/feed-generator/settings.default.py | 12 +++++------- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/examples/feed-generator/generate.py b/examples/feed-generator/generate.py index 1211b91..1856e57 100755 --- a/examples/feed-generator/generate.py +++ b/examples/feed-generator/generate.py @@ -5,7 +5,7 @@ import sys import json import os from pymisp import ExpandedPyMISP -from settings import entries, url, key, ssl, outputdir, filters, valid_attribute_distribution_levels +from settings import url, key, ssl, outputdir, filters, valid_attribute_distribution_levels try: from settings import include_deleted @@ -13,9 +13,9 @@ except ImportError: include_deleted = False try: - from settings import exclude_malware_samples + from settings import exclude_attribute_types except ImportError: - exclude_malware_samples = False + exclude_attribute_types = [] valid_attribute_distributions = [] @@ -62,7 +62,7 @@ def saveManifest(manifest): if __name__ == '__main__': misp = init() try: - events = misp.search_index(minimal=True, limit=entries, **filters, pythonify=False) + events = misp.search_index(minimal=True, **filters, pythonify=False) except Exception as e: print(e) sys.exit("Invalid response received from MISP.") @@ -74,14 +74,14 @@ if __name__ == '__main__': total = len(events) for event in events: try: - e = misp.get_event(event.uuid, deleted=include_deleted, pythonify=True) - if exclude_malware_samples: + e = misp.get_event(event['uuid'], deleted=include_deleted, pythonify=True) + if exclude_attribute_types: for i, attribute in enumerate(e.attributes): - if attribute.type == 'malware-sample': - del e.attributes[i] + if attribute.type in exclude_attribute_types: + e.attributes.pop(i) e_feed = e.to_feed(valid_distributions=valid_attribute_distributions, with_meta=True) except Exception as err: - print(err, event.uuid) + print(err, event['uuid']) continue if not e_feed: print(f'Invalid distribution {e.distribution}, skipping') diff --git a/examples/feed-generator/settings.default.py b/examples/feed-generator/settings.default.py index e5de19d..c9e19b0 100755 --- a/examples/feed-generator/settings.default.py +++ b/examples/feed-generator/settings.default.py @@ -12,9 +12,6 @@ ssl = False # sure that you use a directory dedicated to the feed outputdir = 'output' -# Determine the number of entries to output -entries = 200 - # The filters to be used for by the feed. You can use any filter that # you can use on the event index, such as organisation, tags, etc. # It uses the same joining and condition rules as the API parameters @@ -42,9 +39,10 @@ include_deleted = False # 5: Inherit Event valid_attribute_distribution_levels = ['0', '1', '2', '3', '4', '5'] - # By default, all attribute passing the filtering rules will be exported. -# This setting can be used to filter out attributes being of the type `malaware-sample`. +# This setting can be used to filter out any attributes being of the type contained in the list. # Warning: Keep in mind that if you propagate data (via synchronisation/feeds/...), recipients -# will not be able to get the malware samples back. -exclude_malware_samples = False \ No newline at end of file +# will not be able to get these attributes back unless their events get updated. +# For example: +# exclude_attribute_types = ['malware-sample'] +exclude_attribute_types = [] \ No newline at end of file