From 82ff568feb1019b8e623801c7ac628693f47c090 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Wed, 21 Dec 2022 14:20:13 +0100 Subject: [PATCH] chg: [CVE] migrate CVE + get CVEs by daterange --- bin/DB_KVROCKS_MIGRATION.py | 26 ++++++++++-- bin/lib/objects/Cves.py | 42 +++++++++++++------- bin/lib/objects/Pgps.py | 35 ++++++++-------- bin/lib/objects/abstract_daterange_object.py | 6 +-- bin/modules/CveModule.py | 2 +- var/www/blueprints/objects_cve.py | 15 ++++--- 6 files changed, 81 insertions(+), 45 deletions(-) diff --git a/bin/DB_KVROCKS_MIGRATION.py b/bin/DB_KVROCKS_MIGRATION.py index bb05ef2a..12cbcd55 100755 --- a/bin/DB_KVROCKS_MIGRATION.py +++ b/bin/DB_KVROCKS_MIGRATION.py @@ -840,23 +840,43 @@ def statistics_migration(): pass + +############################### +# # +# CVES MIGRATION # +# # +############################### + +from modules.CveModule import CveModule + +def cves_migration(): + module = CveModule() + tag = 'infoleak:automatic-detection="cve"' + first = Tag.get_tag_first_seen(tag) + last = Tag.get_tag_last_seen(tag) + if first and last: + for date in Date.substract_date(first, last): + for item_id in Tag.get_tag_objects(tag, 'item', date=date): + module.compute(f'{item_id} 0') + + if __name__ == '__main__': #core_migration() #user_migration() #tags_migration() - items_migration() + # items_migration() #crawler_migration() # domain_migration() # TO TEST ########################### #decodeds_migration() # screenshots_migration() - subtypes_obj_migration() + # subtypes_obj_migration() # ail_2_ail_migration() # trackers_migration() # investigations_migration() # statistics_migration() - + cves_migration() # custom tags # crawler queues + auto_crawlers diff --git a/bin/lib/objects/Cves.py b/bin/lib/objects/Cves.py index 3e8037dc..16b8e766 100755 --- a/bin/lib/objects/Cves.py +++ b/bin/lib/objects/Cves.py @@ -5,6 +5,7 @@ import os import sys from flask import url_for +from pymisp import MISPObject sys.path.append(os.environ['AIL_BIN']) ################################## @@ -20,12 +21,6 @@ baseurl = config_loader.get_config_str("Notifications", "ail_domain") config_loader = None -################################################################################ -################################################################################ -################################################################################ - -# # TODO: COMPLETE CLASS - class Cve(AbstractDaterangeObject): """ AIL Cve Object. @@ -55,14 +50,21 @@ class Cve(AbstractDaterangeObject): def get_svg_icon(self): return {'style': 'fas', 'icon': '\uf188', 'color': '#1E88E5', 'radius': 5} - # TODO # TODO # TODO # TODO # TODO # TODO # TODO # TODO def get_misp_object(self): - pass + obj_attrs = [] + obj = MISPObject('vulnerability') + obj.first_seen = self.get_first_seen() + obj.last_seen = self.get_last_seen() + + obj_attrs.append(obj.add_attribute('id', value=self.id)) + for obj_attr in obj_attrs: + for tag in self.get_tags(): + obj_attr.add_tag(tag) + return obj def get_meta(self, options=set()): meta = self._get_meta(options=options) meta['id'] = self.id - meta['subtype'] = self.subtype meta['tags'] = self.get_tags(r_list=True) return meta @@ -70,19 +72,21 @@ class Cve(AbstractDaterangeObject): self._add(date, item_id) -# TODO # TODO # TODO # TODO # TODO # TODO # TODO # TODO # TODO # TODO +# TODO ADD SEARCH FUNCTION + def get_all_cves(): - cves = [] - return cves + return r_objects.smembers(f'cve:all') def get_cves_by_date(date): - # return r_objects.zrange(f'cve:date:{date}', 0, -1) - return set(r_objects.hkeys(f'cve:date:{date}')) + return r_objects.zrange(f'cve:date:{date}', 0, -1) + +def get_nb_cves_by_date(date): + return r_objects.zcard(f'cve:date:{date}') def get_cves_by_daterange(date_from, date_to): cves = set() for date in Date.substract_date(date_from, date_to): - cves | get_cves_by_date(date) + cves = cves | set(get_cves_by_date(date)) return cves def get_cves_meta(cves_id, options=set()): @@ -92,6 +96,14 @@ def get_cves_meta(cves_id, options=set()): dict_cve[cve_id] = cve.get_meta(options=options) return dict_cve +def api_get_cves_range_by_daterange(date_from, date_to): + cves = [] + for date in Date.substract_date(date_from, date_to): + d = {'date': f'{date[0:4]}-{date[4:6]}-{date[6:8]}', + 'CVE': get_nb_cves_by_date(date)} + cves.append(d) + return cves + def api_get_cves_meta_by_daterange(date_from, date_to): date = Date.sanitise_date_range(date_from, date_to) return get_cves_meta(get_cves_by_daterange(date['date_from'], date['date_to']), options=['sparkline']) diff --git a/bin/lib/objects/Pgps.py b/bin/lib/objects/Pgps.py index a9a5cf33..cf169850 100755 --- a/bin/lib/objects/Pgps.py +++ b/bin/lib/objects/Pgps.py @@ -3,18 +3,19 @@ import os import sys -import redis -# sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/')) - -sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) -import ConfigLoader - -from lib.objects.abstract_subtype_object import AbstractSubtypeObject, get_all_id from flask import url_for +from pymisp import MISPObject -config_loader = ConfigLoader.ConfigLoader() +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from lib.ConfigLoader import ConfigLoader +from lib.objects.abstract_subtype_object import AbstractSubtypeObject, get_all_id +config_loader = ConfigLoader() +baseurl = config_loader.get_config_str("Notifications", "ail_domain") config_loader = None @@ -64,7 +65,7 @@ class Pgp(AbstractSubtypeObject): icon = '\uf1fa' else: icon = 'times' - return {'style': 'fas', 'icon': icon, 'color': '#44AA99', 'radius':5} + return {'style': 'fas', 'icon': icon, 'color': '#44AA99', 'radius': 5} def get_misp_object(self): obj_attrs = [] @@ -72,12 +73,12 @@ class Pgp(AbstractSubtypeObject): obj.first_seen = self.get_first_seen() obj.last_seen = self.get_last_seen() - if self.subtype=='key': - obj_attrs.append( obj.add_attribute('key-id', value=self.id) ) - elif self.subtype=='name': - obj_attrs.append( obj.add_attribute('user-id-name', value=self.id) ) - else: # mail - obj_attrs.append( obj.add_attribute('user-id-email', value=self.id) ) + if self.subtype == 'key': + obj_attrs.append(obj.add_attribute('key-id', value=self.id)) + elif self.subtype == 'name': + obj_attrs.append(obj.add_attribute('user-id-name', value=self.id)) + else: # mail + obj_attrs.append(obj.add_attribute('user-id-email', value=self.id)) for obj_attr in obj_attrs: for tag in self.get_tags(): @@ -88,7 +89,6 @@ class Pgp(AbstractSubtypeObject): ############################################################################ def get_all_subtypes(): - #return get_object_all_subtypes(self.type) return ['key', 'mail', 'name'] def get_all_pgps(): @@ -101,5 +101,4 @@ def get_all_pgps_by_subtype(subtype): return get_all_id('pgp', subtype) - -#if __name__ == '__main__': +# if __name__ == '__main__': diff --git a/bin/lib/objects/abstract_daterange_object.py b/bin/lib/objects/abstract_daterange_object.py index 2c72570d..fd721da6 100755 --- a/bin/lib/objects/abstract_daterange_object.py +++ b/bin/lib/objects/abstract_daterange_object.py @@ -68,7 +68,7 @@ class AbstractDaterangeObject(AbstractObject, ABC): return r_object.hget(f'meta:{self.type}:{self.id}', 'nb') def get_nb_seen_by_date(self, date): - nb = r_object.hget(f'{self.type}:date:{date}', self.id) + nb = r_object.zscore(f'{self.type}:date:{date}', self.id) if nb is None: return 0 else: @@ -118,8 +118,8 @@ class AbstractDaterangeObject(AbstractObject, ABC): update_obj_date(date, self.type) # NB Object seen by day - r_object.hincrby(f'{self.type}:date:{date}', self.id, 1) - r_object.zincrby(f'{self.type}:date:{date}', 1, self.id) # # # # # # # # # # + print(f'{self.type}:date:{date}', 1, self.id) + r_object.zincrby(f'{self.type}:date:{date}', 1, self.id) # NB Object seen r_object.hincrby(f'meta:{self.type}:{self.id}', 'nb', 1) diff --git a/bin/modules/CveModule.py b/bin/modules/CveModule.py index 35fef165..60c1899f 100755 --- a/bin/modules/CveModule.py +++ b/bin/modules/CveModule.py @@ -51,7 +51,7 @@ class CveModule(AbstractModule): cves = self.regex_findall(self.reg_cve, item_id, item.get_content()) if cves: - print(cves) + # print(cves) date = item.get_date() for cve_id in cves: cve = Cves.Cve(cve_id) diff --git a/var/www/blueprints/objects_cve.py b/var/www/blueprints/objects_cve.py index 4dc098ec..a1178568 100644 --- a/var/www/blueprints/objects_cve.py +++ b/var/www/blueprints/objects_cve.py @@ -41,11 +41,11 @@ def objects_cves(): date_from = date['date_from'] date_to = date['date_to'] - # barchart_type - # correlation_type_search_endpoint + if show_objects: + dict_objects = Cves.api_get_cves_meta_by_daterange(date_from, date_to) + else: + dict_objects = {} - dict_objects = Cves.api_get_cves_meta_by_daterange(date_from, date_to) - print(date_from, date_to, dict_objects) return render_template("CveDaterange.html", date_from=date_from, date_to=date_to, dict_objects=dict_objects, show_objects=show_objects) @@ -62,7 +62,12 @@ def objects_cves_post(): @login_required @login_read_only def objects_cve_range_json(): - return None + date_from = request.args.get('date_from') + date_to = request.args.get('date_to') + date = Date.sanitise_date_range(date_from, date_to) + date_from = date['date_from'] + date_to = date['date_to'] + return jsonify(Cves.api_get_cves_range_by_daterange(date_from, date_to)) @objects_cve.route("/objects/cve/search", methods=['POST']) @login_required