diff --git a/bin/lib/Duplicate.py b/bin/lib/Duplicate.py index 2c597689..adee7bf5 100755 --- a/bin/lib/Duplicate.py +++ b/bin/lib/Duplicate.py @@ -85,18 +85,18 @@ def add_obj_duplicate(algo, similarity, obj_type, subtype, obj_id, id_2): r_serv_db.sadd(f'obj:duplicates:{obj_type}:{subtype}:{obj_id}', f'{similarity}:{algo}:{id_2}') -def add_duplicate(algo, hash_, similarity, obj_type, subtype, id, date_ymonth): +def add_duplicate(algo, hash_, similarity, obj_type, subtype, obj_id, date_ymonth): obj2_id = get_object_id_by_hash(algo, hash_, date_ymonth) # same content if similarity == 100: - dups = get_obj_duplicates(obj_type, subtype, id) + dups = get_obj_duplicates(obj_type, subtype, obj_id) for dup_id in dups: for algo_dict in dups[dup_id]: if algo_dict['similarity'] == 100 and algo_dict['algo'] == algo: - add_obj_duplicate(algo, similarity, obj_type, subtype, id, dups[dup_id]) - add_obj_duplicate(algo, similarity, obj_type, subtype, dups[dup_id], id) - add_obj_duplicate(algo, similarity, obj_type, subtype, id, obj2_id) - add_obj_duplicate(algo, similarity, obj_type, subtype, obj2_id, id) + add_obj_duplicate(algo, similarity, obj_type, subtype, obj_id, dups[dup_id]) + add_obj_duplicate(algo, similarity, obj_type, subtype, dups[dup_id], obj_id) + add_obj_duplicate(algo, similarity, obj_type, subtype, obj_id, obj2_id) + add_obj_duplicate(algo, similarity, obj_type, subtype, obj2_id, obj_id) # TODO def delete_obj_duplicates(): diff --git a/bin/lib/Tag.py b/bin/lib/Tag.py index 619dde67..94b2eca4 100755 --- a/bin/lib/Tag.py +++ b/bin/lib/Tag.py @@ -96,8 +96,6 @@ def get_taxonomies(): def get_active_taxonomies(): return r_tags.smembers('taxonomies:enabled') -'active_taxonomies' - def is_taxonomy_enabled(taxonomy): # enabled = r_tags.sismember('taxonomies:enabled', taxonomy) try: @@ -641,23 +639,23 @@ def get_tag_objects(tag, obj_type, subtype='', date=''): def get_object_tags(obj_type, obj_id, subtype=''): return r_tags.smembers(f'tag:{obj_type}:{subtype}:{obj_id}') -def add_object_tag(tag, obj_type, id, subtype=''): - if r_tags.sadd(f'tag:{obj_type}:{subtype}:{id}', tag) == 1: +def add_object_tag(tag, obj_type, obj_id, subtype=''): + if r_tags.sadd(f'tag:{obj_type}:{subtype}:{obj_id}', tag) == 1: r_tags.sadd('list_tags', tag) r_tags.sadd(f'list_tags:{obj_type}', tag) r_tags.sadd(f'list_tags:{obj_type}:{subtype}', tag) if obj_type == 'item': - date = item_basic.get_item_date(id) - r_tags.sadd(f'{obj_type}:{subtype}:{tag}:{date}', id) + date = item_basic.get_item_date(obj_id) + r_tags.sadd(f'{obj_type}:{subtype}:{tag}:{date}', obj_id) # add domain tag - if item_basic.is_crawled(id) and tag != 'infoleak:submission="crawler"' and tag != 'infoleak:submission="manual"': - domain = item_basic.get_item_domain(id) + if item_basic.is_crawled(obj_id) and tag != 'infoleak:submission="crawler"' and tag != 'infoleak:submission="manual"': + domain = item_basic.get_item_domain(obj_id) add_object_tag(tag, "domain", domain) update_tag_metadata(tag, date) else: - r_tags.sadd(f'{obj_type}:{subtype}:{tag}', id) + r_tags.sadd(f'{obj_type}:{subtype}:{tag}', obj_id) r_tags.hincrby(f'daily_tags:{datetime.date.today().strftime("%Y%m%d")}', tag, 1) diff --git a/bin/lib/objects/CryptoCurrencies.py b/bin/lib/objects/CryptoCurrencies.py index be1e1463..01ff0c5e 100755 --- a/bin/lib/objects/CryptoCurrencies.py +++ b/bin/lib/objects/CryptoCurrencies.py @@ -107,8 +107,15 @@ class CryptoCurrency(AbstractSubtypeObject): def get_misp_object(self): obj_attrs = [] obj = MISPObject('coin-address') - obj.first_seen = self.get_first_seen() - obj.last_seen = self.get_last_seen() + first_seen = self.get_first_seen() + last_seen = self.get_last_seen() + if first_seen: + obj.first_seen = first_seen + if last_seen: + obj.last_seen = last_seen + if not first_seen or not last_seen: + self.logger.warning( + f'Export error, None seen {self.type}:{self.subtype}:{self.id}, first={first_seen}, last={last_seen}') obj_attrs.append(obj.add_attribute('address', value=self.id)) crypto_symbol = self.get_currency_symbol() diff --git a/bin/lib/objects/Cves.py b/bin/lib/objects/Cves.py index 63333ed4..ed550822 100755 --- a/bin/lib/objects/Cves.py +++ b/bin/lib/objects/Cves.py @@ -57,8 +57,15 @@ class Cve(AbstractDaterangeObject): def get_misp_object(self): obj_attrs = [] obj = MISPObject('vulnerability') - obj.first_seen = self.get_first_seen() - obj.last_seen = self.get_last_seen() + first_seen = self.get_first_seen() + last_seen = self.get_last_seen() + if first_seen: + obj.first_seen = first_seen + if last_seen: + obj.last_seen = last_seen + if not first_seen or not last_seen: + self.logger.warning( + f'Export error, None seen {self.type}:{self.subtype}:{self.id}, first={first_seen}, last={last_seen}') obj_attrs.append(obj.add_attribute('id', value=self.id)) for obj_attr in obj_attrs: diff --git a/bin/lib/objects/Decodeds.py b/bin/lib/objects/Decodeds.py index ae776a1f..abe45584 100755 --- a/bin/lib/objects/Decodeds.py +++ b/bin/lib/objects/Decodeds.py @@ -144,8 +144,15 @@ class Decoded(AbstractDaterangeObject): def get_misp_object(self): obj_attrs = [] obj = MISPObject('file') - obj.first_seen = self.get_first_seen() - obj.last_seen = self.get_last_seen() + first_seen = self.get_first_seen() + last_seen = self.get_last_seen() + if first_seen: + obj.first_seen = first_seen + if last_seen: + obj.last_seen = last_seen + if not first_seen or not last_seen: + self.logger.warning( + f'Export error, None seen {self.type}:{self.subtype}:{self.id}, first={first_seen}, last={last_seen}') obj_attrs.append(obj.add_attribute('sha1', value=self.id)) obj_attrs.append(obj.add_attribute('mimetype', value=self.get_mimetype())) diff --git a/bin/lib/objects/Domains.py b/bin/lib/objects/Domains.py index b4b7b6af..811ea6f7 100755 --- a/bin/lib/objects/Domains.py +++ b/bin/lib/objects/Domains.py @@ -344,8 +344,15 @@ class Domain(AbstractObject): # create domain-ip obj obj_attrs = [] obj = MISPObject('domain-crawled', standalone=True) - obj.first_seen = self.get_first_seen() - obj.last_seen = self.get_last_check() + first_seen = self.get_first_seen() + last_seen = self.get_last_check() + if first_seen: + obj.first_seen = first_seen + if last_seen: + obj.last_seen = last_seen + if not first_seen or not last_seen: + self.logger.warning( + f'Export error, None seen {self.type}:{self.subtype}:{self.id}, first={first_seen}, last={last_seen}') obj_attrs.append(obj.add_attribute('domain', value=self.id)) urls = self.get_all_urls(date=True, epoch=epoch) diff --git a/bin/lib/objects/Items.py b/bin/lib/objects/Items.py index 43221284..2e35497e 100755 --- a/bin/lib/objects/Items.py +++ b/bin/lib/objects/Items.py @@ -211,9 +211,13 @@ class Item(AbstractObject): return {'style': '', 'icon': '', 'color': color, 'radius': 5} def get_misp_object(self): - obj_date = self.get_date() obj = MISPObject('ail-leak', standalone=True) - obj.first_seen = obj_date + obj_date = self.get_date() + if obj_date: + obj.first_seen = obj_date + else: + self.logger.warning( + f'Export error, None seen {self.type}:{self.subtype}:{self.id}, first={obj_date}') obj_attrs = [obj.add_attribute('first-seen', value=obj_date), obj.add_attribute('raw-data', value=self.id, data=self.get_raw_content()), diff --git a/bin/lib/objects/Pgps.py b/bin/lib/objects/Pgps.py index 0560c231..81485b06 100755 --- a/bin/lib/objects/Pgps.py +++ b/bin/lib/objects/Pgps.py @@ -71,8 +71,15 @@ class Pgp(AbstractSubtypeObject): def get_misp_object(self): obj_attrs = [] obj = MISPObject('pgp-meta') - obj.first_seen = self.get_first_seen() - obj.last_seen = self.get_last_seen() + first_seen = self.get_first_seen() + last_seen = self.get_last_seen() + if first_seen: + obj.first_seen = first_seen + if last_seen: + obj.last_seen = last_seen + if not first_seen or not last_seen: + self.logger.warning( + f'Export error, None seen {self.type}:{self.subtype}:{self.id}, first={first_seen}, last={last_seen}') if self.subtype == 'key': obj_attrs.append(obj.add_attribute('key-id', value=self.id)) diff --git a/bin/lib/objects/Screenshots.py b/bin/lib/objects/Screenshots.py index efe3e48b..19ae3754 100755 --- a/bin/lib/objects/Screenshots.py +++ b/bin/lib/objects/Screenshots.py @@ -80,8 +80,8 @@ class Screenshot(AbstractObject): obj_attrs = [] obj = MISPObject('file') - obj_attrs.append( obj.add_attribute('sha256', value=self.id) ) - obj_attrs.append( obj.add_attribute('attachment', value=self.id, data=self.get_file_content()) ) + obj_attrs.append(obj.add_attribute('sha256', value=self.id)) + obj_attrs.append(obj.add_attribute('attachment', value=self.id, data=self.get_file_content())) for obj_attr in obj_attrs: for tag in self.get_tags(): obj_attr.add_tag(tag) diff --git a/bin/lib/objects/Titles.py b/bin/lib/objects/Titles.py index 5a8186f0..59db2abe 100755 --- a/bin/lib/objects/Titles.py +++ b/bin/lib/objects/Titles.py @@ -57,8 +57,15 @@ class Title(AbstractDaterangeObject): def get_misp_object(self): obj_attrs = [] obj = MISPObject('tsk-web-history') - obj.first_seen = self.get_first_seen() - obj.last_seen = self.get_last_seen() + first_seen = self.get_first_seen() + last_seen = self.get_last_seen() + if first_seen: + obj.first_seen = first_seen + if last_seen: + obj.last_seen = last_seen + if not first_seen or not last_seen: + self.logger.warning( + f'Export error, None seen {self.type}:{self.subtype}:{self.id}, first={first_seen}, last={last_seen}') obj_attrs.append(obj.add_attribute('title', value=self.get_content())) for obj_attr in obj_attrs: diff --git a/bin/lib/objects/Usernames.py b/bin/lib/objects/Usernames.py index 8770cfc1..c9154b99 100755 --- a/bin/lib/objects/Usernames.py +++ b/bin/lib/objects/Usernames.py @@ -82,8 +82,16 @@ class Username(AbstractSubtypeObject): obj = MISPObject('user-account', standalone=True) obj_attrs.append(obj.add_attribute('username', value=self.id)) - obj.first_seen = self.get_first_seen() - obj.last_seen = self.get_last_seen() + first_seen = self.get_first_seen() + last_seen = self.get_last_seen() + if first_seen: + obj.first_seen = first_seen + if last_seen: + obj.last_seen = last_seen + if not first_seen or not last_seen: + self.logger.warning( + f'Export error, None seen {self.type}:{self.subtype}:{self.id}, first={first_seen}, last={last_seen}') + for obj_attr in obj_attrs: for tag in self.get_tags(): obj_attr.add_tag(tag) diff --git a/bin/lib/objects/abstract_object.py b/bin/lib/objects/abstract_object.py index 631597c4..cb7595ad 100755 --- a/bin/lib/objects/abstract_object.py +++ b/bin/lib/objects/abstract_object.py @@ -7,6 +7,7 @@ Base Class for AIL Objects # Import External packages ################################## import os +import logging.config import sys from abc import ABC, abstractmethod from pymisp import MISPObject @@ -17,23 +18,20 @@ sys.path.append(os.environ['AIL_BIN']) ################################## # Import Project packages ################################## +from lib import ail_logger from lib import Tag from lib import Duplicate from lib.correlations_engine import get_nb_correlations, get_correlations, add_obj_correlation, delete_obj_correlation, delete_obj_correlations, exists_obj_correlation, is_obj_correlated, get_nb_correlation_by_correl_type from lib.Investigations import is_object_investigated, get_obj_investigations, delete_obj_investigations from lib.Tracker import is_obj_tracked, get_obj_trackers, delete_obj_trackers +logging.config.dictConfig(ail_logger.get_config(name='ail')) class AbstractObject(ABC): """ Abstract Object """ - # first seen last/seen ?? - # # TODO: - tags - # - handle + refactor correlations - # - creates others objects - def __init__(self, obj_type, id, subtype=None): """ Abstract for all the AIL object @@ -44,6 +42,8 @@ class AbstractObject(ABC): self.type = obj_type self.subtype = subtype + self.logger = logging.getLogger(f'{self.__class__.__name__}') + def get_id(self): return self.id @@ -74,7 +74,6 @@ class AbstractObject(ABC): tags = list(tags) return tags - ## ADD TAGS ???? def add_tag(self, tag): Tag.add_object_tag(tag, self.type, self.id, subtype=self.get_subtype(r_str=True)) @@ -83,7 +82,7 @@ class AbstractObject(ABC): tags = self.get_tags() return Tag.is_tags_safe(tags) - #- Tags -# + ## -Tags- ## @abstractmethod def get_content(self): @@ -98,10 +97,9 @@ class AbstractObject(ABC): def add_duplicate(self, algo, similarity, id_2): return Duplicate.add_obj_duplicate(algo, similarity, self.type, self.get_subtype(r_str=True), self.id, id_2) - # -Duplicates -# + ## -Duplicates- ## ## Investigations ## - # # TODO: unregister ===== def is_investigated(self): if not self.subtype: @@ -124,7 +122,7 @@ class AbstractObject(ABC): unregistered = delete_obj_investigations(self.id, self.type, self.subtype) return unregistered - #- Investigations -# + ## -Investigations- ## ## Trackers ## @@ -137,7 +135,7 @@ class AbstractObject(ABC): def delete_trackers(self): return delete_obj_trackers(self.type, self.subtype, self.id) - #- Trackers -# + ## -Trackers- ## def _delete(self): # DELETE TAGS @@ -186,15 +184,6 @@ class AbstractObject(ABC): def get_misp_object(self): pass - @staticmethod - def get_misp_object_first_last_seen(misp_obj): # TODO REMOVE ME ???? - """ - :type misp_obj: MISPObject - """ - first_seen = misp_obj.get('first_seen') - last_seen = misp_obj.get('last_seen') - return first_seen, last_seen - @staticmethod def get_misp_object_tags(misp_obj): """ @@ -264,8 +253,3 @@ class AbstractObject(ABC): Get object correlations """ delete_obj_correlation(self.type, self.subtype, self.id, type2, subtype2, id2) - - - # # TODO: get favicon - # # TODO: get url - # # TODO: get metadata