fix: [MISP export] fix ail object first/last seen + obj logger

bin/lib/Duplicate.py

@@ -85,18 +85,18 @@ def add_obj_duplicate(algo, similarity, obj_type, subtype, obj_id, id_2):
     r_serv_db.sadd(f'obj:duplicates:{obj_type}:{subtype}:{obj_id}', f'{similarity}:{algo}:{id_2}')
 
 
-def add_duplicate(algo, hash_, similarity, obj_type, subtype, id, date_ymonth):
+def add_duplicate(algo, hash_, similarity, obj_type, subtype, obj_id, date_ymonth):
     obj2_id = get_object_id_by_hash(algo, hash_, date_ymonth)
     # same content
     if similarity == 100:
-        dups = get_obj_duplicates(obj_type, subtype, id)
+        dups = get_obj_duplicates(obj_type, subtype, obj_id)
         for dup_id in dups:
             for algo_dict in dups[dup_id]:
                 if algo_dict['similarity'] == 100 and algo_dict['algo'] == algo:
-                    add_obj_duplicate(algo, similarity, obj_type, subtype, id, dups[dup_id])
+                    add_obj_duplicate(algo, similarity, obj_type, subtype, obj_id, dups[dup_id])
-                    add_obj_duplicate(algo, similarity, obj_type, subtype, dups[dup_id], id)
+                    add_obj_duplicate(algo, similarity, obj_type, subtype, dups[dup_id], obj_id)
-    add_obj_duplicate(algo, similarity, obj_type, subtype, id, obj2_id)
+    add_obj_duplicate(algo, similarity, obj_type, subtype, obj_id, obj2_id)
-    add_obj_duplicate(algo, similarity, obj_type, subtype, obj2_id, id)
+    add_obj_duplicate(algo, similarity, obj_type, subtype, obj2_id, obj_id)
 
 # TODO
 def delete_obj_duplicates():

bin/lib/Tag.py

@@ -96,8 +96,6 @@ def get_taxonomies():
 def get_active_taxonomies():
     return r_tags.smembers('taxonomies:enabled')
 
-'active_taxonomies'
-
 def is_taxonomy_enabled(taxonomy):
     # enabled = r_tags.sismember('taxonomies:enabled', taxonomy)
     try:
@@ -641,23 +639,23 @@ def get_tag_objects(tag, obj_type, subtype='', date=''):
 def get_object_tags(obj_type, obj_id, subtype=''):
     return r_tags.smembers(f'tag:{obj_type}:{subtype}:{obj_id}')
 
-def add_object_tag(tag, obj_type, id, subtype=''):
+def add_object_tag(tag, obj_type, obj_id, subtype=''):
-    if r_tags.sadd(f'tag:{obj_type}:{subtype}:{id}', tag) == 1:
+    if r_tags.sadd(f'tag:{obj_type}:{subtype}:{obj_id}', tag) == 1:
         r_tags.sadd('list_tags', tag)
         r_tags.sadd(f'list_tags:{obj_type}', tag)
         r_tags.sadd(f'list_tags:{obj_type}:{subtype}', tag)
         if obj_type == 'item':
-            date = item_basic.get_item_date(id)
+            date = item_basic.get_item_date(obj_id)
-            r_tags.sadd(f'{obj_type}:{subtype}:{tag}:{date}', id)
+            r_tags.sadd(f'{obj_type}:{subtype}:{tag}:{date}', obj_id)
 
             # add domain tag
-            if item_basic.is_crawled(id) and tag != 'infoleak:submission="crawler"' and tag != 'infoleak:submission="manual"':
+            if item_basic.is_crawled(obj_id) and tag != 'infoleak:submission="crawler"' and tag != 'infoleak:submission="manual"':
-                domain = item_basic.get_item_domain(id)
+                domain = item_basic.get_item_domain(obj_id)
                 add_object_tag(tag, "domain", domain)
 
             update_tag_metadata(tag, date)
         else:
-            r_tags.sadd(f'{obj_type}:{subtype}:{tag}', id)
+            r_tags.sadd(f'{obj_type}:{subtype}:{tag}', obj_id)
 
         r_tags.hincrby(f'daily_tags:{datetime.date.today().strftime("%Y%m%d")}', tag, 1)
 

bin/lib/objects/CryptoCurrencies.py

@@ -107,8 +107,15 @@ class CryptoCurrency(AbstractSubtypeObject):
     def get_misp_object(self):
         obj_attrs = []
         obj = MISPObject('coin-address')
-        obj.first_seen = self.get_first_seen()
+        first_seen = self.get_first_seen()
-        obj.last_seen = self.get_last_seen()
+        last_seen = self.get_last_seen()
+        if first_seen:
+            obj.first_seen = first_seen
+        if last_seen:
+            obj.last_seen = last_seen
+        if not first_seen or not last_seen:
+            self.logger.warning(
+                f'Export error, None seen {self.type}:{self.subtype}:{self.id}, first={first_seen}, last={last_seen}')
 
         obj_attrs.append(obj.add_attribute('address', value=self.id))
         crypto_symbol = self.get_currency_symbol()

bin/lib/objects/Cves.py

@@ -57,8 +57,15 @@ class Cve(AbstractDaterangeObject):
     def get_misp_object(self):
         obj_attrs = []
         obj = MISPObject('vulnerability')
-        obj.first_seen = self.get_first_seen()
+        first_seen = self.get_first_seen()
-        obj.last_seen = self.get_last_seen()
+        last_seen = self.get_last_seen()
+        if first_seen:
+            obj.first_seen = first_seen
+        if last_seen:
+            obj.last_seen = last_seen
+        if not first_seen or not last_seen:
+            self.logger.warning(
+                f'Export error, None seen {self.type}:{self.subtype}:{self.id}, first={first_seen}, last={last_seen}')
 
         obj_attrs.append(obj.add_attribute('id', value=self.id))
         for obj_attr in obj_attrs:

bin/lib/objects/Decodeds.py

@@ -144,8 +144,15 @@ class Decoded(AbstractDaterangeObject):
     def get_misp_object(self):
         obj_attrs = []
         obj = MISPObject('file')
-        obj.first_seen = self.get_first_seen()
+        first_seen = self.get_first_seen()
-        obj.last_seen = self.get_last_seen()
+        last_seen = self.get_last_seen()
+        if first_seen:
+            obj.first_seen = first_seen
+        if last_seen:
+            obj.last_seen = last_seen
+        if not first_seen or not last_seen:
+            self.logger.warning(
+                f'Export error, None seen {self.type}:{self.subtype}:{self.id}, first={first_seen}, last={last_seen}')
 
         obj_attrs.append(obj.add_attribute('sha1', value=self.id))
         obj_attrs.append(obj.add_attribute('mimetype', value=self.get_mimetype()))

bin/lib/objects/Domains.py

@@ -344,8 +344,15 @@ class Domain(AbstractObject):
         # create domain-ip obj
         obj_attrs = []
         obj = MISPObject('domain-crawled', standalone=True)
-        obj.first_seen = self.get_first_seen()
+        first_seen = self.get_first_seen()
-        obj.last_seen = self.get_last_check()
+        last_seen = self.get_last_check()
+        if first_seen:
+            obj.first_seen = first_seen
+        if last_seen:
+            obj.last_seen = last_seen
+        if not first_seen or not last_seen:
+            self.logger.warning(
+                f'Export error, None seen {self.type}:{self.subtype}:{self.id}, first={first_seen}, last={last_seen}')
 
         obj_attrs.append(obj.add_attribute('domain', value=self.id))
         urls = self.get_all_urls(date=True, epoch=epoch)

bin/lib/objects/Items.py

@@ -211,9 +211,13 @@ class Item(AbstractObject):
         return {'style': '', 'icon': '', 'color': color, 'radius': 5}
 
     def get_misp_object(self):
-        obj_date = self.get_date()
         obj = MISPObject('ail-leak', standalone=True)
+        obj_date = self.get_date()
+        if obj_date:
             obj.first_seen = obj_date
+        else:
+            self.logger.warning(
+                f'Export error, None seen {self.type}:{self.subtype}:{self.id}, first={obj_date}')
 
         obj_attrs = [obj.add_attribute('first-seen', value=obj_date),
                      obj.add_attribute('raw-data', value=self.id, data=self.get_raw_content()),

bin/lib/objects/Pgps.py

@@ -71,8 +71,15 @@ class Pgp(AbstractSubtypeObject):
     def get_misp_object(self):
         obj_attrs = []
         obj = MISPObject('pgp-meta')
-        obj.first_seen = self.get_first_seen()
+        first_seen = self.get_first_seen()
-        obj.last_seen = self.get_last_seen()
+        last_seen = self.get_last_seen()
+        if first_seen:
+            obj.first_seen = first_seen
+        if last_seen:
+            obj.last_seen = last_seen
+        if not first_seen or not last_seen:
+            self.logger.warning(
+                f'Export error, None seen {self.type}:{self.subtype}:{self.id}, first={first_seen}, last={last_seen}')
 
         if self.subtype == 'key':
             obj_attrs.append(obj.add_attribute('key-id', value=self.id))

bin/lib/objects/Screenshots.py

@@ -80,8 +80,8 @@ class Screenshot(AbstractObject):
         obj_attrs = []
         obj = MISPObject('file')
 
-        obj_attrs.append( obj.add_attribute('sha256', value=self.id) )
+        obj_attrs.append(obj.add_attribute('sha256', value=self.id))
-        obj_attrs.append( obj.add_attribute('attachment', value=self.id, data=self.get_file_content()) )
+        obj_attrs.append(obj.add_attribute('attachment', value=self.id, data=self.get_file_content()))
         for obj_attr in obj_attrs:
             for tag in self.get_tags():
                 obj_attr.add_tag(tag)

bin/lib/objects/Titles.py

@@ -57,8 +57,15 @@ class Title(AbstractDaterangeObject):
     def get_misp_object(self):
         obj_attrs = []
         obj = MISPObject('tsk-web-history')
-        obj.first_seen = self.get_first_seen()
+        first_seen = self.get_first_seen()
-        obj.last_seen = self.get_last_seen()
+        last_seen = self.get_last_seen()
+        if first_seen:
+            obj.first_seen = first_seen
+        if last_seen:
+            obj.last_seen = last_seen
+        if not first_seen or not last_seen:
+            self.logger.warning(
+                f'Export error, None seen {self.type}:{self.subtype}:{self.id}, first={first_seen}, last={last_seen}')
 
         obj_attrs.append(obj.add_attribute('title', value=self.get_content()))
         for obj_attr in obj_attrs:

bin/lib/objects/Usernames.py

@@ -82,8 +82,16 @@ class Username(AbstractSubtypeObject):
             obj = MISPObject('user-account', standalone=True)
             obj_attrs.append(obj.add_attribute('username', value=self.id))
 
-        obj.first_seen = self.get_first_seen()
+        first_seen = self.get_first_seen()
-        obj.last_seen = self.get_last_seen()
+        last_seen = self.get_last_seen()
+        if first_seen:
+            obj.first_seen = first_seen
+        if last_seen:
+            obj.last_seen = last_seen
+        if not first_seen or not last_seen:
+            self.logger.warning(
+                f'Export error, None seen {self.type}:{self.subtype}:{self.id}, first={first_seen}, last={last_seen}')
+
         for obj_attr in obj_attrs:
             for tag in self.get_tags():
                 obj_attr.add_tag(tag)

bin/lib/objects/abstract_object.py

@@ -7,6 +7,7 @@ Base Class for AIL Objects
 # Import External packages
 ##################################
 import os
+import logging.config
 import sys
 from abc import ABC, abstractmethod
 from pymisp import MISPObject
@@ -17,23 +18,20 @@ sys.path.append(os.environ['AIL_BIN'])
 ##################################
 # Import Project packages
 ##################################
+from lib import ail_logger
 from lib import Tag
 from lib import Duplicate
 from lib.correlations_engine import get_nb_correlations, get_correlations, add_obj_correlation, delete_obj_correlation, delete_obj_correlations, exists_obj_correlation, is_obj_correlated, get_nb_correlation_by_correl_type
 from lib.Investigations import is_object_investigated, get_obj_investigations, delete_obj_investigations
 from lib.Tracker import is_obj_tracked, get_obj_trackers, delete_obj_trackers
 
+logging.config.dictConfig(ail_logger.get_config(name='ail'))
 
 class AbstractObject(ABC):
     """
     Abstract Object
     """
 
-    # first seen last/seen ??
-    # # TODO: - tags
-    #         - handle + refactor correlations
-    #         - creates others objects
-
     def __init__(self, obj_type, id, subtype=None):
         """ Abstract for all the AIL object
 
@@ -44,6 +42,8 @@ class AbstractObject(ABC):
         self.type = obj_type
         self.subtype = subtype
 
+        self.logger = logging.getLogger(f'{self.__class__.__name__}')
+
     def get_id(self):
         return self.id
 
@@ -74,7 +74,6 @@ class AbstractObject(ABC):
             tags = list(tags)
         return tags
 
-    ## ADD TAGS ????
     def add_tag(self, tag):
         Tag.add_object_tag(tag, self.type, self.id, subtype=self.get_subtype(r_str=True))
 
@@ -83,7 +82,7 @@ class AbstractObject(ABC):
             tags = self.get_tags()
         return Tag.is_tags_safe(tags)
 
-    #- Tags -#
+    ## -Tags- ##
 
     @abstractmethod
     def get_content(self):
@@ -98,10 +97,9 @@ class AbstractObject(ABC):
 
     def add_duplicate(self, algo, similarity, id_2):
         return Duplicate.add_obj_duplicate(algo, similarity, self.type, self.get_subtype(r_str=True), self.id, id_2)
-    # -Duplicates -#
+    ## -Duplicates- ##
 
     ## Investigations ##
-    # # TODO: unregister =====
 
     def is_investigated(self):
         if not self.subtype:
@@ -124,7 +122,7 @@ class AbstractObject(ABC):
             unregistered = delete_obj_investigations(self.id, self.type, self.subtype)
         return unregistered
 
-    #- Investigations -#
+    ## -Investigations- ##
 
     ## Trackers ##
 
@@ -137,7 +135,7 @@ class AbstractObject(ABC):
     def delete_trackers(self):
         return delete_obj_trackers(self.type, self.subtype, self.id)
 
-    #- Trackers -#
+    ## -Trackers- ##
 
     def _delete(self):
         # DELETE TAGS
@@ -186,15 +184,6 @@ class AbstractObject(ABC):
     def get_misp_object(self):
         pass
 
-    @staticmethod
-    def get_misp_object_first_last_seen(misp_obj): # TODO REMOVE ME ????
-        """
-        :type misp_obj: MISPObject
-        """
-        first_seen = misp_obj.get('first_seen')
-        last_seen = misp_obj.get('last_seen')
-        return first_seen, last_seen
-
     @staticmethod
     def get_misp_object_tags(misp_obj):
         """
@@ -264,8 +253,3 @@ class AbstractObject(ABC):
         Get object correlations
         """
         delete_obj_correlation(self.type, self.subtype, self.id, type2, subtype2, id2)
-
-
-    # # TODO: get favicon
-    # # TODO: get url
-    # # TODO: get metadata