CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

fix: [MISP export] fix ail object first/last seen + obj logger

pull/604/head
Terrtia 2023-06-09 11:19:22 +02:00
parent f540df0ff2
commit 580879ee5c
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
12 changed files with 94 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
bin/lib/Duplicate.py
View File

@ -85,18 +85,18 @@ def add_obj_duplicate(algo, similarity, obj_type, subtype, obj_id, id_2):
r_serv_db.sadd(f'obj:duplicates:{obj_type}:{subtype}:{obj_id}', f'{similarity}:{algo}:{id_2}')
def add_duplicate(algo, hash_, similarity, obj_type, subtype, id, date_ymonth):
def add_duplicate(algo, hash_, similarity, obj_type, subtype, obj_id, date_ymonth):
obj2_id = get_object_id_by_hash(algo, hash_, date_ymonth)
# same content
if similarity == 100:
dups = get_obj_duplicates(obj_type, subtype, id)
dups = get_obj_duplicates(obj_type, subtype, obj_id)
for dup_id in dups:
for algo_dict in dups[dup_id]:
if algo_dict['similarity'] == 100 and algo_dict['algo'] == algo:
add_obj_duplicate(algo, similarity, obj_type, subtype, id, dups[dup_id])
add_obj_duplicate(algo, similarity, obj_type, subtype, dups[dup_id], id)
add_obj_duplicate(algo, similarity, obj_type, subtype, id, obj2_id)
add_obj_duplicate(algo, similarity, obj_type, subtype, obj2_id, id)
add_obj_duplicate(algo, similarity, obj_type, subtype, obj_id, dups[dup_id])
add_obj_duplicate(algo, similarity, obj_type, subtype, dups[dup_id], obj_id)
add_obj_duplicate(algo, similarity, obj_type, subtype, obj_id, obj2_id)
add_obj_duplicate(algo, similarity, obj_type, subtype, obj2_id, obj_id)
# TODO
def delete_obj_duplicates():

16
bin/lib/Tag.py
View File

@ -96,8 +96,6 @@ def get_taxonomies():
def get_active_taxonomies():
return r_tags.smembers('taxonomies:enabled')
'active_taxonomies'
def is_taxonomy_enabled(taxonomy):
# enabled = r_tags.sismember('taxonomies:enabled', taxonomy)
try:
@ -641,23 +639,23 @@ def get_tag_objects(tag, obj_type, subtype='', date=''):
def get_object_tags(obj_type, obj_id, subtype=''):
return r_tags.smembers(f'tag:{obj_type}:{subtype}:{obj_id}')
def add_object_tag(tag, obj_type, id, subtype=''):
if r_tags.sadd(f'tag:{obj_type}:{subtype}:{id}', tag) == 1:
def add_object_tag(tag, obj_type, obj_id, subtype=''):
if r_tags.sadd(f'tag:{obj_type}:{subtype}:{obj_id}', tag) == 1:
r_tags.sadd('list_tags', tag)
r_tags.sadd(f'list_tags:{obj_type}', tag)
r_tags.sadd(f'list_tags:{obj_type}:{subtype}', tag)
if obj_type == 'item':
date = item_basic.get_item_date(id)
r_tags.sadd(f'{obj_type}:{subtype}:{tag}:{date}', id)
date = item_basic.get_item_date(obj_id)
r_tags.sadd(f'{obj_type}:{subtype}:{tag}:{date}', obj_id)
# add domain tag
if item_basic.is_crawled(id) and tag != 'infoleak:submission="crawler"' and tag != 'infoleak:submission="manual"':
domain = item_basic.get_item_domain(id)
if item_basic.is_crawled(obj_id) and tag != 'infoleak:submission="crawler"' and tag != 'infoleak:submission="manual"':
domain = item_basic.get_item_domain(obj_id)
add_object_tag(tag, "domain", domain)
update_tag_metadata(tag, date)
else:
r_tags.sadd(f'{obj_type}:{subtype}:{tag}', id)
r_tags.sadd(f'{obj_type}:{subtype}:{tag}', obj_id)
r_tags.hincrby(f'daily_tags:{datetime.date.today().strftime("%Y%m%d")}', tag, 1)

11
bin/lib/objects/CryptoCurrencies.py
View File

@ -107,8 +107,15 @@ class CryptoCurrency(AbstractSubtypeObject):
def get_misp_object(self):
obj_attrs = []
obj = MISPObject('coin-address')
obj.first_seen = self.get_first_seen()
obj.last_seen = self.get_last_seen()
first_seen = self.get_first_seen()
last_seen = self.get_last_seen()
if first_seen:
obj.first_seen = first_seen
if last_seen:
obj.last_seen = last_seen
if not first_seen or not last_seen:
self.logger.warning(
f'Export error, None seen {self.type}:{self.subtype}:{self.id}, first={first_seen}, last={last_seen}')
obj_attrs.append(obj.add_attribute('address', value=self.id))
crypto_symbol = self.get_currency_symbol()

11
bin/lib/objects/Cves.py
View File

@ -57,8 +57,15 @@ class Cve(AbstractDaterangeObject):
def get_misp_object(self):
obj_attrs = []
obj = MISPObject('vulnerability')
obj.first_seen = self.get_first_seen()
obj.last_seen = self.get_last_seen()
first_seen = self.get_first_seen()
last_seen = self.get_last_seen()
if first_seen:
obj.first_seen = first_seen
if last_seen:
obj.last_seen = last_seen
if not first_seen or not last_seen:
self.logger.warning(
f'Export error, None seen {self.type}:{self.subtype}:{self.id}, first={first_seen}, last={last_seen}')
obj_attrs.append(obj.add_attribute('id', value=self.id))
for obj_attr in obj_attrs:

11
bin/lib/objects/Decodeds.py
View File

@ -144,8 +144,15 @@ class Decoded(AbstractDaterangeObject):
def get_misp_object(self):
obj_attrs = []
obj = MISPObject('file')
obj.first_seen = self.get_first_seen()
obj.last_seen = self.get_last_seen()
first_seen = self.get_first_seen()
last_seen = self.get_last_seen()
if first_seen:
obj.first_seen = first_seen
if last_seen:
obj.last_seen = last_seen
if not first_seen or not last_seen:
self.logger.warning(
f'Export error, None seen {self.type}:{self.subtype}:{self.id}, first={first_seen}, last={last_seen}')
obj_attrs.append(obj.add_attribute('sha1', value=self.id))
obj_attrs.append(obj.add_attribute('mimetype', value=self.get_mimetype()))

11
bin/lib/objects/Domains.py
View File

@ -344,8 +344,15 @@ class Domain(AbstractObject):
# create domain-ip obj
obj_attrs = []
obj = MISPObject('domain-crawled', standalone=True)
obj.first_seen = self.get_first_seen()
obj.last_seen = self.get_last_check()
first_seen = self.get_first_seen()
last_seen = self.get_last_check()
if first_seen:
obj.first_seen = first_seen
if last_seen:
obj.last_seen = last_seen
if not first_seen or not last_seen:
self.logger.warning(
f'Export error, None seen {self.type}:{self.subtype}:{self.id}, first={first_seen}, last={last_seen}')
obj_attrs.append(obj.add_attribute('domain', value=self.id))
urls = self.get_all_urls(date=True, epoch=epoch)

6
bin/lib/objects/Items.py
View File

@ -211,9 +211,13 @@ class Item(AbstractObject):
return {'style': '', 'icon': '', 'color': color, 'radius': 5}
def get_misp_object(self):
obj_date = self.get_date()
obj = MISPObject('ail-leak', standalone=True)
obj_date = self.get_date()
if obj_date:
obj.first_seen = obj_date
else:
self.logger.warning(
f'Export error, None seen {self.type}:{self.subtype}:{self.id}, first={obj_date}')
obj_attrs = [obj.add_attribute('first-seen', value=obj_date),
obj.add_attribute('raw-data', value=self.id, data=self.get_raw_content()),

11
bin/lib/objects/Pgps.py
View File

@ -71,8 +71,15 @@ class Pgp(AbstractSubtypeObject):
def get_misp_object(self):
obj_attrs = []
obj = MISPObject('pgp-meta')
obj.first_seen = self.get_first_seen()
obj.last_seen = self.get_last_seen()
first_seen = self.get_first_seen()
last_seen = self.get_last_seen()
if first_seen:
obj.first_seen = first_seen
if last_seen:
obj.last_seen = last_seen
if not first_seen or not last_seen:
self.logger.warning(
f'Export error, None seen {self.type}:{self.subtype}:{self.id}, first={first_seen}, last={last_seen}')
if self.subtype == 'key':
obj_attrs.append(obj.add_attribute('key-id', value=self.id))

11
bin/lib/objects/Titles.py
View File

@ -57,8 +57,15 @@ class Title(AbstractDaterangeObject):
def get_misp_object(self):
obj_attrs = []
obj = MISPObject('tsk-web-history')
obj.first_seen = self.get_first_seen()
obj.last_seen = self.get_last_seen()
first_seen = self.get_first_seen()
last_seen = self.get_last_seen()
if first_seen:
obj.first_seen = first_seen
if last_seen:
obj.last_seen = last_seen
if not first_seen or not last_seen:
self.logger.warning(
f'Export error, None seen {self.type}:{self.subtype}:{self.id}, first={first_seen}, last={last_seen}')
obj_attrs.append(obj.add_attribute('title', value=self.get_content()))
for obj_attr in obj_attrs:

12
bin/lib/objects/Usernames.py
View File

@ -82,8 +82,16 @@ class Username(AbstractSubtypeObject):
obj = MISPObject('user-account', standalone=True)
obj_attrs.append(obj.add_attribute('username', value=self.id))
obj.first_seen = self.get_first_seen()
obj.last_seen = self.get_last_seen()
first_seen = self.get_first_seen()
last_seen = self.get_last_seen()
if first_seen:
obj.first_seen = first_seen
if last_seen:
obj.last_seen = last_seen
if not first_seen or not last_seen:
self.logger.warning(
f'Export error, None seen {self.type}:{self.subtype}:{self.id}, first={first_seen}, last={last_seen}')
for obj_attr in obj_attrs:
for tag in self.get_tags():
obj_attr.add_tag(tag)

34
bin/lib/objects/abstract_object.py
View File

@ -7,6 +7,7 @@ Base Class for AIL Objects
# Import External packages
##################################
import os
import logging.config
import sys
from abc import ABC, abstractmethod
from pymisp import MISPObject
@ -17,23 +18,20 @@ sys.path.append(os.environ['AIL_BIN'])
##################################
# Import Project packages
##################################
from lib import ail_logger
from lib import Tag
from lib import Duplicate
from lib.correlations_engine import get_nb_correlations, get_correlations, add_obj_correlation, delete_obj_correlation, delete_obj_correlations, exists_obj_correlation, is_obj_correlated, get_nb_correlation_by_correl_type
from lib.Investigations import is_object_investigated, get_obj_investigations, delete_obj_investigations
from lib.Tracker import is_obj_tracked, get_obj_trackers, delete_obj_trackers
logging.config.dictConfig(ail_logger.get_config(name='ail'))
class AbstractObject(ABC):
"""
Abstract Object
"""
# first seen last/seen ??
# # TODO: - tags
# - handle + refactor correlations
# - creates others objects
def __init__(self, obj_type, id, subtype=None):
""" Abstract for all the AIL object
@ -44,6 +42,8 @@ class AbstractObject(ABC):
self.type = obj_type
self.subtype = subtype
self.logger = logging.getLogger(f'{self.__class__.__name__}')
def get_id(self):
return self.id
@ -74,7 +74,6 @@ class AbstractObject(ABC):
tags = list(tags)
return tags
## ADD TAGS ????
def add_tag(self, tag):
Tag.add_object_tag(tag, self.type, self.id, subtype=self.get_subtype(r_str=True))
@ -83,7 +82,7 @@ class AbstractObject(ABC):
tags = self.get_tags()
return Tag.is_tags_safe(tags)
#- Tags -#
## -Tags- ##
@abstractmethod
def get_content(self):
@ -98,10 +97,9 @@ class AbstractObject(ABC):
def add_duplicate(self, algo, similarity, id_2):
return Duplicate.add_obj_duplicate(algo, similarity, self.type, self.get_subtype(r_str=True), self.id, id_2)
# -Duplicates -#
## -Duplicates- ##
## Investigations ##
# # TODO: unregister =====
def is_investigated(self):
if not self.subtype:
@ -124,7 +122,7 @@ class AbstractObject(ABC):
unregistered = delete_obj_investigations(self.id, self.type, self.subtype)
return unregistered
#- Investigations -#
## -Investigations- ##
## Trackers ##
@ -137,7 +135,7 @@ class AbstractObject(ABC):
def delete_trackers(self):
return delete_obj_trackers(self.type, self.subtype, self.id)
#- Trackers -#
## -Trackers- ##
def _delete(self):
# DELETE TAGS
@ -186,15 +184,6 @@ class AbstractObject(ABC):
def get_misp_object(self):
pass
@staticmethod
def get_misp_object_first_last_seen(misp_obj): # TODO REMOVE ME ????
"""
:type misp_obj: MISPObject
"""
first_seen = misp_obj.get('first_seen')
last_seen = misp_obj.get('last_seen')
return first_seen, last_seen
@staticmethod
def get_misp_object_tags(misp_obj):
"""
@ -264,8 +253,3 @@ class AbstractObject(ABC):
Get object correlations
"""
delete_obj_correlation(self.type, self.subtype, self.id, type2, subtype2, id2)
# # TODO: get favicon
# # TODO: get url
# # TODO: get metadata