2019-07-30 13:49:21 +02:00
|
|
|
#!/usr/bin/env python3
|
|
|
|
# -*-coding:UTF-8 -*
|
|
|
|
|
|
|
|
import os
|
2019-10-29 16:52:33 +01:00
|
|
|
import sys
|
2019-07-30 13:49:21 +02:00
|
|
|
import redis
|
2020-01-06 17:07:52 +01:00
|
|
|
import datetime
|
2019-07-30 13:49:21 +02:00
|
|
|
|
2020-01-07 16:14:56 +01:00
|
|
|
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
|
2019-07-30 13:49:21 +02:00
|
|
|
import Date
|
|
|
|
|
2019-10-29 16:52:33 +01:00
|
|
|
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
|
|
|
import ConfigLoader
|
2020-06-19 13:36:03 +02:00
|
|
|
import item_basic
|
2019-10-29 16:52:33 +01:00
|
|
|
|
2019-07-30 13:49:21 +02:00
|
|
|
from pytaxonomies import Taxonomies
|
|
|
|
from pymispgalaxies import Galaxies, Clusters
|
|
|
|
|
2019-10-29 16:52:33 +01:00
|
|
|
config_loader = ConfigLoader.ConfigLoader()
|
|
|
|
r_serv_tags = config_loader.get_redis_conn("ARDB_Tags")
|
|
|
|
r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
|
|
|
|
config_loader = None
|
2019-07-30 13:49:21 +02:00
|
|
|
|
2019-11-19 14:03:23 +01:00
|
|
|
def build_unsafe_tags():
|
|
|
|
unsafe_tags = set()
|
|
|
|
## CE content
|
|
|
|
unsafe_tags.add('dark-web:topic="pornography-child-exploitation"')
|
|
|
|
# add copine-scale tags
|
|
|
|
taxonomies = Taxonomies()
|
|
|
|
copine_scale = taxonomies.get('copine-scale')
|
|
|
|
if copine_scale:
|
|
|
|
for tag in copine_scale.machinetags():
|
|
|
|
unsafe_tags.add(tag)
|
|
|
|
return unsafe_tags
|
|
|
|
|
|
|
|
# set of unsafe tags
|
|
|
|
unsafe_tags = build_unsafe_tags()
|
|
|
|
|
2020-01-06 17:07:52 +01:00
|
|
|
def is_tags_safe(ltags):
|
|
|
|
'''
|
|
|
|
Check if a list of tags contain an unsafe tag (CE, ...)
|
|
|
|
|
|
|
|
:param ltags: list of tags
|
|
|
|
:type ltags: list
|
|
|
|
:return: is a tag in the unsafe set
|
|
|
|
:rtype: boolean
|
|
|
|
'''
|
|
|
|
return unsafe_tags.isdisjoint(ltags)
|
|
|
|
|
|
|
|
#### Taxonomies - Galaxies ####
|
|
|
|
|
2019-07-30 13:49:21 +02:00
|
|
|
def get_taxonomie_from_tag(tag):
|
2021-04-26 15:08:36 +02:00
|
|
|
try:
|
|
|
|
return tag.split(':')[0]
|
|
|
|
except IndexError:
|
|
|
|
return None
|
2019-07-30 13:49:21 +02:00
|
|
|
|
|
|
|
def get_galaxy_from_tag(tag):
|
2021-04-26 15:08:36 +02:00
|
|
|
try:
|
|
|
|
galaxy = tag.split(':')[1]
|
|
|
|
galaxy = galaxy.split('=')[0]
|
|
|
|
return galaxy
|
|
|
|
except IndexError:
|
|
|
|
return None
|
2019-07-30 13:49:21 +02:00
|
|
|
|
2021-11-22 23:45:41 +01:00
|
|
|
def get_active_taxonomies(r_set=False):
|
|
|
|
res = r_serv_tags.smembers('active_taxonomies')
|
|
|
|
if r_set:
|
|
|
|
return set(res)
|
|
|
|
return res
|
2019-07-30 13:49:21 +02:00
|
|
|
|
2021-11-22 23:45:41 +01:00
|
|
|
def get_active_galaxies(r_set=False):
|
|
|
|
res = r_serv_tags.smembers('active_galaxies')
|
|
|
|
if r_set:
|
|
|
|
return set(res)
|
|
|
|
return res
|
2019-07-30 13:49:21 +02:00
|
|
|
|
2020-01-08 17:11:37 +01:00
|
|
|
def get_all_taxonomies_tags(): # # TODO: add + REMOVE + Update
|
|
|
|
return r_serv_tags.smembers('active_taxonomies_tags')
|
|
|
|
|
|
|
|
def get_all_galaxies_tags(): # # TODO: add + REMOVE + Update
|
|
|
|
return r_serv_tags.smembers('active_galaxies_tags')
|
|
|
|
|
2021-11-22 23:45:41 +01:00
|
|
|
def get_taxonomies_enabled_tags(r_list=False):
|
|
|
|
l_tag_keys = []
|
|
|
|
for taxonomie in get_active_taxonomies():
|
|
|
|
l_tag_keys.append(f'active_tag_{taxonomie}')
|
|
|
|
if len(l_tag_keys) > 1:
|
|
|
|
res = r_serv_tags.sunion(l_tag_keys[0], *l_tag_keys[1:])
|
|
|
|
elif l_tag_keys:
|
|
|
|
res = r_serv_tags.smembers(l_tag_keys[0])
|
|
|
|
if r_list:
|
|
|
|
return list(res)
|
|
|
|
else:
|
|
|
|
return res
|
|
|
|
|
|
|
|
def get_galaxies_enabled_tags():
|
|
|
|
l_tag_keys = []
|
|
|
|
for galaxy in get_active_galaxies():
|
|
|
|
l_tag_keys.append(f'active_tag_galaxies_{galaxy}')
|
|
|
|
if len(l_tag_keys) > 1:
|
|
|
|
return r_serv_tags.sunion(l_tag_keys[0], *l_tag_keys[1:])
|
|
|
|
elif l_tag_keys:
|
|
|
|
return r_serv_tags.smembers(l_tag_keys[0])
|
|
|
|
else:
|
|
|
|
return []
|
|
|
|
|
|
|
|
def get_taxonomie_enabled_tags(taxonomie, r_list=False):
|
|
|
|
res = r_serv_tags.smembers(f'active_tag_{taxonomie}')
|
|
|
|
if r_list:
|
|
|
|
return list(res)
|
|
|
|
else:
|
|
|
|
return res
|
|
|
|
|
|
|
|
def get_galaxy_enabled_tags(galaxy, r_list=False):
|
|
|
|
res = r_serv_tags.smembers(f'active_tag_galaxies_{galaxy}')
|
|
|
|
if r_list:
|
|
|
|
return list(res)
|
|
|
|
else:
|
|
|
|
return res
|
|
|
|
|
2019-07-30 13:49:21 +02:00
|
|
|
def is_taxonomie_tag_enabled(taxonomie, tag):
|
|
|
|
if tag in r_serv_tags.smembers('active_tag_' + taxonomie):
|
|
|
|
return True
|
|
|
|
else:
|
|
|
|
return False
|
|
|
|
|
|
|
|
def is_galaxy_tag_enabled(galaxy, tag):
|
|
|
|
if tag in r_serv_tags.smembers('active_tag_galaxies_' + galaxy):
|
|
|
|
return True
|
|
|
|
else:
|
|
|
|
return False
|
|
|
|
|
2019-11-18 09:46:15 +01:00
|
|
|
def enable_taxonomy(taxonomie, enable_tags=True):
|
|
|
|
'''
|
|
|
|
Enable a taxonomy. (UI)
|
|
|
|
|
|
|
|
:param taxonomie: MISP taxonomy
|
|
|
|
:type taxonomie: str
|
|
|
|
:param enable_tags: crawled domain
|
|
|
|
:type enable_tags: boolean
|
|
|
|
'''
|
|
|
|
taxonomies = Taxonomies()
|
|
|
|
if enable_tags:
|
|
|
|
taxonomie_info = taxonomies.get(taxonomie)
|
|
|
|
if taxonomie_info:
|
|
|
|
# activate taxonomie
|
|
|
|
r_serv_tags.sadd('active_taxonomies', taxonomie)
|
|
|
|
# activate taxonomie tags
|
|
|
|
for tag in taxonomie_info.machinetags():
|
|
|
|
r_serv_tags.sadd('active_tag_{}'.format(taxonomie), tag)
|
2020-01-08 17:11:37 +01:00
|
|
|
#r_serv_tags.sadd('active_taxonomies_tags', tag)
|
2019-11-18 09:46:15 +01:00
|
|
|
else:
|
|
|
|
print('Error: {}, please update pytaxonomies'.format(taxonomie))
|
|
|
|
|
2019-07-30 13:49:21 +02:00
|
|
|
# Check if tags are enabled in AIL
|
|
|
|
def is_valid_tags_taxonomies_galaxy(list_tags, list_tags_galaxy):
|
|
|
|
if list_tags:
|
|
|
|
active_taxonomies = get_active_taxonomies()
|
|
|
|
|
|
|
|
for tag in list_tags:
|
|
|
|
taxonomie = get_taxonomie_from_tag(tag)
|
2021-04-26 15:08:36 +02:00
|
|
|
if taxonomie is None:
|
|
|
|
return False
|
2019-07-30 13:49:21 +02:00
|
|
|
if taxonomie not in active_taxonomies:
|
|
|
|
return False
|
|
|
|
if not is_taxonomie_tag_enabled(taxonomie, tag):
|
|
|
|
return False
|
|
|
|
|
|
|
|
if list_tags_galaxy:
|
|
|
|
active_galaxies = get_active_galaxies()
|
|
|
|
|
|
|
|
for tag in list_tags_galaxy:
|
|
|
|
galaxy = get_galaxy_from_tag(tag)
|
2021-04-26 15:08:36 +02:00
|
|
|
if galaxy is None:
|
|
|
|
return False
|
2019-07-30 13:49:21 +02:00
|
|
|
if galaxy not in active_galaxies:
|
|
|
|
return False
|
|
|
|
if not is_galaxy_tag_enabled(galaxy, tag):
|
|
|
|
return False
|
|
|
|
return True
|
|
|
|
|
2021-11-22 23:45:41 +01:00
|
|
|
def is_taxonomie_tag(tag, namespace=None):
|
|
|
|
if not namespace:
|
|
|
|
namespace = tag.split(':')[0]
|
|
|
|
if namespace != 'misp-galaxy':
|
|
|
|
return True
|
|
|
|
else:
|
|
|
|
return False
|
|
|
|
|
|
|
|
def is_galaxy_tag(tag, namespace=None):
|
|
|
|
if not namespace:
|
|
|
|
namespace = tag.split(':')[0]
|
|
|
|
if namespace == 'misp-galaxy':
|
|
|
|
return True
|
|
|
|
else:
|
|
|
|
return False
|
|
|
|
|
|
|
|
# # TODO:
|
|
|
|
# def is_valid_tag(tag):
|
|
|
|
# pass
|
|
|
|
|
|
|
|
def is_enabled_tag(tag, enabled_namespace=None):
|
|
|
|
if is_taxonomie_tag(tag):
|
|
|
|
return is_enabled_taxonomie_tag(tag, enabled_taxonomies=enabled_namespace)
|
|
|
|
else:
|
|
|
|
return is_enabled_galaxy_tag(tag, enabled_galaxies=enabled_namespace)
|
|
|
|
|
|
|
|
def are_enabled_tags(tags):
|
|
|
|
enabled_taxonomies = get_active_taxonomies(r_set=True)
|
|
|
|
enabled_galaxies = get_active_galaxies(r_set=True)
|
|
|
|
for tag in tags:
|
|
|
|
if is_taxonomie_tag(tag):
|
|
|
|
res = is_enabled_taxonomie_tag(tag, enabled_taxonomies=enabled_taxonomies)
|
|
|
|
else:
|
|
|
|
res = is_enabled_galaxy_tag(tag, enabled_galaxies=enabled_galaxies)
|
|
|
|
if not res:
|
|
|
|
return False
|
|
|
|
return True
|
|
|
|
|
|
|
|
def is_enabled_taxonomie_tag(tag, enabled_taxonomies=None):
|
|
|
|
if not enabled_taxonomies:
|
|
|
|
enabled_taxonomies = get_active_taxonomies()
|
|
|
|
taxonomie = get_taxonomie_from_tag(tag)
|
|
|
|
if taxonomie is None:
|
|
|
|
return False
|
|
|
|
if taxonomie not in enabled_taxonomies:
|
|
|
|
return False
|
|
|
|
if not is_taxonomie_tag_enabled(taxonomie, tag):
|
|
|
|
return False
|
|
|
|
|
|
|
|
def is_enabled_galaxy_tag(tag, enabled_galaxies=None):
|
|
|
|
if not enabled_galaxies:
|
|
|
|
enabled_galaxies = get_active_galaxies()
|
|
|
|
galaxy = get_galaxy_from_tag(tag)
|
|
|
|
if galaxy is None:
|
|
|
|
return False
|
|
|
|
if galaxy not in enabled_galaxies:
|
|
|
|
return False
|
|
|
|
if not is_galaxy_tag_enabled(galaxy, tag):
|
|
|
|
return False
|
|
|
|
return True
|
|
|
|
|
2020-01-06 17:07:52 +01:00
|
|
|
#### ####
|
2019-11-19 14:03:23 +01:00
|
|
|
|
2019-08-01 14:36:52 +02:00
|
|
|
def is_tag_in_all_tag(tag):
|
|
|
|
if r_serv_tags.sismember('list_tags', tag):
|
|
|
|
return True
|
|
|
|
else:
|
|
|
|
return False
|
|
|
|
|
2021-11-22 23:45:41 +01:00
|
|
|
def get_tag_synonyms(tag):
|
|
|
|
return r_serv_tags.smembers(f'synonym_tag_{tag}')
|
|
|
|
|
|
|
|
def get_tag_dislay_name(tag):
|
|
|
|
tag_synonyms = get_tag_synonyms(tag)
|
|
|
|
if not tag_synonyms:
|
|
|
|
return tag
|
|
|
|
else:
|
|
|
|
return tag + ', '.join(tag_synonyms)
|
|
|
|
|
|
|
|
def get_tags_selector_dict(tags):
|
|
|
|
list_tags = []
|
|
|
|
for tag in tags:
|
|
|
|
list_tags.append(get_tag_selector_dict(tag))
|
|
|
|
return list_tags
|
|
|
|
|
|
|
|
def get_tag_selector_dict(tag):
|
|
|
|
return {'name':get_tag_dislay_name(tag),'id':tag}
|
|
|
|
|
|
|
|
def get_tags_selector_data():
|
|
|
|
dict_selector = {}
|
|
|
|
dict_selector['active_taxonomies'] = get_active_taxonomies()
|
|
|
|
dict_selector['active_galaxies'] = get_active_galaxies()
|
|
|
|
return dict_selector
|
|
|
|
|
2019-10-31 17:14:23 +01:00
|
|
|
def get_min_tag(tag):
|
|
|
|
tag = tag.split('=')
|
|
|
|
if len(tag) > 1:
|
|
|
|
if tag[1] != '':
|
|
|
|
tag = tag[1][1:-1]
|
|
|
|
# no value
|
|
|
|
else:
|
|
|
|
tag = tag[0][1:-1]
|
|
|
|
# custom tags
|
|
|
|
else:
|
|
|
|
tag = tag[0]
|
|
|
|
return tag
|
|
|
|
|
2020-01-06 17:07:52 +01:00
|
|
|
def get_obj_tags_minimal(item_id):
|
|
|
|
return [ {"tag": tag, "min_tag": get_min_tag(tag)} for tag in get_obj_tag(item_id) ]
|
2019-10-31 17:14:23 +01:00
|
|
|
|
2019-11-25 18:11:20 +01:00
|
|
|
def unpack_str_tags_list(str_tags_list):
|
|
|
|
str_tags_list = str_tags_list.replace('"','\"')
|
2019-12-02 17:15:48 +01:00
|
|
|
if str_tags_list:
|
|
|
|
return str_tags_list.split(',')
|
|
|
|
else:
|
|
|
|
return []
|
2019-11-25 18:11:20 +01:00
|
|
|
|
2020-01-06 17:07:52 +01:00
|
|
|
# used by modal
|
|
|
|
def get_modal_add_tags(item_id, object_type='item'):
|
|
|
|
'''
|
|
|
|
Modal: add tags to domain or Paste
|
|
|
|
'''
|
|
|
|
return {"active_taxonomies": get_active_taxonomies(), "active_galaxies": get_active_galaxies(),
|
2020-01-07 16:14:56 +01:00
|
|
|
"object_id": item_id, "object_type": object_type}
|
2020-01-06 17:07:52 +01:00
|
|
|
|
|
|
|
######## NEW VERSION ########
|
|
|
|
def get_tag_first_seen(tag, r_int=False):
|
|
|
|
'''
|
|
|
|
Get tag first seen (current: item only)
|
|
|
|
'''
|
|
|
|
res = r_serv_tags.hget('tag_metadata:{}'.format(tag), 'first_seen')
|
|
|
|
if r_int:
|
|
|
|
if res is None:
|
|
|
|
return 99999999
|
|
|
|
else:
|
|
|
|
return int(res)
|
|
|
|
return res
|
|
|
|
|
|
|
|
def get_tag_last_seen(tag, r_int=False):
|
|
|
|
'''
|
|
|
|
Get tag last seen (current: item only)
|
|
|
|
'''
|
|
|
|
res = r_serv_tags.hget('tag_metadata:{}'.format(tag), 'last_seen')
|
|
|
|
if r_int:
|
|
|
|
if res is None:
|
|
|
|
return 0
|
|
|
|
else:
|
|
|
|
return int(res)
|
|
|
|
return res
|
|
|
|
|
|
|
|
def get_tag_metadata(tag, r_int=False):
|
|
|
|
'''
|
|
|
|
Get tag metadata (current: item only)
|
|
|
|
'''
|
|
|
|
tag_metadata = {"tag": tag}
|
2020-01-21 15:23:59 +01:00
|
|
|
tag_metadata['first_seen'] = get_tag_first_seen(tag, r_int=r_int)
|
|
|
|
tag_metadata['last_seen'] = get_tag_last_seen(tag, r_int=r_int)
|
2020-01-06 17:07:52 +01:00
|
|
|
return tag_metadata
|
|
|
|
|
2020-01-10 16:52:55 +01:00
|
|
|
def get_tags_min_last_seen(l_tags, r_int=False):
|
|
|
|
'''
|
|
|
|
Get max last seen from a list of tags (current: item only)
|
|
|
|
'''
|
|
|
|
min_last_seen = 99999999
|
|
|
|
for tag in l_tags:
|
|
|
|
last_seen = get_tag_last_seen(tag, r_int=True)
|
|
|
|
if last_seen < min_last_seen:
|
|
|
|
min_last_seen = last_seen
|
|
|
|
if r_int:
|
|
|
|
return min_last_seen
|
|
|
|
else:
|
|
|
|
return str(min_last_seen)
|
|
|
|
|
2020-01-06 17:07:52 +01:00
|
|
|
def is_obj_tagged(object_id, tag):
|
|
|
|
'''
|
|
|
|
Check if a object is tagged
|
|
|
|
|
|
|
|
:param object_id: object id
|
|
|
|
:type domain: str
|
|
|
|
:param tag: object type
|
|
|
|
:type domain: str
|
|
|
|
|
|
|
|
:return: is object tagged
|
|
|
|
:rtype: boolean
|
|
|
|
'''
|
2020-01-07 16:14:56 +01:00
|
|
|
return r_serv_metadata.sismember('tag:{}'.format(object_id), tag)
|
2020-01-06 17:07:52 +01:00
|
|
|
|
|
|
|
def get_all_tags():
|
|
|
|
return list(r_serv_tags.smembers('list_tags'))
|
|
|
|
|
|
|
|
def get_all_obj_tags(object_type):
|
|
|
|
return list(r_serv_tags.smembers('list_tags:{}'.format(object_type)))
|
|
|
|
|
|
|
|
def get_obj_tag(object_id):
|
|
|
|
'''
|
|
|
|
Retun all the tags of a given object.
|
|
|
|
:param object_id: (item_id, domain, ...)
|
|
|
|
'''
|
|
|
|
res = r_serv_metadata.smembers('tag:{}'.format(object_id))
|
|
|
|
if res:
|
|
|
|
return list(res)
|
|
|
|
else:
|
|
|
|
return []
|
|
|
|
|
2019-07-30 13:49:21 +02:00
|
|
|
def update_tag_first_seen(tag, tag_first_seen, tag_last_seen):
|
|
|
|
if tag_first_seen == tag_last_seen:
|
|
|
|
if r_serv_tags.scard('{}:{}'.format(tag, tag_first_seen)) > 0:
|
|
|
|
r_serv_tags.hset('tag_metadata:{}'.format(tag), 'first_seen', tag_first_seen)
|
|
|
|
# no tag in db
|
|
|
|
else:
|
|
|
|
r_serv_tags.hdel('tag_metadata:{}'.format(tag), 'first_seen')
|
|
|
|
r_serv_tags.hdel('tag_metadata:{}'.format(tag), 'last_seen')
|
|
|
|
else:
|
|
|
|
if r_serv_tags.scard('{}:{}'.format(tag, tag_first_seen)) > 0:
|
|
|
|
r_serv_tags.hset('tag_metadata:{}'.format(tag), 'first_seen', tag_first_seen)
|
|
|
|
else:
|
|
|
|
tag_first_seen = Date.date_add_day(tag_first_seen)
|
|
|
|
update_tag_first_seen(tag, tag_first_seen, tag_last_seen)
|
|
|
|
|
|
|
|
def update_tag_last_seen(tag, tag_first_seen, tag_last_seen):
|
|
|
|
if tag_first_seen == tag_last_seen:
|
|
|
|
if r_serv_tags.scard('{}:{}'.format(tag, tag_last_seen)) > 0:
|
|
|
|
r_serv_tags.hset('tag_metadata:{}'.format(tag), 'last_seen', tag_last_seen)
|
|
|
|
# no tag in db
|
|
|
|
else:
|
|
|
|
r_serv_tags.hdel('tag_metadata:{}'.format(tag), 'first_seen')
|
|
|
|
r_serv_tags.hdel('tag_metadata:{}'.format(tag), 'last_seen')
|
|
|
|
else:
|
|
|
|
if r_serv_tags.scard('{}:{}'.format(tag, tag_last_seen)) > 0:
|
|
|
|
r_serv_tags.hset('tag_metadata:{}'.format(tag), 'last_seen', tag_last_seen)
|
|
|
|
else:
|
2021-04-26 15:08:36 +02:00
|
|
|
# # TODO: # FIXME:
|
2020-08-21 10:50:03 +02:00
|
|
|
#tag_last_seen = Date.date_substract_day(str(tag_last_seen))
|
|
|
|
#update_tag_last_seen(tag, tag_first_seen, tag_last_seen)
|
|
|
|
pass
|
2019-11-05 09:49:51 +01:00
|
|
|
|
2020-01-06 17:07:52 +01:00
|
|
|
def update_tag_metadata(tag, tag_date, object_type=None, add_tag=True):
|
|
|
|
'''
|
|
|
|
Update tag metadata (current: item only)
|
|
|
|
'''
|
2020-01-21 13:35:52 +01:00
|
|
|
if object_type=="item": # # TODO: use another getter (get all object with date)
|
2020-01-06 17:07:52 +01:00
|
|
|
# get object metadata
|
|
|
|
tag_metadata = get_tag_metadata(tag, r_int=True)
|
|
|
|
#############
|
|
|
|
## ADD tag ##
|
|
|
|
if add_tag:
|
|
|
|
# update fisrt_seen
|
|
|
|
if tag_date < tag_metadata['first_seen']:
|
|
|
|
r_serv_tags.hset('tag_metadata:{}'.format(tag), 'first_seen', tag_date)
|
|
|
|
# update last_seen
|
|
|
|
if tag_date > tag_metadata['last_seen']:
|
|
|
|
r_serv_tags.hset('tag_metadata:{}'.format(tag), 'last_seen', tag_date)
|
|
|
|
################
|
|
|
|
## REMOVE tag ##
|
|
|
|
else:
|
|
|
|
if tag_date == tag_metadata['first_seen']:
|
2020-01-21 15:27:40 +01:00
|
|
|
update_tag_first_seen(tag, tag_metadata['first_seen'], tag_metadata['last_seen'])
|
2020-01-06 17:07:52 +01:00
|
|
|
if tag_date == tag_metadata['last_seen']:
|
|
|
|
update_tag_last_seen(tag, tag_metadata['first_seen'], tag_metadata['last_seen'])
|
2019-11-05 09:49:51 +01:00
|
|
|
|
2020-01-07 14:30:52 +01:00
|
|
|
def update_tag_global_by_obj_type(object_type, tag):
|
|
|
|
tag_deleted = False
|
|
|
|
if object_type=='item':
|
|
|
|
if not r_serv_tags.exists('tag_metadata:{}'.format(tag)):
|
|
|
|
tag_deleted = True
|
|
|
|
else:
|
|
|
|
if not r_serv_tags.exists('{}:{}'.format(object_type, tag)):
|
|
|
|
tag_deleted = True
|
|
|
|
if tag_deleted:
|
|
|
|
# update object global tags
|
|
|
|
r_serv_tags.srem('list_tags:{}'.format(object_type), tag)
|
|
|
|
# update global tags
|
2020-02-11 15:48:30 +01:00
|
|
|
for obj_type in get_all_objects():
|
2020-01-07 14:30:52 +01:00
|
|
|
if r_serv_tags.exists('{}:{}'.format(obj_type, tag)):
|
|
|
|
tag_deleted = False
|
|
|
|
if tag_deleted:
|
|
|
|
r_serv_tags.srem('list_tags', tag)
|
|
|
|
|
2020-02-11 15:48:30 +01:00
|
|
|
def get_all_objects():
|
|
|
|
return ['domain', 'item', 'pgp', 'cryptocurrency', 'decoded', 'image']
|
|
|
|
|
2020-01-06 17:07:52 +01:00
|
|
|
def add_global_tag(tag, object_type=None):
|
2019-11-05 09:49:51 +01:00
|
|
|
'''
|
2020-01-06 17:07:52 +01:00
|
|
|
Create a set of all tags used in AIL (all + by object)
|
|
|
|
|
|
|
|
:param tag: tag
|
|
|
|
:type domain: str
|
|
|
|
:param object_type: object type
|
|
|
|
:type domain: str
|
2019-11-05 09:49:51 +01:00
|
|
|
'''
|
2020-01-06 17:07:52 +01:00
|
|
|
r_serv_tags.sadd('list_tags', tag)
|
|
|
|
if object_type:
|
|
|
|
r_serv_tags.sadd('list_tags:{}'.format(object_type), tag)
|
|
|
|
|
2020-01-07 14:30:52 +01:00
|
|
|
def add_obj_tags(object_id, object_type, tags=[], galaxy_tags=[]):
|
|
|
|
obj_date = get_obj_date(object_type, object_id)
|
|
|
|
for tag in tags:
|
|
|
|
if tag:
|
|
|
|
taxonomie = get_taxonomie_from_tag(tag)
|
|
|
|
if is_taxonomie_tag_enabled(taxonomie, tag):
|
|
|
|
add_tag(object_type, tag, object_id, obj_date=obj_date)
|
|
|
|
else:
|
|
|
|
return ({'status': 'error', 'reason': 'Tags or Galaxy not enabled', 'value': tag}, 400)
|
|
|
|
|
|
|
|
for tag in galaxy_tags:
|
|
|
|
if tag:
|
|
|
|
galaxy = get_galaxy_from_tag(tag)
|
|
|
|
if is_galaxy_tag_enabled(galaxy, tag):
|
|
|
|
add_tag(object_type, tag, object_id, obj_date=obj_date)
|
|
|
|
else:
|
|
|
|
return ({'status': 'error', 'reason': 'Tags or Galaxy not enabled', 'value': tag}, 400)
|
|
|
|
|
|
|
|
# TEMPLATE + API QUERY
|
|
|
|
def api_add_obj_tags(tags=[], galaxy_tags=[], object_id=None, object_type="item"):
|
|
|
|
res_dict = {}
|
|
|
|
if object_id == None:
|
|
|
|
return ({'status': 'error', 'reason': 'object_id id not found'}, 404)
|
|
|
|
if not tags and not galaxy_tags:
|
|
|
|
return ({'status': 'error', 'reason': 'Tags or Galaxy not specified'}, 400)
|
2020-01-20 16:42:26 +01:00
|
|
|
if object_type not in ('item', 'domain', 'image', 'decoded'): # # TODO: put me in another file
|
2020-01-07 14:30:52 +01:00
|
|
|
return ({'status': 'error', 'reason': 'Incorrect object_type'}, 400)
|
|
|
|
|
2020-01-07 16:14:56 +01:00
|
|
|
# remove empty tags
|
|
|
|
tags = list(filter(bool, tags))
|
|
|
|
galaxy_tags = list(filter(bool, galaxy_tags))
|
|
|
|
|
|
|
|
res = add_obj_tags(object_id, object_type, tags=tags, galaxy_tags=galaxy_tags)
|
2020-01-07 14:30:52 +01:00
|
|
|
if res:
|
|
|
|
return res
|
|
|
|
|
|
|
|
res_dict['tags'] = tags + galaxy_tags
|
2020-01-07 16:14:56 +01:00
|
|
|
res_dict['id'] = object_id
|
|
|
|
res_dict['type'] = object_type
|
2020-01-07 14:30:52 +01:00
|
|
|
return (res_dict, 200)
|
|
|
|
|
|
|
|
def add_obj_tag(object_type, object_id, tag, obj_date=None):
|
2020-01-06 17:07:52 +01:00
|
|
|
if object_type=="item": # # TODO: # FIXME: # REVIEW: rename me !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
|
2020-01-07 14:30:52 +01:00
|
|
|
if obj_date is None:
|
|
|
|
raise ValueError("obj_date is None")
|
|
|
|
|
2020-01-06 17:07:52 +01:00
|
|
|
# add tag
|
|
|
|
r_serv_metadata.sadd('tag:{}'.format(object_id), tag)
|
|
|
|
r_serv_tags.sadd('{}:{}'.format(tag, obj_date), object_id)
|
|
|
|
|
|
|
|
# add domain tag
|
2020-06-19 13:36:03 +02:00
|
|
|
if item_basic.is_crawled(object_id) and tag!='infoleak:submission="crawler"' and tag != 'infoleak:submission="manual"':
|
|
|
|
domain = item_basic.get_item_domain(object_id)
|
2020-01-06 17:07:52 +01:00
|
|
|
add_tag("domain", tag, domain)
|
|
|
|
else:
|
|
|
|
r_serv_metadata.sadd('tag:{}'.format(object_id), tag)
|
|
|
|
r_serv_tags.sadd('{}:{}'.format(object_type, tag), object_id)
|
|
|
|
|
2020-01-07 14:30:52 +01:00
|
|
|
def add_tag(object_type, tag, object_id, obj_date=None):
|
2020-01-06 17:07:52 +01:00
|
|
|
# new tag
|
|
|
|
if not is_obj_tagged(object_id, tag):
|
|
|
|
# # TODO: # FIXME: sanityze object_type
|
2020-01-30 11:31:33 +01:00
|
|
|
if obj_date:
|
|
|
|
try:
|
|
|
|
obj_date = int(obj_date)
|
|
|
|
except:
|
|
|
|
obj_date = None
|
2020-01-07 14:30:52 +01:00
|
|
|
if not obj_date:
|
|
|
|
obj_date = get_obj_date(object_type, object_id)
|
2020-01-06 17:07:52 +01:00
|
|
|
add_global_tag(tag, object_type=object_type)
|
2020-01-07 14:30:52 +01:00
|
|
|
add_obj_tag(object_type, object_id, tag, obj_date=obj_date)
|
2020-01-21 13:35:52 +01:00
|
|
|
update_tag_metadata(tag, obj_date, object_type=object_type)
|
2020-01-06 17:07:52 +01:00
|
|
|
|
|
|
|
# create tags stats # # TODO: put me in cache
|
|
|
|
r_serv_tags.hincrby('daily_tags:{}'.format(datetime.date.today().strftime("%Y%m%d")), tag, 1)
|
|
|
|
|
|
|
|
def delete_obj_tag(object_type, object_id, tag, obj_date):
|
2021-10-29 18:48:12 +02:00
|
|
|
if object_type=="item": # # TODO: # FIXME: # REVIEW: !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
|
2020-01-06 17:07:52 +01:00
|
|
|
obj_date = get_obj_date(object_type, object_id)
|
|
|
|
r_serv_metadata.srem('tag:{}'.format(object_id), tag)
|
|
|
|
r_serv_tags.srem('{}:{}'.format(tag, obj_date), object_id)
|
|
|
|
else:
|
|
|
|
r_serv_metadata.srem('tag:{}'.format(object_id), tag)
|
|
|
|
r_serv_tags.srem('{}:{}'.format(object_type, tag), object_id)
|
|
|
|
|
2020-01-07 14:30:52 +01:00
|
|
|
def delete_tag(object_type, tag, object_id, obj_date=None):
|
2020-01-06 17:07:52 +01:00
|
|
|
# tag exist
|
|
|
|
if is_obj_tagged(object_id, tag):
|
2020-01-07 14:30:52 +01:00
|
|
|
if not obj_date:
|
|
|
|
obj_date = get_obj_date(object_type, object_id)
|
2020-01-06 17:07:52 +01:00
|
|
|
delete_obj_tag(object_type, object_id, tag, obj_date)
|
|
|
|
update_tag_metadata(tag, obj_date, object_type=object_type, add_tag=False)
|
2020-01-07 14:30:52 +01:00
|
|
|
update_tag_global_by_obj_type(object_type, tag)
|
|
|
|
|
|
|
|
else:
|
|
|
|
return ({'status': 'error', 'reason': 'object id or tag not found', 'value': tag}, 400)
|
|
|
|
|
2020-02-11 15:48:30 +01:00
|
|
|
# # TODO: move me
|
|
|
|
def get_obj_date(object_type, object_id):
|
|
|
|
if object_type == "item":
|
2020-06-19 13:36:03 +02:00
|
|
|
return int(item_basic.get_item_date(object_id))
|
2020-02-11 15:48:30 +01:00
|
|
|
else:
|
|
|
|
return None
|
|
|
|
|
2020-01-07 14:30:52 +01:00
|
|
|
# API QUERY
|
|
|
|
def api_delete_obj_tags(tags=[], object_id=None, object_type="item"):
|
|
|
|
if not object_id:
|
|
|
|
return ({'status': 'error', 'reason': 'object id not found'}, 404)
|
|
|
|
if not tags:
|
|
|
|
return ({'status': 'error', 'reason': 'No Tag(s) specified'}, 400)
|
2020-01-06 17:07:52 +01:00
|
|
|
|
2021-10-29 18:48:12 +02:00
|
|
|
res = delete_obj_tags(object_id, object_type, tags)
|
2020-01-07 14:30:52 +01:00
|
|
|
if res:
|
|
|
|
return res
|
|
|
|
|
|
|
|
dict_res = {}
|
|
|
|
dict_res['tags'] = tags
|
|
|
|
dict_res['id'] = object_id
|
|
|
|
return (dict_res, 200)
|
|
|
|
|
2021-10-29 18:48:12 +02:00
|
|
|
def delete_obj_tags(object_id, object_type, tags):
|
2020-01-07 14:30:52 +01:00
|
|
|
obj_date = get_obj_date(object_type, object_id)
|
|
|
|
for tag in tags:
|
|
|
|
res = delete_tag(object_type, tag, object_id, obj_date=obj_date)
|
|
|
|
if res:
|
|
|
|
return res
|
2020-01-06 17:07:52 +01:00
|
|
|
|
2021-10-29 18:48:12 +02:00
|
|
|
def delete_obj_all_tags(obj_id, obj_type):
|
|
|
|
delete_obj_tags(obj_id, obj_type, get_obj_tag(obj_id))
|
|
|
|
|
2020-01-10 16:52:55 +01:00
|
|
|
def sanitise_tags_date_range(l_tags, date_from=None, date_to=None):
|
2020-02-07 10:12:38 +01:00
|
|
|
if date_from is None or date_to is None:
|
2020-01-10 16:52:55 +01:00
|
|
|
date_from = get_tags_min_last_seen(l_tags, r_int=False)
|
|
|
|
date_to = date_from
|
|
|
|
return Date.sanitise_date_range(date_from, date_to)
|
|
|
|
|
|
|
|
|
|
|
|
# # TODO: verify tags + object_type
|
|
|
|
# get set_keys: intersection
|
|
|
|
def get_obj_keys_by_tags(object_type, l_tags, date_day=None):
|
|
|
|
l_set_keys = []
|
|
|
|
if object_type=='item':
|
|
|
|
for tag in l_tags:
|
|
|
|
l_set_keys.append('{}:{}'.format(tag, date_day))
|
|
|
|
else:
|
|
|
|
for tag in l_tags:
|
|
|
|
l_set_keys.append('{}:{}'.format(object_type, tag))
|
|
|
|
return l_set_keys
|
|
|
|
|
|
|
|
def get_obj_by_tag(key_tag):
|
|
|
|
return r_serv_tags.smembers(key_tag)
|
|
|
|
|
|
|
|
def get_obj_by_tags(object_type, l_tags, date_from=None, date_to=None, nb_obj=50, page=1): # remove old object
|
|
|
|
# with daterange
|
|
|
|
l_tagged_obj = []
|
|
|
|
if object_type=='item':
|
|
|
|
#sanityze date
|
|
|
|
date_range = sanitise_tags_date_range(l_tags, date_from=date_from, date_to=date_to)
|
2020-01-20 15:13:03 +01:00
|
|
|
l_dates = Date.substract_date(date_range['date_from'], date_range['date_to'])
|
2020-01-10 16:52:55 +01:00
|
|
|
|
|
|
|
for date_day in l_dates:
|
|
|
|
l_set_keys = get_obj_keys_by_tags(object_type, l_tags, date_day)
|
|
|
|
# if len(l_set_keys) > nb_obj:
|
|
|
|
# return l_tagged_obj
|
|
|
|
if len(l_set_keys) < 2:
|
|
|
|
date_day_obj = get_obj_by_tag(l_set_keys[0])
|
|
|
|
else:
|
|
|
|
date_day_obj = r_serv_tags.sinter(l_set_keys[0], *l_set_keys[1:])
|
|
|
|
|
|
|
|
# next_nb_start = len(l_tagged_obj) + len(date_day_obj) - nb_obj
|
|
|
|
# if next_nb_start > 0:
|
|
|
|
# get + filter nb_start
|
|
|
|
l_tagged_obj.extend( date_day_obj )
|
|
|
|
|
|
|
|
# handle pagination
|
2020-01-13 11:10:03 +01:00
|
|
|
nb_all_elem = len(l_tagged_obj)
|
|
|
|
nb_pages = nb_all_elem / nb_obj
|
2020-01-10 16:52:55 +01:00
|
|
|
if not nb_pages.is_integer():
|
|
|
|
nb_pages = int(nb_pages)+1
|
|
|
|
else:
|
|
|
|
nb_pages = int(nb_pages)
|
|
|
|
if page > nb_pages:
|
|
|
|
page = nb_pages
|
|
|
|
|
|
|
|
start = nb_obj*(page -1)
|
2020-01-20 15:13:03 +01:00
|
|
|
if nb_pages > 1:
|
|
|
|
stop = (nb_obj*page)
|
|
|
|
l_tagged_obj = l_tagged_obj[start:stop]
|
|
|
|
# only one page
|
|
|
|
else:
|
|
|
|
stop = nb_all_elem
|
|
|
|
l_tagged_obj = l_tagged_obj[start:]
|
|
|
|
|
|
|
|
if stop > nb_all_elem:
|
|
|
|
stop = nb_all_elem
|
|
|
|
stop = stop -1
|
2020-01-10 16:52:55 +01:00
|
|
|
|
2020-01-20 15:13:03 +01:00
|
|
|
return {"tagged_obj":l_tagged_obj, "date" : date_range,
|
|
|
|
"page":page, "nb_pages":nb_pages, "nb_first_elem":start+1, "nb_last_elem":stop+1, "nb_all_elem":nb_all_elem}
|
2020-01-10 16:52:55 +01:00
|
|
|
|
|
|
|
# without daterange
|
|
|
|
else:
|
|
|
|
l_set_keys = get_obj_keys_by_tags(object_type, l_tags)
|
|
|
|
if len(l_set_keys) < 2:
|
|
|
|
l_tagged_obj = get_obj_by_tag(l_set_keys[0])
|
|
|
|
else:
|
|
|
|
l_tagged_obj = r_serv_tags.sinter(l_set_keys[0], *l_set_keys[1:])
|
|
|
|
|
|
|
|
if not l_tagged_obj:
|
|
|
|
return {"tagged_obj":l_tagged_obj, "page":0, "nb_pages":0}
|
|
|
|
|
|
|
|
# handle pagination
|
2020-01-13 11:10:03 +01:00
|
|
|
nb_all_elem = len(l_tagged_obj)
|
|
|
|
nb_pages = nb_all_elem / nb_obj
|
2020-01-10 16:52:55 +01:00
|
|
|
if not nb_pages.is_integer():
|
|
|
|
nb_pages = int(nb_pages)+1
|
|
|
|
else:
|
|
|
|
nb_pages = int(nb_pages)
|
|
|
|
if page > nb_pages:
|
|
|
|
page = nb_pages
|
|
|
|
|
|
|
|
# multiple pages
|
|
|
|
if nb_pages > 1:
|
|
|
|
start = nb_obj*(page -1)
|
|
|
|
stop = (nb_obj*page) -1
|
|
|
|
current_index = 0
|
|
|
|
l_obj = []
|
|
|
|
for elem in l_tagged_obj:
|
|
|
|
if current_index > stop:
|
|
|
|
break
|
|
|
|
if start <= current_index and stop >= current_index:
|
|
|
|
l_obj.append(elem)
|
|
|
|
current_index += 1
|
|
|
|
l_tagged_obj = l_obj
|
2020-01-13 11:10:03 +01:00
|
|
|
stop += 1
|
|
|
|
if stop > nb_all_elem:
|
|
|
|
stop = nb_all_elem
|
2020-01-10 16:52:55 +01:00
|
|
|
# only one page
|
|
|
|
else:
|
2020-01-13 11:10:03 +01:00
|
|
|
start = 0
|
|
|
|
stop = nb_all_elem
|
2020-01-10 16:52:55 +01:00
|
|
|
l_tagged_obj = list(l_tagged_obj)
|
|
|
|
|
2020-01-13 11:10:03 +01:00
|
|
|
return {"tagged_obj":l_tagged_obj, "page":page, "nb_pages":nb_pages, "nb_first_elem":start+1, "nb_last_elem":stop, "nb_all_elem":nb_all_elem}
|
2020-06-19 13:36:03 +02:00
|
|
|
|
|
|
|
|
|
|
|
#### TAGS EXPORT ####
|
|
|
|
# # TODO:
|
|
|
|
def is_updated_tags_to_export(): # by type
|
|
|
|
return False
|
|
|
|
|
|
|
|
def get_list_of_solo_tags_to_export_by_type(export_type): # by type
|
|
|
|
if export_type in ['misp', 'thehive']:
|
|
|
|
return r_serv_db.smembers('whitelist_{}'.format(export_type))
|
|
|
|
else:
|
|
|
|
return None
|
|
|
|
#r_serv_db.smembers('whitelist_hive')
|
|
|
|
|
|
|
|
|
|
|
|
#### -- ####
|