diff --git a/.gitignore b/.gitignore index 0f7dfafa..fe57bf5c 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ *.swp *.pyc *.swo +.idea # Install Dirs AILENV diff --git a/.gitmodules b/.gitmodules index b9e14b36..bfb96ef8 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,3 +4,6 @@ [submodule "files/misp-taxonomies"] path = files/misp-taxonomies url = https://github.com/MISP/misp-taxonomies.git +[submodule "files/misp-galaxy"] + path = files/misp-galaxy + url = https://github.com/MISP/misp-galaxy.git diff --git a/bin/DB_KVROCKS_MIGRATION.py b/bin/DB_KVROCKS_MIGRATION.py index 6da1f7f2..1fb54348 100755 --- a/bin/DB_KVROCKS_MIGRATION.py +++ b/bin/DB_KVROCKS_MIGRATION.py @@ -73,7 +73,7 @@ old_crawlers.r_serv_onion = r_crawler # CREATE FUNCTION BY DB/FEATURES -# /!\ ISSUE WITH FILE DUPLICATES => NEED TO BE REFACTORED +# /!\ TODO MIGRATE DUPLICATES def get_item_date(item_id): @@ -670,27 +670,37 @@ def get_subtype_object(obj_type, subtype, obj_id): return Username(obj_id, subtype) def migrate_subtype_obj(Obj, obj_type, subtype, obj_id): - first_seen = get_obj_subtype_first_seen(obj_type, subtype, obj_id) - last_seen = get_obj_subtype_last_seen(obj_type, subtype, obj_id) + # first_seen = get_obj_subtype_first_seen(obj_type, subtype, obj_id) + # last_seen = get_obj_subtype_last_seen(obj_type, subtype, obj_id) # dates for item_id in get_item_correlation_obj(obj_type, subtype, obj_id): date = get_item_date(item_id) Obj.add(date, item_id) + dict_obj_subtypes = {'cryptocurrency': ['bitcoin', 'bitcoin-cash', 'dash', 'ethereum', 'litecoin', 'monero', 'zcash'], - 'pgpdump': ['key', 'mail', 'name'], - 'username': ['telegram', 'twitter', 'jabber']} + 'pgpdump': ['key', 'mail', 'name'], + 'username': ['telegram', 'twitter', 'jabber']} def subtypes_obj_migration(): - print('SUBPTYPE MIGRATION...') + print('SUBTYPE MIGRATION...') + pgp_symmetrical_key = '0x0000000000000000' for obj_type in dict_obj_subtypes: print(f'{obj_type} MIGRATION...') for subtype in dict_obj_subtypes[obj_type]: for obj_id in get_all_subtype_id(obj_type, subtype): - Obj = get_subtype_object(obj_type, subtype, obj_id) - migrate_subtype_obj(Obj, obj_type, subtype, obj_id) + if obj_type == 'pgp' and subtype == 'key' and obj_id == pgp_symmetrical_key: + pass + else: + Obj = get_subtype_object(obj_type, subtype, obj_id) + migrate_subtype_obj(Obj, obj_type, subtype, obj_id) + + # ADD PGP Symmetrical tag to item + for item_id in get_item_correlation_obj('pgpdump', 'key', pgp_symmetrical_key): + item = Items.Item(item_id) + item.add_tag(f'infoleak:automatic-detection="pgp-symmetric";{item_id}') # TODO SELECT TAG # # # # # # # # # # # # # # # # # STATISTICS @@ -702,10 +712,9 @@ def get_all_provider(): return r_serv_trend.smembers('all_provider_set') def get_item_source_stats_by_date(date, source): - stats = {} - stats['num'] = r_serv_trend.hget(f'{source}_num', date) - stats['size'] = r_serv_trend.hget(f'{source}_size', date) - stats['avg'] = r_serv_trend.hget(f'{source}_avg', date) + stats = {'num': r_serv_trend.hget(f'{source}_num', date), + 'size': r_serv_trend.hget(f'{source}_size', date), + 'avg': r_serv_trend.hget(f'{source}_avg', date)} return stats def get_item_stats_size_avg_by_date(date): diff --git a/bin/IPAddress.py b/bin/IPAddress.py index 81cf2e14..8b5476fb 100755 --- a/bin/IPAddress.py +++ b/bin/IPAddress.py @@ -74,7 +74,6 @@ if __name__ == '__main__': print('Please provide a list of valid IP addresses') sys.exit(0) - # Sent to the logging a description of the module publisher.info("Run IP module") diff --git a/bin/export/AILObjects.py b/bin/export/AILObjects.py index 37560545..f3c5b21b 100755 --- a/bin/export/AILObjects.py +++ b/bin/export/AILObjects.py @@ -4,7 +4,6 @@ import os import sys import uuid -import redis sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib')) sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages')) diff --git a/bin/export/MispImport.py b/bin/export/MispImport.py index d62a486f..b67bc886 100755 --- a/bin/export/MispImport.py +++ b/bin/export/MispImport.py @@ -42,7 +42,7 @@ def get_global_id_from_id(global_id): obj_meta['id'] = global_id[2] else: obj_meta['type'] = global_id[0] - obj_meta['subtype'] = None + obj_meta['subtype'] = '' obj_meta['id'] = global_id[1] return obj_meta diff --git a/bin/lib/Correlate_object.py b/bin/lib/Correlate_object.py index 3d09b478..a17e2bd3 100755 --- a/bin/lib/Correlate_object.py +++ b/bin/lib/Correlate_object.py @@ -433,25 +433,7 @@ def get_obj_global_id(obj_type, obj_id, obj_sub_type=None): return '{}:{}'.format(obj_type, obj_id) -def get_global_id_from_id(global_id): - obj_meta = {} - global_id = global_id.split(':', 3) - if len(global_id) > 2: - obj_meta['type'] = global_id[0] - obj_meta['subtype'] = global_id[1] - obj_meta['id'] = global_id[2] - else: - obj_meta['type'] = global_id[0] - obj_meta['subtype'] = None - obj_meta['id'] = global_id[1] - return obj_meta -# used by UI -def get_obj_str_type_subtype(obj_type, obj_subtype): - if obj_subtype: - return '{};{}'.format(obj_type, obj_subtype) - else: - return obj_type def sanitise_correlation_names(correlation_names): ''' diff --git a/bin/lib/Duplicate.py b/bin/lib/Duplicate.py index 2cae8d7a..ab457e85 100755 --- a/bin/lib/Duplicate.py +++ b/bin/lib/Duplicate.py @@ -36,7 +36,6 @@ config_loader = None # Duplicates domains != Duplicates items - def get_ssdeep_hash(content): return ssdeep.hash(content) @@ -100,8 +99,6 @@ def add_obj_duplicate(algo, hash, similarity, obj_type, subtype, id, date_ymonth _add_obj_duplicate(algo, similarity, obj_type, subtype, obj2_id, id) - - def get_last_x_month_dates(nb_months): now = datetime.datetime.now() result = [now.strftime("%Y%m")] @@ -111,20 +108,7 @@ def get_last_x_month_dates(nb_months): return result - if __name__ == '__main__': res = get_last_x_month_dates(7) print(res) - - - - - - - - - - - -################################# diff --git a/bin/lib/Tag.py b/bin/lib/Tag.py index b3f55024..10281596 100755 --- a/bin/lib/Tag.py +++ b/bin/lib/Tag.py @@ -1,11 +1,15 @@ #!/usr/bin/env python3 # -*-coding:UTF-8 -* -import os -import sys import redis import datetime +import os +import json +import sys + +from glob import glob + sys.path.append(os.environ['AIL_BIN']) ################################## # Import Project packages @@ -24,37 +28,41 @@ config_loader = None #### CORE FUNCTIONS #### +# # # # UNSAFE TAGS # # # # + def build_unsafe_tags(): - unsafe_tags = set() - ## CE content - unsafe_tags.add('dark-web:topic="pornography-child-exploitation"') + tags = set() + # CE content + tags.add('dark-web:topic="pornography-child-exploitation"') # add copine-scale tags taxonomies = Taxonomies() copine_scale = taxonomies.get('copine-scale') if copine_scale: for tag in copine_scale.machinetags(): - unsafe_tags.add(tag) - return unsafe_tags + tags.add(tag) + return tags + +def is_tags_safe(ltags): + """ + Check if a list of tags contain unsafe tags (CE, ...) + :param ltags: list of tags + :type ltags: list + :return: is a tag in the set unsafe + :rtype: boolean + """ + return unsafe_tags.isdisjoint(ltags) + # set of unsafe tags unsafe_tags = build_unsafe_tags() -def is_tags_safe(ltags): - ''' - Check if a list of tags contain an unsafe tag (CE, ...) - - :param ltags: list of tags - :type ltags: list - :return: is a tag in the unsafe set - :rtype: boolean - ''' - return unsafe_tags.isdisjoint(ltags) +# - - - UNSAFE TAGS - - - # # # TODO: verify tags + object_type # get set_keys: intersection def get_obj_keys_by_tags(tags, obj_type, subtype='', date=None): l_set_keys = [] - if obj_type=='item': + if obj_type == 'item': for tag in tags: l_set_keys.append(f'{obj_type}:{subtype}:{tag}:{date}') else: @@ -65,7 +73,405 @@ def get_obj_keys_by_tags(tags, obj_type, subtype='', date=None): def get_obj_by_tag(key_tag): return r_tags.smembers(key_tag) -##-- CORE FUNCTIONS --## +# -- CORE FUNCTIONS -- # + + +#### Taxonomies #### + +TAXONOMIES = {} +def load_taxonomies(): + global TAXONOMIES + manifest = os.path.join(os.environ['AIL_HOME'], 'files/misp-taxonomies/MANIFEST.json') + TAXONOMIES = Taxonomies(manifest_path=manifest) + + +load_taxonomies() + +def get_taxonomies(): + return TAXONOMIES.keys() + +# r_tags.sadd(f'active_taxonomies', taxonomy) +def is_taxonomy_enabled(taxonomy): + # enabled = r_tags.sismember('taxonomies:enabled', taxonomy) + try: + enabled = r_tags.sismember('taxonomies:enabled', taxonomy) + except redis.exceptions.ResponseError: + enabled = False + return enabled + +def enable_taxonomy(taxonomy): + r_tags.sadd('taxonomies:enabled', taxonomy) + +def disable_taxonomy(taxonomy): + r_tags.srem('taxonomies:enabled', taxonomy) + +def exists_taxonomy(taxonomy): + return TAXONOMIES.get(taxonomy) is not None + +def get_taxonomy_description(taxonomy): + return TAXONOMIES.get(taxonomy).description + +def get_taxonomy_name(taxonomy): + return TAXONOMIES.get(taxonomy).name + +def get_taxonomy_predicates(taxonomy): + meta = {} + predicates = taxonomy.predicates + for predicate in predicates: + meta[predicate] = {} + expanded = predicates[predicate].expanded + if expanded: + meta[predicate]['expanded'] = expanded + description = predicates[predicate].description + if description: + meta[predicate]['description'] = description + return meta + +def get_taxonomy_refs(taxonomy): + return TAXONOMIES.get(taxonomy).refs + +def get_taxonomy_version(taxonomy): + return TAXONOMIES.get(taxonomy).version + +def get_taxonomy_tags(taxonomy, enabled=False): + taxonomy_obj = TAXONOMIES.get(taxonomy) + tags = [] + for p, content in taxonomy_obj.items(): + if content: + for k, entry in content.items(): + tag = f'{taxonomy_obj.name}:{p}="{k}"' + expanded = entry.expanded + meta_tag = {'tag': tag, 'expanded': expanded} + if enabled: + meta_tag['enabled'] = is_taxonomy_tag_enabled(taxonomy, tag) + tags.append(meta_tag) + else: + tag = f'{taxonomy_obj.name}:{p}' + expanded = content.expanded + meta_tag = {'tag': tag, 'expanded': expanded} + if enabled: + meta_tag['enabled'] = is_taxonomy_tag_enabled(taxonomy, tag) + tags.append(meta_tag) + return tags + +# TODO GET ACTIVE TAGS +# TODO GET IF TAXONOMY IS ACTIVATED +def get_taxonomy_meta(taxonomy_name, enabled=False, enabled_tags=False, nb_active_tags=False, predicates=False, tags=False, expanded=True): + meta = {} + if not exists_taxonomy(taxonomy_name): + return meta + taxonomy = TAXONOMIES.get(taxonomy_name) + meta['description'] = taxonomy.description + meta['name'] = taxonomy.name + meta['version'] = taxonomy.version + if enabled: + meta['enabled'] = is_taxonomy_enabled(taxonomy_name) + if predicates: + meta['predicates'] = get_taxonomy_predicates(taxonomy) + if taxonomy.expanded: + meta['expanded'] = taxonomy.expanded + if taxonomy.refs: + meta['refs'] = taxonomy.refs + + # TODO PERF SAVE IN DB ????? + if tags: + if expanded: + meta['tags'] = get_taxonomy_tags(taxonomy_name, enabled=enabled_tags) + else: + meta['tags'] = taxonomy.machinetags() + meta['nb_tags'] = len(meta['tags']) + if nb_active_tags: + meta['nb_active_tags'] = get_taxonomy_nb_tags_enabled(taxonomy_name) + if not tags: + meta['nb_tags'] = len(taxonomy.machinetags()) + return meta + +def get_taxonomies_meta(): + meta = {} + for taxonomy in get_taxonomies(): + meta[taxonomy] = get_taxonomy_meta(taxonomy, enabled=True, nb_active_tags=True) + return meta + + #### Enabled Tags #### +# r_tags.sadd('active_taxonomies', taxonomy) +def get_taxonomy_tags_enabled(taxonomy): + return r_tags.smembers(f'taxonomy:tags:enabled:{taxonomy}') + +def get_taxonomy_nb_tags_enabled(taxonomy): + return r_tags.scard(f'taxonomy:tags:enabled:{taxonomy}') + +def is_taxonomy_tag_enabled(taxonomy, tag): + # r_tags.sismember('list_tags', tag) + try: + enabled = r_tags.sismember(f'taxonomy:tags:enabled:{taxonomy}', tag) + except redis.exceptions.ResponseError: + enabled = False + return enabled + +def add_taxonomy_tag_enabled(taxonomy, tag): + r_tags.sadd(f'taxonomy:tags:enabled:{taxonomy}', tag) + +def remove_taxonomy_tag_enabled(taxonomy, tag): + r_tags.srem(f'taxonomy:tags:enabled:{taxonomy}', tag) + +def disable_taxonomy_tags_enabled(taxonomy): + r_tags.delete(f'taxonomy:tags:enabled:{taxonomy}') + +def update_taxonomy_tag_enabled(taxonomy, tags): + if not tags: + return None + enable_taxonomy(taxonomy) + tags = set(tags) + enabled_tags = get_taxonomy_tags_enabled(taxonomy) + if tags != enabled_tags: + # disable tags + for tag in enabled_tags.difference(tags): + remove_taxonomy_tag_enabled(taxonomy, tag) + # enable tags + for tag in tags: + add_taxonomy_tag_enabled(taxonomy, tag) + +def api_update_taxonomy_tag_enabled(data): + taxonomy = data.get('taxonomy') + if not exists_taxonomy(taxonomy): + return {'error': f'taxonomy {taxonomy} not found'}, 404 + tags = data.get('tags', []) + taxonomy_tags = set(TAXONOMIES.get(taxonomy).machinetags()) + for tag in tags: + if tag not in taxonomy_tags: + return {'error': f'tag {tag} not found'}, 404 + update_taxonomy_tag_enabled(taxonomy, tags) + +def enable_taxonomy_tags(taxonomy): + enable_taxonomy(taxonomy) + for tag in TAXONOMIES.get(taxonomy).machinetags(): + add_taxonomy_tag_enabled(taxonomy, tag) + +def api_enable_taxonomy_tags(data): + taxonomy = data.get('taxonomy') + if not exists_taxonomy(taxonomy): + return {'error': f'taxonomy {taxonomy} not found'}, 404 + enable_taxonomy_tags(taxonomy) + +def disable_taxonomy_tags(taxonomy): + disable_taxonomy(taxonomy) + disable_taxonomy_tags_enabled(taxonomy) + +def api_disable_taxonomy_tags(data): + taxonomy = data.get('taxonomy') + if not exists_taxonomy(taxonomy): + return {'error': f'taxonomy {taxonomy} not found'}, 404 + disable_taxonomy_tags(taxonomy) + + # -- Enabled Tags -- # + +# -- Taxonomies -- # + +#### GALAXIES #### + +# +# var galaxy = galaxy type +# + + +GALAXIES = {} +CLUSTERS = {} +def load_galaxies(): + global GALAXIES + galaxies = [] + root_dir_galaxies = os.path.join(os.environ['AIL_HOME'], 'files/misp-galaxy/galaxies') + for galaxy_file in glob(os.path.join(root_dir_galaxies, '*.json')): + with open(galaxy_file, 'r') as f: + galaxies.append(json.load(f)) + GALAXIES = Galaxies(galaxies=galaxies) + global CLUSTERS + clusters = [] + root_dir_clusters = os.path.join(os.environ['AIL_HOME'], 'files/misp-galaxy/clusters') + for cluster_file in glob(os.path.join(root_dir_clusters, '*.json')): + with open(cluster_file, 'r') as f: + clusters.append(json.load(f)) + CLUSTERS = Clusters(clusters) + + +# LOAD GALAXY + CLUSTERS +load_galaxies() + +def get_galaxies(): + return GALAXIES.keys() + +def get_galaxy(galaxy_name): + return GALAXIES.get(galaxy_name) + +def exists_galaxy(galaxy): + return CLUSTERS.get(galaxy) is not None + +# r_tags.sadd('active_galaxies', galaxy) +def is_galaxy_enabled(galaxy): + try: + enabled = r_tags.sismember('galaxies:enabled', galaxy) + except redis.exceptions.ResponseError: + enabled = False + return enabled + +def enable_galaxy(galaxy): + r_tags.sadd('galaxies:enabled', galaxy) + +def disable_galaxy(galaxy): + r_tags.srem('galaxies:enabled', galaxy) + +def get_galaxy_meta(galaxy_name, nb_active_tags=False): + galaxy = get_galaxy(galaxy_name) + meta = {'name': galaxy.name, 'namespace': galaxy.namespace, 'description': galaxy.description, + 'type': galaxy.type, 'version': galaxy.version, 'enabled': is_galaxy_enabled(galaxy.type)} + icon = galaxy.icon + if icon == 'android' or icon == 'optin-monster' or icon == 'internet-explorer' or icon == 'btc': + meta['icon'] = f'fab fa-{icon}' + else: + meta['icon'] = f'fas fa-{icon}' + if nb_active_tags: + meta['nb_active_tags'] = get_galaxy_nb_tags_enabled(galaxy) + meta['nb_tags'] = len(get_galaxy_tags(galaxy.type)) + return meta + +def get_galaxies_meta(): + galaxies = [] + for galaxy_name in get_galaxies(): + galaxies.append(get_galaxy_meta(galaxy_name, nb_active_tags=True)) + return galaxies + +def get_galaxy_tag_meta(galaxy_type, tag): + cluster = get_cluster(galaxy_type) + if not cluster: + return {} + try: + tag_val = tag.rsplit('=', 1)[1][1:-1] + except: + return {} + cluster_val = cluster.cluster_values.get(tag_val) + if not cluster_val: + return {} + meta = cluster_val.to_dict() + if 'meta' in meta: + meta['meta'] = json.loads(meta.get('meta').to_json()) + meta['meta'] = json.dumps(meta['meta'], ensure_ascii=False, indent=4) + meta['tag'] = f'misp-galaxy:{galaxy_type}="{cluster_val.value}"' + meta['enabled'] = is_galaxy_tag_enabled(galaxy_type, meta['tag']) + return meta + + +def get_clusters(): + return CLUSTERS.keys() + +def get_cluster(cluster_type): + return CLUSTERS.get(cluster_type) + +def get_galaxy_tags(galaxy_type): + cluster = get_cluster(galaxy_type) + return cluster.machinetags() + +# TODO synonym +def get_cluster_tags(cluster_type, enabled=False): + tags = [] + cluster = get_cluster(cluster_type) + for cluster_val in cluster.values(): + tag = f'misp-galaxy:{cluster_type}="{cluster_val.value}"' + meta_tag = {'tag': tag, 'description': cluster_val.description} + if enabled: + meta_tag['enabled'] = is_galaxy_tag_enabled(cluster_type, tag) + synonyms = cluster_val.meta.synonyms + if not synonyms: + synonyms = [] + meta_tag['synonyms'] = synonyms + tags.append(meta_tag) + return tags + +def get_cluster_meta(cluster_type, tags=False, enabled=False): + cluster = get_cluster(cluster_type) + if not cluster: + return {} + meta = {'name': cluster.name, 'type': cluster.type, 'source': cluster.source, + 'authors': cluster.authors, 'description': cluster.description, 'version': cluster.version, + 'category': cluster.category} + if enabled: + meta['enabled'] = is_galaxy_enabled(cluster_type) + if tags: + meta['tags'] = get_cluster_tags(cluster_type, enabled=enabled) + + return meta + + #### Enabled Tags #### + +def get_galaxy_tags_enabled(galaxy): + return r_tags.smembers(f'galaxy:tags:enabled:{galaxy}') + +def get_galaxy_nb_tags_enabled(galaxy): + return r_tags.scard(f'galaxy:tags:enabled:{galaxy}') + +def is_galaxy_tag_enabled(galaxy, tag): + try: + enabled = r_tags.sismember(f'galaxy:tags:enabled:{galaxy}', tag) + except redis.exceptions.ResponseError: + enabled = False + return enabled + +def add_galaxy_tag_enabled(galaxy, tag): + r_tags.sadd(f'galaxy:tags:enabled:{galaxy}', tag) + +def remove_galaxy_tag_enabled(galaxy, tag): + r_tags.srem(f'galaxy:tags:enabled:{galaxy}', tag) + +def disable_galaxy_tags_enabled(galaxy): + r_tags.delete(f'galaxy:tags:enabled:{galaxy}') + +def update_galaxy_tag_enabled(galaxy, tags): + if not tags: + return None + enable_galaxy(galaxy) + tags = set(tags) + enabled_tags = get_galaxy_tags_enabled(galaxy) + if tags != enabled_tags: + # disable tags + for tag in enabled_tags.difference(tags): + remove_galaxy_tag_enabled(galaxy, tag) + # enable tags + for tag in tags: + add_galaxy_tag_enabled(galaxy, tag) + +def api_update_galaxy_tag_enabled(data): + galaxy = data.get('galaxy') + if not exists_galaxy(galaxy): + return {'error': f'galaxy {galaxy} not found'}, 404 + tags = data.get('tags', []) + galaxy_tags = set(get_galaxy_tags(galaxy)) + for tag in tags: + if tag not in galaxy_tags: + return {'error': f'tag {tag} not found'}, 404 + update_galaxy_tag_enabled(galaxy, tags) + +def enable_galaxy_tags(galaxy): + enable_galaxy(galaxy) + for tag in get_galaxy_tags(galaxy): + add_galaxy_tag_enabled(galaxy, tag) + +def api_enable_galaxy_tags(data): + galaxy = data.get('galaxy') + if not exists_galaxy(galaxy): + return {'error': f'galaxy {galaxy} not found'}, 404 + enable_galaxy_tags(galaxy) + +def disable_galaxy_tags(galaxy): + disable_galaxy(galaxy) + disable_galaxy_tags_enabled(galaxy) + +def api_disable_galaxy_tags(data): + galaxy = data.get('galaxy') + if not exists_galaxy(galaxy): + return {'error': f'galaxy {galaxy} not found'}, 404 + disable_galaxy_tags(galaxy) + + # -- Enabled Tags -- # + +# -- GALAXIES -- # ################################################################################ ################################################################################ @@ -74,38 +480,44 @@ def get_obj_by_tag(key_tag): ################################################################################ def is_obj_tagged(obj_type, obj_id, subtype=''): - ''' + """ Check if a object is tagged - :param object_id: object id - :type domain: str + :param obj_type: object type + :type obj_type: str + :param subtype: object subtype + :type subtype: str + :param obj_id: object ID + :type obj_id: str :return: is object tagged :rtype: boolean - ''' + """ return r_tags.exists(f'tag:{obj_type}:{subtype}:{obj_id}') def is_obj_tagged_by_tag(obj_type, obj_id, tag, subtype=''): - ''' - Check if a object is tagged + """ + Check if a object is tagged by a specified tag - :param object_id: object id - :type domain: str - :param tag: object type - :type domain: str + :param obj_type: object type + :type obj_type: str + :param subtype: object subtype + :type subtype: str + :param obj_id: object ID + :type obj_id: str + :param tag: tag + :type tag: str - :return: is object tagged + :return: is object tagged by a specified tag :rtype: boolean - ''' + """ return r_tags.sismember(f'tag:{obj_type}:{subtype}:{obj_id}', tag) - - # -# f'tag:{obj_type}:{subtype}:{id}' f'tag:{id}' +# 'tag:{obj_type}:{subtype}:{id}' 'tag:{id}' # -# f'list_tags:{obj_type}:{subtype}' f'list_tags:{obj_type}' +# 'list_tags:{obj_type}:{subtype}' 'list_tags:{obj_type}' # # graph tags by days ??????????????????????????????? # @@ -217,7 +629,7 @@ def get_tag_objects(obj_type, subtype='', date=''): def get_object_tags(obj_type, obj_id, subtype=''): return r_tags.smembers(f'tag:{obj_type}:{subtype}:{obj_id}') -def add_object_tag(tag, obj_type, id, subtype=''): ############################# +def add_object_tag(tag, obj_type, id, subtype=''): if r_tags.sadd(f'tag:{obj_type}:{subtype}:{id}', tag) == 1: r_tags.sadd('list_tags', tag) r_tags.sadd(f'list_tags:{obj_type}', tag) @@ -227,7 +639,7 @@ def add_object_tag(tag, obj_type, id, subtype=''): ############################# r_tags.sadd(f'{obj_type}:{subtype}:{tag}:{date}', id) # add domain tag - if item_basic.is_crawled(id) and tag!='infoleak:submission="crawler"' and tag != 'infoleak:submission="manual"': + if item_basic.is_crawled(id) and tag != 'infoleak:submission="crawler"' and tag != 'infoleak:submission="manual"': domain = item_basic.get_item_domain(id) add_object_tag(tag, "domain", domain) @@ -237,18 +649,28 @@ def add_object_tag(tag, obj_type, id, subtype=''): ############################# r_tags.hincrby(f'daily_tags:{datetime.date.today().strftime("%Y%m%d")}', tag, 1) -def update_tag_global_by_obj_type(tag, object_type, subtype=''): +# obj -> Object() +def confirm_tag(tag, obj): + if tag.startswith('infoleak:automatic-detection'): + tag = tag.replace('automatic-detection', 'analyst-detection', 1) + obj.add_tag(tag) + return True + return False + +# FIXME +# TODO REVIEW ME +def update_tag_global_by_obj_type(tag, obj_type, subtype=''): tag_deleted = False - if object_type=='item': + if obj_type == 'item': if not r_tags.exists(f'tag_metadata:{tag}'): tag_deleted = True else: - if not r_tags.exists(f'{object_type}:{subtype}:{tag}'): + if not r_tags.exists(f'{obj_type}:{subtype}:{tag}'): r_tags.srem(f'list_tags:{obj_type}:{subtype}', tag) # Iterate on all subtypes delete_global_obj_tag = True for obj_subtype in ail_core.get_object_all_subtypes(): - if r_tags.exists(f'list_tags:{obj_type}:{subtype}'): + if r_tags.exists(f'list_tags:{obj_type}:{obj_subtype}'): delete_global_obj_tag = False break if delete_global_obj_tag: @@ -280,6 +702,7 @@ def delete_object_tag(tag, obj_type, id, subtype=''): ################################################################################################################ +# TODO: REWRITE OLD # TODO: rewrite me # TODO: other objects def get_obj_by_tags(obj_type, l_tags, date_from=None, date_to=None, nb_obj=50, page=1): @@ -338,7 +761,7 @@ def get_obj_by_tags(obj_type, l_tags, date_from=None, date_to=None, nb_obj=50, p l_tagged_obj = r_tags.sinter(l_set_keys[0], *l_set_keys[1:]) if not l_tagged_obj: - return {"tagged_obj":l_tagged_obj, "page":0, "nb_pages":0} + return {"tagged_obj": l_tagged_obj, "page": 0, "nb_pages": 0} # handle pagination nb_all_elem = len(l_tagged_obj) @@ -359,7 +782,7 @@ def get_obj_by_tags(obj_type, l_tags, date_from=None, date_to=None, nb_obj=50, p for elem in l_tagged_obj: if current_index > stop: break - if start <= current_index and stop >= current_index: + if start <= current_index <= stop: l_obj.append(elem) current_index += 1 l_tagged_obj = l_obj @@ -375,102 +798,6 @@ def get_obj_by_tags(obj_type, l_tags, date_from=None, date_to=None, nb_obj=50, p return {"tagged_obj":l_tagged_obj, "page":page, "nb_pages":nb_pages, "nb_first_elem":start+1, "nb_last_elem":stop, "nb_all_elem":nb_all_elem} - -################################################################################ -################################################################################ -################################################################################ -################################################################################ - -#### Taxonomies - Galaxies #### - -################################################################################ -# galaxies = Galaxies() -# clusters = Clusters(skip_duplicates=True) -# -# list_all_tags = {} -# for name, c in clusters.items(): #galaxy name + tags -# list_all_tags[name] = c -# -# list_galaxies = [] -# for g in galaxies.values(): -# list_galaxies.append(g.to_json()) -# -# list_clusters = [] -# for c in clusters.values(): -# list_clusters.append(c.to_json()) -# -# # tags numbers in galaxies -# total_tags = {} -# for name, tags in clusters.items(): #galaxie name + tags -# total_tags[name] = len(tags) -################################################################################ - -#### Taxonomies #### - -def get_taxonomy_tags_from_cluster(taxonomy_name): - taxonomies = Taxonomies() - taxonomy = taxonomies[taxonomy_name] - return taxonomy.machinetags() - -# TODO: ADD api handler -def enable_taxonomy(taxonomy): - tags = get_taxonomy_tags_from_cluster(taxonomy) - r_tags.sadd('active_taxonomies', taxonomy) - for tag in tags: - r_tags.sadd(f'active_tag_{taxonomy}', tag) - -# def enable_taxonomy(taxonomie, enable_tags=True): -# ''' -# Enable a taxonomy. (UI) -# -# :param taxonomie: MISP taxonomy -# :type taxonomie: str -# :param enable_tags: crawled domain -# :type enable_tags: boolean -# ''' -# taxonomies = Taxonomies() -# if enable_tags: -# taxonomie_info = taxonomies.get(taxonomie) -# if taxonomie_info: -# # activate taxonomie -# r_tags.sadd('active_taxonomies', taxonomie) -# # activate taxonomie tags -# for tag in taxonomie_info.machinetags(): -# r_tags.sadd('active_tag_{}'.format(taxonomie), tag) -# #r_tags.sadd('active_taxonomies_tags', tag) -# else: -# print('Error: {}, please update pytaxonomies'.format(taxonomie)) - -#### Galaxies #### - -def get_galaxy_tags_from_cluster(galaxy_name): - clusters = Clusters(skip_duplicates=True) - cluster = clusters[galaxy_name] - return cluster.machinetags() - -def get_galaxy_tags_with_sysnonym_from_cluster(galaxy_name): - tags = {} - clusters = Clusters(skip_duplicates=True) - cluster = clusters[galaxy_name] - for data in cluster.to_dict()['values']: - tag = f'misp-galaxy:{cluster.type}="{data.value}"' - synonyms = data.meta.synonyms - if not synonyms: - synonyms = [] - tags[tag] = synonyms - return tags - -def enable_galaxy(galaxy): - tags = get_galaxy_tags_with_sysnonym_from_cluster(galaxy) - r_tags.sadd('active_galaxies', galaxy) - for tag in tags: - r_tags.sadd(f'active_tag_galaxies_{galaxy}', tag) - # synonyms - for synonym in tags[tag]: - r_tags.sadd(f'synonym_tag_{tag}', synonym) - - - ################################################################################ ################################################################################ ################################################################################ @@ -490,9 +817,6 @@ def get_galaxy_from_tag(tag): except IndexError: return None -def get_taxonomies(): - return Taxonomies().keys() - def is_taxonomie(taxonomie, taxonomies=[]): if not taxonomies: taxonomies = get_taxonomies() @@ -579,11 +903,11 @@ def is_taxonomie_tag_enabled(taxonomie, tag): else: return False -def is_galaxy_tag_enabled(galaxy, tag): - if tag in r_tags.smembers('active_tag_galaxies_' + galaxy): - return True - else: - return False +# def is_galaxy_tag_enabled(galaxy, tag): +# if tag in r_tags.smembers('active_tag_galaxies_' + galaxy): +# return True +# else: +# return False def is_custom_tag_enabled(tag): return r_tags.sismember('tags:custom:enabled_tags', tag) @@ -748,12 +1072,12 @@ def unpack_str_tags_list(str_tags_list): return [] # used by modal -def get_modal_add_tags(item_id, object_type='item'): +def get_modal_add_tags(object_id, object_type='item', object_subtype=''): ''' Modal: add tags to domain or Paste ''' return {"active_taxonomies": get_active_taxonomies(), "active_galaxies": get_active_galaxies(), - "object_id": item_id, "object_type": object_type} + "object_id": object_id, "object_type": object_type, "object_subtype": object_subtype} ######## NEW VERSION ######## def create_custom_tag(tag): @@ -790,6 +1114,27 @@ def get_all_tags(): def get_all_obj_tags(obj_type): return list(r_tags.smembers(f'list_tags:{obj_type}')) +# # # UI # # # + +def get_enabled_tags_with_synonyms_ui(): + list_tags = [] + for tag in get_all_tags(): + t = tag.split(':')[0] + # add synonym + str_synonyms = ' - synonyms: ' + if t == 'misp-galaxy': + synonyms = get_tag_synonyms(tag) + for synonym in synonyms: + str_synonyms = str_synonyms + synonym + ', ' + # add real tag + if str_synonyms != ' - synonyms: ': + list_tags.append({'name': tag + str_synonyms, 'id': tag}) + else: + list_tags.append({'name': tag, 'id': tag}) + return list_tags + +# - - UI - - # + ## Objects tags ## ################################################################################### @@ -807,59 +1152,59 @@ def add_global_tag(tag, object_type=None): Create a set of all tags used in AIL (all + by object) :param tag: tag - :type domain: str + :type tag: str :param object_type: object type - :type domain: str + :type object_type: str ''' r_tags.sadd('list_tags', tag) if object_type: r_tags.sadd('list_tags:{}'.format(object_type), tag) -def add_obj_tags(object_id, object_type, tags=[], galaxy_tags=[]): - obj_date = get_obj_date(object_type, object_id) +def add_obj_tags(object_id, object_subtype, object_type, tags=[], galaxy_tags=[]): for tag in tags: if tag: - taxonomie = get_taxonomie_from_tag(tag) - if is_taxonomie_tag_enabled(taxonomie, tag): - add_object_tag(tag, object_type, object_id) + taxonomy = get_taxonomie_from_tag(tag) + if is_taxonomie_tag_enabled(taxonomy, tag): + add_object_tag(tag, object_type, object_id, object_subtype) else: - return ({'status': 'error', 'reason': 'Tags or Galaxy not enabled', 'value': tag}, 400) + return {'status': 'error', 'reason': 'Tags or Galaxy not enabled', 'value': tag}, 400 for tag in galaxy_tags: if tag: galaxy = get_galaxy_from_tag(tag) if is_galaxy_tag_enabled(galaxy, tag): - add_object_tag(tag, object_type, object_id) + add_object_tag(tag, object_type, object_id, object_subtype) else: - return ({'status': 'error', 'reason': 'Tags or Galaxy not enabled', 'value': tag}, 400) + return {'status': 'error', 'reason': 'Tags or Galaxy not enabled', 'value': tag}, 400 # TEMPLATE + API QUERY -def api_add_obj_tags(tags=[], galaxy_tags=[], object_id=None, object_type="item"): +# WARNING CHECK IF OBJECT EXISTS +def api_add_obj_tags(tags=[], galaxy_tags=[], object_id=None, object_type="item", object_subtype=''): res_dict = {} - if object_id == None: - return ({'status': 'error', 'reason': 'object_id id not found'}, 404) + if not object_id: + return {'status': 'error', 'reason': 'object_id id not found'}, 404 if not tags and not galaxy_tags: - return ({'status': 'error', 'reason': 'Tags or Galaxy not specified'}, 400) - if object_type not in ('item', 'domain', 'image', 'decoded'): # # TODO: put me in another file - return ({'status': 'error', 'reason': 'Incorrect object_type'}, 400) + return {'status': 'error', 'reason': 'Tags or Galaxy not specified'}, 400 + if object_type not in ail_core.get_all_objects(): + return {'status': 'error', 'reason': 'Incorrect object_type'}, 400 # remove empty tags tags = list(filter(bool, tags)) galaxy_tags = list(filter(bool, galaxy_tags)) - res = add_obj_tags(object_id, object_type, tags=tags, galaxy_tags=galaxy_tags) + res = add_obj_tags(object_id, object_subtype, object_type, tags=tags, galaxy_tags=galaxy_tags) if res: return res res_dict['tags'] = tags + galaxy_tags res_dict['id'] = object_id res_dict['type'] = object_type - return (res_dict, 200) + return res_dict, 200 # def add_tag(object_type, tag, object_id, obj_date=None): # # new tag # if not is_obj_tagged(object_id, tag): -# # # TODO: # FIXME: sanityze object_type +# # # TODO: # FIXME: sanitize object_type # if obj_date: # try: # obj_date = int(obj_date) @@ -950,5 +1295,7 @@ def get_list_of_solo_tags_to_export_by_type(export_type): # by type #r_serv_db.smembers('whitelist_hive') # if __name__ == '__main__': -# galaxy = 'infoleak' -# get_taxonomy_tags_from_cluster(galaxy) +# taxo = 'accessnow' +# # taxo = TAXONOMIES.get(taxo) +# res = is_taxonomy_tag_enabled(taxo, 'test') +# print(res) diff --git a/bin/lib/ail_core.py b/bin/lib/ail_core.py index dc22f21b..7b555afb 100755 --- a/bin/lib/ail_core.py +++ b/bin/lib/ail_core.py @@ -15,14 +15,15 @@ config_loader = ConfigLoader() config_loader = None +AIL_OBJECTS = {'cve', 'cryptocurrency', 'decoded', 'domain', 'item', 'pgp', 'screenshot', 'username'} + def get_ail_uuid(): pass #### AIL OBJECTS #### -# # TODO: check change paste => item def get_all_objects(): - return ['cve', 'domain', 'item', 'pgp', 'cryptocurrency', 'decoded', 'screenshot', 'username'] + return AIL_OBJECTS def get_object_all_subtypes(obj_type): if obj_type == 'cryptocurrency': diff --git a/bin/lib/correlations_engine.py b/bin/lib/correlations_engine.py index 432f6c25..0db0ccdb 100755 --- a/bin/lib/correlations_engine.py +++ b/bin/lib/correlations_engine.py @@ -69,10 +69,10 @@ def get_nb_correlation_by_correl_type(obj_type, subtype, obj_id, correl_type): def get_nb_correlations(obj_type, subtype, obj_id, filter_types=[]): if subtype is None: subtype = '' - nb_correlations = 0 + obj_correlations = {} filter_types = sanityze_obj_correl_types(obj_type, filter_types) for correl_type in filter_types: - obj_correlations += get_nb_correlation_by_correl_type(obj_type, subtype, obj_id, correl_type) + obj_correlations[correl_type] = get_nb_correlation_by_correl_type(obj_type, subtype, obj_id, correl_type) return obj_correlations def get_correlation_by_correl_type(obj_type, subtype, obj_id, correl_type): diff --git a/bin/lib/crawlers.py b/bin/lib/crawlers.py index e106d3e4..c8d06a1a 100755 --- a/bin/lib/crawlers.py +++ b/bin/lib/crawlers.py @@ -546,7 +546,7 @@ def blacklist_domain(domain): def load_blacklist(): try: - with open(os.path.join(os.environ['AIL_BIN'], 'torcrawler/blacklist.txt'), 'r') as f: + with open(os.path.join(os.environ['AIL_BIN'], 'crawlers/blacklist.txt'), 'r') as f: r_crawler.delete('blacklist:domain') lines = f.read().splitlines() for line in lines: diff --git a/bin/lib/objects/CryptoCurrencies.py b/bin/lib/objects/CryptoCurrencies.py index 279e135f..f9b2aede 100755 --- a/bin/lib/objects/CryptoCurrencies.py +++ b/bin/lib/objects/CryptoCurrencies.py @@ -92,7 +92,7 @@ class CryptoCurrency(AbstractSubtypeObject): meta = self._get_meta() meta['id'] = self.id meta['subtype'] = self.subtype - meta['tags'] = self.get_tags() + meta['tags'] = self.get_tags(r_list=True) return meta diff --git a/bin/lib/objects/Cves.py b/bin/lib/objects/Cves.py index 6290dc8c..3e8037dc 100755 --- a/bin/lib/objects/Cves.py +++ b/bin/lib/objects/Cves.py @@ -63,7 +63,7 @@ class Cve(AbstractDaterangeObject): meta = self._get_meta(options=options) meta['id'] = self.id meta['subtype'] = self.subtype - meta['tags'] = self.get_tags() + meta['tags'] = self.get_tags(r_list=True) return meta def add(self, date, item_id): diff --git a/bin/lib/objects/Items.py b/bin/lib/objects/Items.py index a38fb83d..b4bbf460 100755 --- a/bin/lib/objects/Items.py +++ b/bin/lib/objects/Items.py @@ -53,7 +53,7 @@ class Item(AbstractObject): super(Item, self).__init__('item', id) def exists(self): - return os.path.isfile(self.get_filename()) + return item_basic.exist_item(self.id) def get_date(self, separator=False): """ @@ -123,6 +123,9 @@ class Item(AbstractObject): payload = {'raw': self.get_gzip_content(b64=True)} return payload + def get_parent(self): + return item_basic.get_item_parent(self.id) + def set_father(self, father_id): # UPDATE KEYS ????????????????????????????? r_serv_metadata.sadd(f'paste_children:{father_id}', self.id) r_serv_metadata.hset(f'paste_metadata:{self.id}', 'father', father_id) @@ -234,7 +237,7 @@ class Item(AbstractObject): def get_screenshot(self): s = self.get_correlation('screenshot') - if s: + if s.get('screenshot'): s = s['screenshot'].pop()[1:] return os.path.join(s[0:2], s[2:4], s[4:6], s[6:8], s[8:10], s[10:12], s[12:]) @@ -267,10 +270,13 @@ class Item(AbstractObject): if 'lines' in options: content = meta.get('content') meta['lines'] = self.get_meta_lines(content=content) + if 'parent' in options: + meta['parent'] = self.get_parent() if 'size' in options: meta['size'] = self.get_size(str=True) - - # # TODO: ADD GET FATHER + if 'mimetype' in options: + content = meta.get('content') + meta['mimetype'] = self.get_mimetype(content=content) # meta['encoding'] = None return meta diff --git a/bin/lib/objects/Pgps.py b/bin/lib/objects/Pgps.py index 8f58ea47..a9a5cf33 100755 --- a/bin/lib/objects/Pgps.py +++ b/bin/lib/objects/Pgps.py @@ -45,7 +45,7 @@ class Pgp(AbstractSubtypeObject): meta = self._get_meta() meta['id'] = self.id meta['subtype'] = self.subtype - meta['tags'] = self.get_tags() + meta['tags'] = self.get_tags(r_list=True) return meta def get_link(self, flask_context=False): diff --git a/bin/lib/objects/Usernames.py b/bin/lib/objects/Usernames.py index 305e672a..68a58428 100755 --- a/bin/lib/objects/Usernames.py +++ b/bin/lib/objects/Usernames.py @@ -67,7 +67,7 @@ class Username(AbstractSubtypeObject): meta = self._get_meta() meta['id'] = self.id meta['subtype'] = self.subtype - meta['tags'] = self.get_tags() + meta['tags'] = self.get_tags(r_list=True) return meta def get_misp_object(self): diff --git a/bin/lib/objects/abstract_object.py b/bin/lib/objects/abstract_object.py index 1cba7e75..e7f6932e 100755 --- a/bin/lib/objects/abstract_object.py +++ b/bin/lib/objects/abstract_object.py @@ -18,7 +18,7 @@ sys.path.append(os.environ['AIL_BIN']) ################################## from lib import Tag from lib import Duplicate -from lib.correlations_engine import get_correlations, add_obj_correlation, delete_obj_correlation, exists_obj_correlation, is_obj_correlated +from lib.correlations_engine import get_nb_correlations, get_correlations, add_obj_correlation, delete_obj_correlation, exists_obj_correlation, is_obj_correlated from lib.Investigations import is_object_investigated, get_obj_investigations, delete_obj_investigations from lib.Tracker import is_obj_tracked, get_obj_all_trackers, delete_obj_trackers @@ -183,6 +183,9 @@ class AbstractObject(ABC): """ return get_correlations(self.type, self.subtype, self.id) + def get_nb_correlations(self, filter_types=[]): + return get_nb_correlations(self.type, self.subtype, self.id, filter_types=filter_types) + def add_correlation(self, type2, subtype2, id2): """ Add object correlation diff --git a/bin/lib/objects/abstract_subtype_object.py b/bin/lib/objects/abstract_subtype_object.py index 66435353..7a3c7f72 100755 --- a/bin/lib/objects/abstract_subtype_object.py +++ b/bin/lib/objects/abstract_subtype_object.py @@ -10,7 +10,7 @@ import os import sys from abc import abstractmethod -#from flask import url_for +# from flask import url_for sys.path.append(os.environ['AIL_BIN']) ################################## @@ -49,6 +49,13 @@ class AbstractSubtypeObject(AbstractObject): def exists(self): return r_metadata.exists(f'{self.type}_metadata_{self.subtype}:{self.id}') + # def exists(self): + # res = r_metadata.zscore(f'{self.type}_all:{self.subtype}', self.id) + # if res is not None: + # return True + # else: + # return False + def get_first_seen(self, r_int=False): first_seen = r_metadata.hget(f'{self.type}_metadata_{self.subtype}:{self.id}', 'first_seen') if r_int: @@ -70,7 +77,7 @@ class AbstractSubtypeObject(AbstractObject): return last_seen def get_nb_seen(self): - return r_metadata.scard(f'set_{self.type}_{self.subtype}:{self.id}') + return int(r_metadata.zscore(f'{self.type}_all:{self.subtype}', self.id)) # # TODO: CHECK RESULT def get_nb_seen_by_date(self, date_day): @@ -81,22 +88,11 @@ class AbstractSubtypeObject(AbstractObject): return int(nb) def _get_meta(self): - meta_dict = {} - meta_dict['first_seen'] = self.get_first_seen() - meta_dict['last_seen'] = self.get_last_seen() - meta_dict['nb_seen'] = self.get_nb_seen() + meta_dict = {'first_seen': self.get_first_seen(), + 'last_seen': self.get_last_seen(), + 'nb_seen': self.get_nb_seen()} return meta_dict - # def exists(self): - # res = r_metadata.zscore(f'{self.type}_all:{self.subtype}', self.id) - # if res is not None: - # return True - # else: - # return False - - def exists(self): - return r_metadata.exists(f'{self.type}_metadata_{self.subtype}:{self.id}') - def set_first_seen(self, first_seen): r_metadata.hset(f'{self.type}_metadata_{self.subtype}:{self.id}', 'first_seen', first_seen) @@ -137,7 +133,7 @@ class AbstractSubtypeObject(AbstractObject): # daily r_metadata.hincrby(f'{self.type}:{self.subtype}:{date}', self.id, 1) # all subtypes - r_metadata.zincrby(f'{self.type}_all:{self.subtype}', self.id, 1) + r_metadata.zincrby(f'{self.type}_all:{self.subtype}', 1, self.id) ####################################################################### ####################################################################### diff --git a/bin/lib/objects/ail_objects.py b/bin/lib/objects/ail_objects.py index 90b5064c..29d3be4a 100755 --- a/bin/lib/objects/ail_objects.py +++ b/bin/lib/objects/ail_objects.py @@ -109,7 +109,7 @@ def get_object_card_meta(obj_type, subtype, id, related_btc=False): meta["vt"]["status"] = obj.is_vt_enabled() # TAGS MODAL if obj.get_type() == 'screenshot' or obj.get_type() == 'decoded': - meta["add_tags_modal"] = Tag.get_modal_add_tags(obj.id, object_type=obj.get_type()) + meta["add_tags_modal"] = Tag.get_modal_add_tags(obj.id, obj.get_type(), obj.get_subtype(r_str=True)) return meta def get_ui_obj_tag_table_keys(obj_type): diff --git a/bin/PgpDump.py b/bin/modules/PgpDump.py similarity index 100% rename from bin/PgpDump.py rename to bin/modules/PgpDump.py diff --git a/bin/trackers/yara/ail-yara-rules b/bin/trackers/yara/ail-yara-rules index edc390c4..3ccbec31 160000 --- a/bin/trackers/yara/ail-yara-rules +++ b/bin/trackers/yara/ail-yara-rules @@ -1 +1 @@ -Subproject commit edc390c4a8d93a028e29938e92aacb399e270cc4 +Subproject commit 3ccbec3145d3dec0ebf46fb996eb79d0932388d9 diff --git a/crawler_hidden_services_install.sh b/crawler_hidden_services_install.sh deleted file mode 100755 index e5831bf9..00000000 --- a/crawler_hidden_services_install.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/bash - -install_docker() { - # install docker - sudo apt install docker.io; - - # pull splah docker - sudo docker pull scrapinghub/splash; -} - -# install_python_requirement() { -# . ./AILENV/bin/activate; -# pip3 install -U -r crawler_requirements.txt; -# } - -install_all() { - read -p "Do you want to install docker? (use local splash server) [y/n] " -n 1 -r - echo # (optional) move to a new line - if [[ $REPLY =~ ^[Yy]$ ]] - then - install_docker; - fi -} - -usage() { - echo "Usage: crawler_hidden_services_install.sh [-y | -n]" 1>&2; - echo " -y: install docker" - echo " -n: don't install docker" - echo "" - echo "example:" - echo "crawler_hidden_services_install.sh -y" - exit 1; -} - -if [[ $1 == "" ]]; then - install_all; - exit; -else - key="$1" - case $key in - "") - install_all; - ;; - -y|--yes) - install_docker; - install_python_requirement; - ;; - -n|--no) - install_python_requirement; - ;; - *) # unknown option - usage; - ;; - esac -fi diff --git a/files/misp-galaxy b/files/misp-galaxy new file mode 160000 index 00000000..aba1321b --- /dev/null +++ b/files/misp-galaxy @@ -0,0 +1 @@ +Subproject commit aba1321b34e18122ec1825b54e2fc8176a4bd25c diff --git a/files/misp-taxonomies b/files/misp-taxonomies index f2fbd0e5..7aeaa0b8 160000 --- a/files/misp-taxonomies +++ b/files/misp-taxonomies @@ -1 +1 @@ -Subproject commit f2fbd0e5fb02decb350232de16c21251db0e7517 +Subproject commit 7aeaa0b890e16f6d9b349f6db3577bf25a9a40ad diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py index 01833bb4..ff4d727f 100755 --- a/var/www/Flask_server.py +++ b/var/www/Flask_server.py @@ -8,7 +8,6 @@ import json import time import uuid -import redis import random import logging import logging.handlers @@ -16,7 +15,6 @@ import logging.handlers from flask import Flask, render_template, jsonify, request, Request, Response, session, redirect, url_for from flask_login import LoginManager, current_user, login_user, logout_user, login_required -import flask import importlib from os.path import join @@ -90,7 +88,7 @@ if not os.path.isdir(log_dir): # ========= TLS =========# ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2) ssl_context.load_cert_chain(certfile=os.path.join(Flask_dir, 'server.crt'), keyfile=os.path.join(Flask_dir, 'server.key')) -#print(ssl_context.get_ciphers()) +# print(ssl_context.get_ciphers()) # ========= =========# Flask_config.app = Flask(__name__, static_url_path=baseUrl+'/static/') @@ -132,6 +130,7 @@ def load_user(user_id): # ========= HEADER GENERATION ======== DEPRECATED + # Get headers items that should be ignored (not displayed) toIgnoreModule = set() try: @@ -166,7 +165,6 @@ for root, dirs, files in os.walk(os.path.join(Flask_dir, 'modules')): to_add_to_header_dico[module_name] = f.read() # create header.html -complete_header = "" with open(os.path.join(Flask_dir, 'templates', 'header_base.html'), 'r') as f: complete_header = f.read() modified_header = complete_header @@ -250,21 +248,20 @@ def page_not_found(e): # ========== INITIAL taxonomies ============ default_taxonomies = ["infoleak", "gdpr", "fpf", "dark-web"] - # enable default taxonomies -for taxo in default_taxonomies: - Tag.enable_taxonomy(taxo) +for taxonomy in default_taxonomies: + Tag.enable_taxonomy_tags(taxonomy) # ========== INITIAL tags auto export ============ -taxonomies = Taxonomies() - -infoleak_tags = taxonomies.get('infoleak').machinetags() -infoleak_automatic_tags = [] -for tag in taxonomies.get('infoleak').machinetags(): - if tag.split('=')[0][:] == 'infoleak:automatic-detection': - r_serv_db.sadd('list_export_tags', tag) - -r_serv_db.sadd('list_export_tags', 'infoleak:submission="manual"') +# taxonomies = Taxonomies() +# +# infoleak_tags = taxonomies.get('infoleak').machinetags() +# infoleak_automatic_tags = [] +# for tag in taxonomies.get('infoleak').machinetags(): +# if tag.split('=')[0][:] == 'infoleak:automatic-detection': +# r_serv_db.sadd('list_export_tags', tag) +# +# r_serv_db.sadd('list_export_tags', 'infoleak:submission="manual"') # ============ MAIN ============ if __name__ == "__main__": diff --git a/var/www/blueprints/ail_2_ail_sync.py b/var/www/blueprints/ail_2_ail_sync.py index ad0a60d0..d7306285 100644 --- a/var/www/blueprints/ail_2_ail_sync.py +++ b/var/www/blueprints/ail_2_ail_sync.py @@ -8,7 +8,6 @@ import os import sys import json -import random from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response, make_response from flask_login import login_required, current_user, login_user, logout_user @@ -19,15 +18,15 @@ import Flask_config # Import Role_Manager from Role_Manager import login_admin, login_analyst, login_read_only -sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages')) -import Tag +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from core import ail_2_ail +from lib import item_basic +from lib import Tag +from lib import Tracker -sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib')) -import item_basic -import Tracker - -sys.path.append(os.path.join(os.environ['AIL_BIN'], 'core')) -import ail_2_ail bootstrap_label = Flask_config.bootstrap_label @@ -37,7 +36,6 @@ ail_2_ail_sync = Blueprint('ail_2_ail_sync', __name__, template_folder=os.path.j # ============ VARIABLES ============ - # ============ FUNCTIONS ============ def api_validator(api_response): if api_response: diff --git a/var/www/blueprints/correlation.py b/var/www/blueprints/correlation.py index 6dc6fa5c..1b5a5aeb 100644 --- a/var/www/blueprints/correlation.py +++ b/var/www/blueprints/correlation.py @@ -136,7 +136,6 @@ def show_correlation(): "filter": filter_types, "filter_str": ",".join(filter_types), "metadata": ail_objects.get_object_meta(obj_type, subtype, obj_id, flask_context=True) } - print(dict_object) if subtype: dict_object["metadata"]['type_id'] = subtype dict_object["metadata_card"] = ail_objects.get_object_card_meta(obj_type, subtype, obj_id, related_btc=related_btc) @@ -196,4 +195,4 @@ def subtype_search(): obj_type = request.form.get('object_type') obj_subtype = request.form.get('object_subtype') obj_id = request.form.get('object_id') - return redirect(url_for('correlation.show_correlation', object_type=obj_type, type_id=obj_subtype, correlation_id=obj_id)) + return redirect(url_for('correlation.show_correlation', type=obj_type, subtype=obj_subtype, id=obj_id)) diff --git a/var/www/blueprints/hunters.py b/var/www/blueprints/hunters.py index 2f1fb29f..e5ded14a 100644 --- a/var/www/blueprints/hunters.py +++ b/var/www/blueprints/hunters.py @@ -19,9 +19,13 @@ import Flask_config # Import Role_Manager from Role_Manager import login_admin, login_analyst, login_read_only -sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib')) -import item_basic -import Tracker +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from lib import item_basic +from lib import Tracker + bootstrap_label = Flask_config.bootstrap_label diff --git a/var/www/blueprints/import_export.py b/var/www/blueprints/import_export.py index c30fe191..76681f9b 100644 --- a/var/www/blueprints/import_export.py +++ b/var/www/blueprints/import_export.py @@ -20,14 +20,16 @@ import Flask_config # Import Role_Manager from Role_Manager import login_admin, login_analyst, login_read_only -sys.path.append(os.path.join(os.environ['AIL_BIN'], 'export')) -import MispImport -import MispExport +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from export import MispExport +from export import MispImport +from export import AILObjects ####################### # # # # ########################## ## # #################################### -sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib')) -import Correlate_object +from lib.objects import ail_objects -import AILObjects # ============ BLUEPRINT ============ import_export = Blueprint('import_export', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/import_export')) @@ -66,10 +68,11 @@ def import_object_file(): map_uuid_global_id = MispImport.import_objs_from_file(filename) os.remove(filename) for obj_uuid in map_uuid_global_id: - dict_obj = Correlate_object.get_global_id_from_id(map_uuid_global_id[obj_uuid]) + dict_obj = MispImport.get_global_id_from_id(map_uuid_global_id[obj_uuid]) + obj = ail_objects.get_object(dict_obj['type'], dict_obj['subtype'], dict_obj['id']) dict_obj['uuid'] = obj_uuid - dict_obj['url'] = Correlate_object.get_item_url(dict_obj['type'], dict_obj['id'], correlation_type=dict_obj['subtype']) - dict_obj['node'] = Correlate_object.get_correlation_node_icon(dict_obj['type'], correlation_type=dict_obj['subtype'], value=dict_obj['id']) + dict_obj['url'] = obj.get_link(flask_context=True) + dict_obj['node'] = obj.get_svg_icon() all_imported_obj.append(dict_obj) if not all_imported_obj: @@ -129,7 +132,7 @@ def export_object_file(): else: dict_misp_event_export[str(obj_tuple)] = request.form.get(obj_tuple) - #print(dict_misp_event_export) + # print(dict_misp_event_export) if dict_misp_event_export.get('export_to_misp', None): export_to_misp = True @@ -142,10 +145,12 @@ def export_object_file(): for obj_dict in l_obj_invalid: # set uuid input obj_dict['uuid'] = str(uuid.uuid4()) - obj_dict['type'] = Correlate_object.get_obj_str_type_subtype(obj_dict['type'], obj_dict.get('subtype', None)) + subtype = obj_dict.get('subtype', None) + if subtype: + obj_dict['type'] = f'{obj_dict["type"]};{subtype}' return render_template("export_object.html", l_obj_to_export=l_obj_to_export, - l_obj_invalid=l_obj_invalid, dict_misp_event_export=dict_misp_event_export) + l_obj_invalid=l_obj_invalid, dict_misp_event_export=dict_misp_event_export) else: if export_to_misp and MispExport.ping_misp(): event = MispExport.create_list_of_objs_to_export(l_obj_to_export, r_type='event') diff --git a/var/www/blueprints/investigations_b.py b/var/www/blueprints/investigations_b.py index 003a3e84..8c1d592b 100644 --- a/var/www/blueprints/investigations_b.py +++ b/var/www/blueprints/investigations_b.py @@ -18,12 +18,13 @@ from Role_Manager import login_admin, login_analyst, login_read_only sys.path.append('modules') import Flask_config -sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib')) -import Investigations +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from lib import Investigations from lib.objects import ail_objects - -sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages')) -import Tag +from lib import Tag # ============ BLUEPRINT ============ investigations_b = Blueprint('investigations_b', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/investigations')) diff --git a/var/www/blueprints/objects_item.py b/var/www/blueprints/objects_item.py index 27903c54..8edf49e5 100644 --- a/var/www/blueprints/objects_item.py +++ b/var/www/blueprints/objects_item.py @@ -5,20 +5,21 @@ Blueprint Flask: crawler splash endpoints: dashboard, onion crawler ... ''' +import difflib import os import sys -import json -from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response, abort, send_file +from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response, abort, send_file, send_from_directory from flask_login import login_required, current_user # Import Role_Manager -from Role_Manager import login_admin, login_analyst, login_read_only +from Role_Manager import login_admin, login_analyst, login_read_only, no_cache sys.path.append(os.environ['AIL_BIN']) ################################## # Import Project packages ################################## +from lib import ConfigLoader from lib import item_basic from lib.objects.Items import Item from lib import Tag @@ -29,13 +30,25 @@ from export import Export objects_item = Blueprint('objects_item', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/objects/item')) # ============ VARIABLES ============ +config_loader = ConfigLoader.ConfigLoader() bootstrap_label = ['primary', 'success', 'danger', 'warning', 'info'] - +DiffMaxLineLength = config_loader.get_config_int('Flask', 'DiffMaxLineLength') +max_preview_modal = config_loader.get_config_int('Flask', 'max_preview_modal') +SCREENSHOT_FOLDER = ConfigLoader.get_screenshots_dir() +config_loader = None # ============ FUNCTIONS ============ # ============= ROUTES ============== + +@objects_item.route('/screenshot/') +@login_required +@login_read_only +@no_cache +def screenshot(filename): + return send_from_directory(SCREENSHOT_FOLDER, f'{filename}.png', as_attachment=True) + @objects_item.route("/object/item") @login_required @login_read_only @@ -89,9 +102,66 @@ def item_raw_content(): # # TODO: support post @objects_item.route("/object/item/download") @login_required @login_read_only -def item_download(): # # TODO: support post +def item_download(): # # TODO: support post item_id = request.args.get('id') if not item_id or not item_basic.exist_item(item_id): abort(404) item = Item(item_id) return send_file(item.get_raw_content(), download_name=item_id, as_attachment=True) + +@objects_item.route("/object/item/content/more") +@login_required +@login_read_only +def item_content_more(): + item_id = request.args.get('id', '') + item = Item(item_id) + item_content = item.get_content() + to_return = item_content[max_preview_modal-1:] + return to_return + +@objects_item.route("/object/item/diff") +@login_required +@login_analyst +def object_item_diff(): + id1 = request.args.get('s1', '') + id2 = request.args.get('s2', '') + item1 = Item(id1) + item2 = Item(id2) + item1_content = item1.get_content() + item2_content = item2.get_content() + i1_max_len = item1.get_meta_lines(content=item1_content)['max_length'] + i2_max_len = item2.get_meta_lines(content=item2_content)['max_length'] + if i1_max_len > DiffMaxLineLength or i2_max_len > DiffMaxLineLength: + return jsonify({'error': "Can't make the difference as the lines are too long."}), 400 + lines1 = item1_content.splitlines() + lines2 = item2_content.splitlines() + htmldiff = difflib.HtmlDiff() + diff = htmldiff.make_file(lines1, lines2) + return diff + +@objects_item.route("/object/item/preview") +@login_required +@login_read_only +def item_preview(): + item_id = request.args.get('id') + if not item_id or not item_basic.exist_item(item_id): + abort(404) + + item = Item(item_id) + meta = item.get_meta(options={'content', 'crawler', 'lines', 'mimetype', 'parent', 'size'}) + initsize = len(meta['content']) + if len(meta['content']) > max_preview_modal: + meta['content'] = meta['content'][0:max_preview_modal] + meta['nb_correlations'] = item.get_nb_correlations() + + misp_eventid = None # TODO SHOW MISP EVENT + misp_url = None # TODO SHOW MISP EVENT + hive_caseid = None # TODO SHOW HIVE CASE + hive_url = None # TODO SHOW HIVE CASE + + return render_template("show_item_min.html", bootstrap_label=bootstrap_label, + meta=meta, initsize=initsize, + + misp_eventid=misp_eventid, misp_url=misp_url, + hive_caseid=hive_caseid, hive_url=hive_url) + diff --git a/var/www/blueprints/tags_ui.py b/var/www/blueprints/tags_ui.py index a00103ed..fd0eb04d 100644 --- a/var/www/blueprints/tags_ui.py +++ b/var/www/blueprints/tags_ui.py @@ -8,9 +8,8 @@ import os import sys import json -import random -from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response +from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, abort from flask_login import login_required, current_user, login_user, logout_user sys.path.append('modules') @@ -39,6 +38,140 @@ tags_ui = Blueprint('tags_ui', __name__, template_folder=os.path.join(os.environ # ============= ROUTES ============== +@tags_ui.route('/tag/taxonomies') +@login_required +@login_read_only +def tags_taxonomies(): + taxonomies = Tag.get_taxonomies_meta() + return render_template("tags/taxonomies.html", taxonomies=taxonomies) + +@tags_ui.route('/tag/taxonomy') +@login_required +@login_read_only +def tags_taxonomy(): + taxonomy_name = request.args.get('taxonomy') + taxonomy = Tag.get_taxonomy_meta(taxonomy_name, enabled=True, predicates=True, tags=True, enabled_tags=True) + if not taxonomy: + abort(404) + return render_template("tags/taxonomy.html", taxonomy=taxonomy) + +@tags_ui.route('/tag/taxonomy/enable') +@login_required +@login_read_only +def taxonomy_enable(): + taxonomy = request.args.get('taxonomy') + res = Tag.api_enable_taxonomy_tags({'taxonomy': taxonomy}) + if res: + return jsonify(res[0]), res[1] + else: + return redirect(url_for('tags_ui.tags_taxonomy', taxonomy=taxonomy)) + +@tags_ui.route('/tag/taxonomy/disable') +@login_required +@login_read_only +def taxonomy_disable(): + taxonomy = request.args.get('taxonomy') + res = Tag.api_disable_taxonomy_tags({'taxonomy': taxonomy}) + if res: + return jsonify(res[0]), res[1] + else: + return redirect(url_for('tags_ui.tags_taxonomy', taxonomy=taxonomy)) + +@tags_ui.route('/tag/taxonomy/enable_tags') +@login_required +@login_read_only +def taxonomy_enable_tags(): + taxonomy = request.args.get('taxonomy') + tags = request.args.getlist('tags') + res = Tag.api_update_taxonomy_tag_enabled({'taxonomy': taxonomy, 'tags': tags}) + if res: + return jsonify(res[0]), 1 + else: + return redirect(url_for('tags_ui.tags_taxonomy', taxonomy=taxonomy)) + +@tags_ui.route('/tag/galaxies') +@login_required +@login_read_only +def tags_galaxies(): + galaxies = Tag.get_galaxies_meta() + return render_template("tags/galaxies.html", galaxies=galaxies) + +@tags_ui.route('/tag/galaxy') +@login_required +@login_read_only +def tags_galaxy(): + galaxy_name = request.args.get('galaxy') + galaxy = Tag.get_cluster_meta(galaxy_name, enabled=True, tags=True) + if not galaxy: + abort(404) + return render_template("tags/galaxy.html", galaxy=galaxy) + +@tags_ui.route('/tag/galaxy/tag') +@login_required +@login_read_only +def tags_galaxy_tag(): + galaxy_type = request.args.get('galaxy') + tag = request.args.get('tag') + tag_meta = Tag.get_galaxy_tag_meta(galaxy_type, tag) + if not tag_meta: + abort(404) + return render_template("tags/galaxy_tag.html", galaxy=galaxy_type, tag=tag_meta) + +@tags_ui.route('/tag/galaxy/enable') +@login_required +@login_read_only +def galaxy_enable(): + galaxy = request.args.get('galaxy') + res = Tag.api_enable_galaxy_tags({'galaxy': galaxy}) + if res: + return jsonify(res[0]), res[1] + else: + return redirect(url_for('tags_ui.tags_galaxy', galaxy=galaxy)) + +@tags_ui.route('/tag/galaxy/disable') +@login_required +@login_read_only +def galaxy_disable(): + galaxy = request.args.get('galaxy') + res = Tag.api_disable_galaxy_tags({'galaxy': galaxy}) + if res: + return jsonify(res[0]), res[1] + else: + return redirect(url_for('tags_ui.tags_galaxy', galaxy=galaxy)) + +@tags_ui.route('/tag/galaxy/enable_tags') +@login_required +@login_read_only +def galaxy_enable_tags(): + galaxy = request.args.get('galaxy') + tags = request.args.getlist('tags') + res = Tag.api_update_galaxy_tag_enabled({'galaxy': galaxy, 'tags': tags}) + if res: + return jsonify(res[0]), 1 + else: + return redirect(url_for('tags_ui.tags_galaxy', galaxy=galaxy)) + + +@tags_ui.route('/tag/enabled') +@login_required +@login_read_only +def get_all_tags_enabled(): + return jsonify(Tags.get_enabled_tags_with_synonyms_ui()) + +@tags_ui.route('/tag/confirm') +@login_required +@login_read_only +def tag_confirm(): + tag = request.args.get('tag') + obj_type = request.args.get('type') + subtype = request.args.get('subtype', '') + obj_id = request.args.get('id', '') + obj = ail_objects.get_object(obj_type, subtype, obj_id) + if not obj.exists(): + abort(404) + Tag.confirm_tag(tag, obj) + return redirect(obj.get_link(flask_context=True)) + @tags_ui.route('/tag/add_tags') @login_required @login_analyst @@ -46,19 +179,20 @@ def add_tags(): tags = request.args.get('tags') tagsgalaxies = request.args.get('tagsgalaxies') - object_id = request.args.get('object_id') - object_type = request.args.get('object_type') - subtype = '' # TODO: handle subtype object + object_type = request.args.get('type') + object_subtype = request.args.get('subtype') + object_id = request.args.get('id') list_tag = tags.split(',') list_tag_galaxies = tagsgalaxies.split(',') - res = Tag.api_add_obj_tags(tags=list_tag, galaxy_tags=list_tag_galaxies, object_id=object_id, object_type=object_type) + res = Tag.api_add_obj_tags(tags=list_tag, galaxy_tags=list_tag_galaxies, + object_id=object_id, object_type=object_type, object_subtype=object_subtype) # error if res[1] != 200: return str(res[0]) - return redirect(ail_objects.get_object_link(object_type, subtype, object_id, flask_context=True)) + return redirect(ail_objects.get_object_link(object_type, object_subtype, object_id, flask_context=True)) @tags_ui.route('/tag/delete_tag') @login_required @@ -95,7 +229,7 @@ def get_all_obj_tags(): object_type = request.args.get('object_type') res = ail_objects.api_sanitize_object_type(object_type) if res: - return jsonify(res) + return jsonify(res[0]), res[1] return jsonify(Tag.get_all_obj_tags(object_type)) @tags_ui.route('/tag/taxonomies/tags/enabled/json') @@ -137,7 +271,7 @@ def tag_galaxy_tags_enabled_json(): @login_read_only def tags_search_items(): object_type = 'item' - dict_tagged = {"object_type":object_type, "object_name":object_type.title() + "s"} + dict_tagged = {"object_type": object_type, "object_name": object_type.title() + "s"} dict_tagged['date'] = Date.sanitise_date_range('', '', separator='-') return render_template("tags/search_obj_by_tags.html", bootstrap_label=bootstrap_label, dict_tagged=dict_tagged) @@ -146,7 +280,7 @@ def tags_search_items(): @login_read_only def tags_search_domains(): object_type = 'domain' - dict_tagged = {"object_type":object_type, "object_name":object_type.title() + "s"} + dict_tagged = {"object_type": object_type, "object_name": object_type.title() + "s"} return render_template("tags/search_obj_by_tags.html", bootstrap_label=bootstrap_label, dict_tagged=dict_tagged) @tags_ui.route('/tag/search/decoded') @@ -154,15 +288,15 @@ def tags_search_domains(): @login_read_only def tags_search_decoded(): object_type = 'decoded' - dict_tagged = {"object_type":object_type, "object_name":object_type.title() + "s"} + dict_tagged = {"object_type": object_type, "object_name": object_type.title() + "s"} return render_template("tags/search_obj_by_tags.html", bootstrap_label=bootstrap_label, dict_tagged=dict_tagged) -@tags_ui.route('/tag/search/image') +@tags_ui.route('/tag/search/screenshot') @login_required @login_read_only -def tags_search_images(): - object_type = 'image' - dict_tagged = {"object_type":object_type, "object_name":object_type.title() + "s"} +def tags_search_screenshot(): + object_type = 'screenshot' + dict_tagged = {"object_type": object_type, "object_name": object_type.title() + "s"} return render_template("tags/search_obj_by_tags.html", bootstrap_label=bootstrap_label, dict_tagged=dict_tagged) @tags_ui.route('/tag/search/get_obj_by_tags') @@ -170,9 +304,9 @@ def tags_search_images(): @login_read_only def get_obj_by_tags(): - # # TODO: sanityze all + # # TODO: sanitize all object_type = request.args.get('object_type') - subtype = '' # TODO: handle subtype + subtype = '' # TODO: handle subtype ltags = request.args.get('ltags') page = request.args.get('page') date_from = request.args.get('date_from') @@ -188,7 +322,7 @@ def get_obj_by_tags(): list_tags = ltags.split(',') list_tag = [] for tag in list_tags: - list_tag.append(tag.replace('"','\"')) + list_tag.append(tag.replace('"', '\"')) # object_type res = ail_objects.api_sanitize_object_type(object_type) @@ -198,19 +332,21 @@ def get_obj_by_tags(): # page try: page = int(page) - except: + except (TypeError, ValueError): page = 1 + # TODO REPLACE ME dict_obj = Tag.get_obj_by_tags(object_type, list_tag, date_from=date_from, date_to=date_to, page=page) + print(dict_obj) if dict_obj['tagged_obj']: - dict_tagged = {"object_type":object_type, "object_name":object_type.title() + "s", - "tagged_obj":[], "page":dict_obj['page'] ,"nb_pages":dict_obj['nb_pages'], - "nb_first_elem":dict_obj['nb_first_elem'], "nb_last_elem":dict_obj['nb_last_elem'], "nb_all_elem":dict_obj['nb_all_elem']} + dict_tagged = {"object_type": object_type, "object_name": object_type.title() + "s", + "tagged_obj": [], "page": dict_obj['page'], "nb_pages": dict_obj['nb_pages'], + "nb_first_elem": dict_obj['nb_first_elem'], "nb_last_elem": dict_obj['nb_last_elem'], + "nb_all_elem": dict_obj['nb_all_elem']} for obj_id in dict_obj['tagged_obj']: obj_metadata = ail_objects.get_object_meta(object_type, subtype, obj_id, flask_context=True) - #ail_objects. obj_metadata['id'] = obj_id dict_tagged["tagged_obj"].append(obj_metadata) @@ -222,9 +358,9 @@ def get_obj_by_tags(): dict_tagged['current_tags'] = list_tag dict_tagged['current_tags_str'] = ltags - #return jsonify(dict_tagged) + # return jsonify(dict_tagged) else: - dict_tagged = {"object_type":object_type, "object_name":object_type.title() + "s"} + dict_tagged = {"object_type": object_type, "object_name": object_type.title() + "s"} if 'date' in dict_obj: dict_tagged['date'] = dict_obj['date'] diff --git a/var/www/modules/Flask_config.py b/var/www/modules/Flask_config.py index b026db1b..f33b0daa 100644 --- a/var/www/modules/Flask_config.py +++ b/var/www/modules/Flask_config.py @@ -29,6 +29,9 @@ r_serv = config_loader.get_redis_conn("Redis_Queues") r_cache = config_loader.get_redis_conn("Redis_Cache") r_serv_log = config_loader.get_redis_conn("Redis_Log") r_serv_log_submit = config_loader.get_redis_conn("Redis_Log_submit") + + + r_serv_charts = config_loader.get_redis_conn("ARDB_Trending") r_serv_sentiment = config_loader.get_redis_conn("ARDB_Sentiment") r_serv_term = config_loader.get_redis_conn("ARDB_Tracker") diff --git a/var/www/modules/Tags/Flask_Tags.py b/var/www/modules/Tags/Flask_Tags.py index dc18aa30..11e67986 100644 --- a/var/www/modules/Tags/Flask_Tags.py +++ b/var/www/modules/Tags/Flask_Tags.py @@ -13,8 +13,6 @@ from flask_login import login_required import json import datetime -import Paste - from pytaxonomies import Taxonomies from pymispgalaxies import Galaxies, Clusters @@ -25,38 +23,10 @@ from lib import Tag app = Flask_config.app baseUrl = Flask_config.baseUrl r_serv_tags = Flask_config.r_serv_tags -r_serv_metadata = Flask_config.r_serv_metadata -r_serv_statistics = Flask_config.r_serv_statistics -max_preview_char = Flask_config.max_preview_char -max_preview_modal = Flask_config.max_preview_modal -bootstrap_label = Flask_config.bootstrap_label -max_tags_result = Flask_config.max_tags_result Tags = Blueprint('Tags', __name__, template_folder='templates') -galaxies = Galaxies() -clusters = Clusters(skip_duplicates=True) - -list_all_tags = {} -for name, c in clusters.items(): #galaxy name + tags - list_all_tags[name] = c - -list_galaxies = [] -for g in galaxies.values(): - list_galaxies.append(g.to_json()) - -list_clusters = [] -for c in clusters.values(): - list_clusters.append(c.to_json()) - -# tags numbers in galaxies -total_tags = {} -for name, tags in clusters.items(): #galaxie name + tags - total_tags[name] = len(tags) - # ============ FUNCTIONS ============ -def one(): - return 1 # TODO: # TODO: @@ -77,230 +47,22 @@ def get_tags_with_synonyms(tag): else: return {'name':tag,'id':tag} -def get_item_date(item_filename): - l_directory = item_filename.split('/') - return '{}{}{}'.format(l_directory[-4], l_directory[-3], l_directory[-2]) -def substract_date(date_from, date_to): - date_from = datetime.date(int(date_from[0:4]), int(date_from[4:6]), int(date_from[6:8])) - date_to = datetime.date(int(date_to[0:4]), int(date_to[4:6]), int(date_to[6:8])) - delta = date_to - date_from # timedelta - l_date = [] - for i in range(delta.days + 1): - date = date_from + datetime.timedelta(i) - l_date.append( date.strftime('%Y%m%d') ) - return l_date -def get_all_dates_range(date_from, date_to): - all_dates = {} - date_range = [] - if date_from is not None and date_to is not None: - #change format - try: - if len(date_from) != 8: - date_from = date_from[0:4] + date_from[5:7] + date_from[8:10] - date_to = date_to[0:4] + date_to[5:7] + date_to[8:10] - date_range = substract_date(date_from, date_to) - except: - pass - if not date_range: - date_range.append(datetime.date.today().strftime("%Y%m%d")) - date_from = date_range[0][0:4] + '-' + date_range[0][4:6] + '-' + date_range[0][6:8] - date_to = date_from - - else: - date_from = date_from[0:4] + '-' + date_from[4:6] + '-' + date_from[6:8] - date_to = date_to[0:4] + '-' + date_to[4:6] + '-' + date_to[6:8] - all_dates['date_from'] = date_from - all_dates['date_to'] = date_to - all_dates['date_range'] = date_range - return all_dates - -def get_last_seen_from_tags_list(list_tags): - min_last_seen = 99999999 - for tag in list_tags: - tag_last_seen = r_serv_tags.hget('tag_metadata:{}'.format(tag), 'last_seen') - if tag_last_seen: - tag_last_seen = int(tag_last_seen) - if tag_last_seen < min_last_seen: - min_last_seen = tag_last_seen - return str(min_last_seen) # ============= ROUTES ============== -# @Tags.route("/tags/", methods=['GET']) -# @login_required -# @login_read_only -# def Tags_page(): -# date_from = request.args.get('date_from') -# date_to = request.args.get('date_to') -# tags = request.args.get('ltags') -# -# if tags is None: -# dates = get_all_dates_range(date_from, date_to) -# return render_template("Tags.html", date_from=dates['date_from'], date_to=dates['date_to']) -# -# # unpack tags -# list_tags = tags.split(',') -# list_tag = [] -# for tag in list_tags: -# list_tag.append(tag.replace('"','\"')) -# -# #no search by date, use last_seen for date_from/date_to -# if date_from is None and date_to is None and tags is not None: -# date_from = get_last_seen_from_tags_list(list_tags) -# date_to = date_from -# -# # TODO verify input -# -# dates = get_all_dates_range(date_from, date_to) -# -# if(type(list_tags) is list): -# # no tag -# if list_tags is False: -# print('empty') -# # 1 tag -# elif len(list_tags) < 2: -# tagged_pastes = [] -# for date in dates['date_range']: -# tagged_pastes.extend(r_serv_tags.smembers('{}:{}'.format(list_tags[0], date))) -# -# # 2 tags or more -# else: -# tagged_pastes = [] -# for date in dates['date_range']: -# tag_keys = [] -# for tag in list_tags: -# tag_keys.append('{}:{}'.format(tag, date)) -# -# if len(tag_keys) > 1: -# daily_items = r_serv_tags.sinter(tag_keys[0], *tag_keys[1:]) -# else: -# daily_items = r_serv_tags.sinter(tag_keys[0]) -# tagged_pastes.extend(daily_items) -# -# else : -# return 'INCORRECT INPUT' -# -# all_content = [] -# paste_date = [] -# paste_linenum = [] -# all_path = [] -# allPastes = list(tagged_pastes) -# paste_tags = [] -# -# try: -# page = int(request.args.get('page')) -# except: -# page = 1 -# if page <= 0: -# page = 1 -# nb_page_max = len(tagged_pastes)/(max_tags_result) -# if not nb_page_max.is_integer(): -# nb_page_max = int(nb_page_max)+1 -# else: -# nb_page_max = int(nb_page_max) -# if page > nb_page_max: -# page = nb_page_max -# start = max_tags_result*(page -1) -# stop = max_tags_result*page -# -# for path in allPastes[start:stop]: -# all_path.append(path) -# paste = Paste.Paste(path) -# content = paste.get_p_content() -# content_range = max_preview_char if len(content)>max_preview_char else len(content)-1 -# all_content.append(content[0:content_range].replace("\"", "\'").replace("\r", " ").replace("\n", " ")) -# curr_date = str(paste._get_p_date()) -# curr_date = curr_date[0:4]+'/'+curr_date[4:6]+'/'+curr_date[6:] -# paste_date.append(curr_date) -# paste_linenum.append(paste.get_lines_info()[0]) -# p_tags = r_serv_metadata.smembers('tag:'+path) -# complete_tags = [] -# l_tags = [] -# for tag in p_tags: -# complete_tag = tag -# -# tag = tag.split('=') -# if len(tag) > 1: -# if tag[1] != '': -# tag = tag[1][1:-1] -# # no value -# else: -# tag = tag[0][1:-1] -# # use for custom tags -# else: -# tag = tag[0] -# -# l_tags.append( (tag,complete_tag) ) -# -# paste_tags.append(l_tags) -# -# if len(allPastes) > 10: -# finished = False -# else: -# finished = True -# -# if len(list_tag) == 1: -# tag_nav=tags.replace('"', '').replace('=', '').replace(':', '') -# else: -# tag_nav='empty' -# -# return render_template("Tags.html", -# all_path=all_path, -# tags=tags, -# tag_nav=tag_nav, -# list_tag = list_tag, -# date_from=dates['date_from'], -# date_to=dates['date_to'], -# page=page, nb_page_max=nb_page_max, -# paste_tags=paste_tags, -# bootstrap_label=bootstrap_label, -# content=all_content, -# paste_date=paste_date, -# paste_linenum=paste_linenum, -# char_to_display=max_preview_modal, -# finished=finished) - - -@Tags.route("/Tags/get_all_tags") -@login_required -@login_read_only -def get_all_tags(): - - all_tags = r_serv_tags.smembers('list_tags') - - list_tags = [] - for tag in all_tags: - t = tag.split(':')[0] - # add synonym - str_synonyms = ' - synonyms: ' - if t == 'misp-galaxy': - synonyms = r_serv_tags.smembers('synonym_tag_' + tag) - for synonym in synonyms: - str_synonyms = str_synonyms + synonym + ', ' - # add real tag - if str_synonyms != ' - synonyms: ': - list_tags.append({'name':tag + str_synonyms,'id':tag}) - else: - list_tags.append({'name':tag,'id':tag}) - - return jsonify(list_tags) - @Tags.route("/Tags/get_all_tags_taxonomies") @login_required @login_read_only def get_all_tags_taxonomies(): - taxonomies = Taxonomies() - list_taxonomies = list(taxonomies.keys()) - active_taxonomie = r_serv_tags.smembers('active_taxonomies') list_tags = [] for taxonomie in active_taxonomie: - #l_tags = taxonomies.get(taxonomie).machinetags() + # l_tags = taxonomies.get(taxonomie).machinetags() l_tags = r_serv_tags.smembers('active_tag_' + taxonomie) for tag in l_tags: list_tags.append( tag ) @@ -372,568 +134,36 @@ def get_tags_galaxy(): else: return 'this galaxy is disable' -@Tags.route("/Tags/remove_tag") -@login_required -@login_analyst -def remove_tag(): #TODO remove me , used by showpaste +# @Tags.route("/Tags/tag_validation") +# @login_required +# @login_analyst +# def tag_validation(): +# +# path = request.args.get('paste') +# tag = request.args.get('tag') +# status = request.args.get('status') +# +# if (status == 'fp' or status == 'tp') and r_serv_tags.sismember('list_tags', tag): +# +# if status == 'tp': +# r_serv_statistics.sadd('tp:'+tag, path) +# r_serv_statistics.srem('fp:'+tag, path) +# else: +# r_serv_statistics.sadd('fp:'+tag, path) +# r_serv_statistics.srem('tp:'+tag, path) +# +# return redirect(url_for('objects_item.showItem', id=path)) +# else: +# return 'input error' - path = request.args.get('paste') - tag = request.args.get('tag') - res = Tag.api_delete_obj_tags(tags=[tag], object_id=path, object_type="item") - if res[1] != 200: - return str(res[0]) - return redirect(url_for('objects_item.showItem', id=path)) -@Tags.route("/Tags/confirm_tag") -@login_required -@login_analyst -def confirm_tag(): - #TODO verify input - path = request.args.get('paste') - tag = request.args.get('tag') - if(tag[9:28] == 'automatic-detection'): - Tag.api_delete_obj_tags(tags=[tag], object_id=path, object_type="item") - tag = tag.replace('automatic-detection','analyst-detection', 1) - #add analyst tag - Tag.add_tag('item', tag, path) - return redirect(url_for('objects_item.showItem', id=path)) - return 'incompatible tag' -@Tags.route("/Tags/tag_validation") -@login_required -@login_analyst -def tag_validation(): - - path = request.args.get('paste') - tag = request.args.get('tag') - status = request.args.get('status') - - if (status == 'fp' or status == 'tp') and r_serv_tags.sismember('list_tags', tag): - - if status == 'tp': - r_serv_statistics.sadd('tp:'+tag, path) - r_serv_statistics.srem('fp:'+tag, path) - else: - r_serv_statistics.sadd('fp:'+tag, path) - r_serv_statistics.srem('tp:'+tag, path) - - return redirect(url_for('objects_item.showItem', id=path)) - else: - return 'input error' - -@Tags.route("/Tags/taxonomies") -@login_required -@login_read_only -def taxonomies(): - - active_taxonomies = r_serv_tags.smembers('active_taxonomies') - - taxonomies = Taxonomies() - list_taxonomies = list(taxonomies.keys()) - - id = [] - name = [] - description = [] - version = [] - enabled = [] - n_tags = [] - - for taxonomie in list_taxonomies: - id.append(taxonomie) - name.append(taxonomies.get(taxonomie).name) - description.append(taxonomies.get(taxonomie).description) - version.append(taxonomies.get(taxonomie).version) - if taxonomie in active_taxonomies: - enabled.append(True) - else: - enabled.append(False) - - n = str(r_serv_tags.scard('active_tag_' + taxonomie)) - n_tags.append(n + '/' + str(len(taxonomies.get(taxonomie).machinetags())) ) - - return render_template("taxonomies.html", - id=id, - all_name = name, - description = description, - version = version, - enabled = enabled, - n_tags=n_tags) - -@Tags.route("/Tags/edit_taxonomie") -@login_required -@login_analyst -def edit_taxonomie(): - - taxonomies = Taxonomies() - list_taxonomies = list(taxonomies.keys()) - - id = request.args.get('taxonomie') - - #verify input - if id in list(taxonomies.keys()): - active_tag = r_serv_tags.smembers('active_tag_' + id) - list_tag = taxonomies.get(id).machinetags() - list_tag_desc = taxonomies.get(id).machinetags_expanded() - - active_taxonomies = r_serv_tags.smembers('active_taxonomies') - if id in active_taxonomies: - active = True - else: - active = False - - n = str(r_serv_tags.scard('active_tag_' + id)) - badge = n + '/' + str(len(taxonomies.get(id).machinetags())) - - name = taxonomies.get(id).name - description = taxonomies.get(id).description - version = taxonomies.get(id).version - - status = [] - for tag in list_tag: - if tag in active_tag: - status.append(True) - else: - status.append(False) - - return render_template("edit_taxonomie.html", - id=id, - name=name, - badge = badge, - description = description, - version = version, - active=active, - all_tags = list_tag, - list_tag_desc=list_tag_desc, - status = status) - - else: - return 'INVALID TAXONOMIE' - -@Tags.route("/Tags/disable_taxonomie") -@login_required -@login_analyst -def disable_taxonomie(): - - taxonomies = Taxonomies() - list_taxonomies = list(taxonomies.keys()) - - id = request.args.get('taxonomie') - - if id in list_taxonomies: - r_serv_tags.srem('active_taxonomies', id) - for tag in taxonomies.get(id).machinetags(): - r_serv_tags.srem('active_tag_' + id, tag) - - return redirect(url_for('Tags.taxonomies')) - - else: - return "INCORRECT INPUT" - - - -@Tags.route("/Tags/active_taxonomie") -@login_required -@login_analyst -def active_taxonomie(): - - taxonomies = Taxonomies() - list_taxonomies = list(taxonomies.keys()) - - id = request.args.get('taxonomie') - - # verify input - if id in list_taxonomies: - r_serv_tags.sadd('active_taxonomies', id) - for tag in taxonomies.get(id).machinetags(): - r_serv_tags.sadd('active_tag_' + id, tag) - - return redirect(url_for('Tags.taxonomies')) - - else: - return "INCORRECT INPUT" - -@Tags.route("/Tags/edit_taxonomie_tag") -@login_required -@login_analyst -def edit_taxonomie_tag(): - - taxonomies = Taxonomies() - list_taxonomies = list(taxonomies.keys()) - - arg1 = request.args.getlist('tag_enabled') - arg2 = request.args.getlist('tag_disabled') - - id = request.args.get('taxonomie') - - #verify input - if id in list_taxonomies: - list_tag = taxonomies.get(id).machinetags() - - #check tags validity - if ( all(elem in list_tag for elem in arg1) or (len(arg1) == 0) ) and ( all(elem in list_tag for elem in arg2) or (len(arg2) == 0) ): - - active_tag = r_serv_tags.smembers('active_tag_' + id) - - diff = list(set(arg1) ^ set(list_tag)) - - #remove tags - for tag in diff: - r_serv_tags.srem('active_tag_' + id, tag) - - #all tags unchecked - if len(arg1) == 0 and len(arg2) == 0: - r_serv_tags.srem('active_taxonomies', id) - - #add new tags - for tag in arg2: - r_serv_tags.sadd('active_taxonomies', id) - r_serv_tags.sadd('active_tag_' + id, tag) - - return redirect(url_for('Tags.taxonomies')) - else: - return "INCORRECT INPUT" - - else: - return "INCORRECT INPUT" - -@Tags.route("/Tags/galaxies") -@login_required -@login_read_only -def galaxies(): - - active_galaxies = r_serv_tags.smembers('active_galaxies') - - name = [] - icon = [] - version = [] - all_type = [] - namespace = [] - description = [] - enabled = [] - n_tags = [] - - for galaxie_json in list_galaxies: - - galaxie = json.loads(galaxie_json) - - name.append(galaxie['name']) - icon.append(galaxie['icon']) - version.append(galaxie['version']) - type = galaxie['type'] - all_type.append(type) - namespace.append(galaxie['namespace']) - description.append(galaxie['description']) - - - if type in active_galaxies: - enabled.append(True) - else: - enabled.append(False) - - n = str(r_serv_tags.scard('active_tag_galaxies_' + type)) - n_tags.append(n + '/' + str(total_tags[type]) ) - - return render_template("galaxies.html", - name=name, - icon = icon, - version = version, - description = description, - namespace = namespace, - all_type = all_type, - enabled = enabled, - n_tags=n_tags) - - -@Tags.route("/Tags/edit_galaxy") -@login_required -@login_analyst -def edit_galaxy(): - - id = request.args.get('galaxy') - - for clusters_json in list_clusters: - - #get clusters - cluster = json.loads(clusters_json) - - if cluster['type'] == id: - - type = id - active_tag = r_serv_tags.smembers('active_tag_galaxies_' + type) - - n = str(r_serv_tags.scard('active_tag_galaxies_' + type)) - badge = n + '/' + str(total_tags[type]) - - name = cluster['name'] - description = cluster['description'] - version = cluster['version'] - source = cluster['source'] - - val = cluster['values'] - - tags = [] - for data in val: - try: - meta = data['meta'] - except KeyError: - meta = [] - tag_name = data['value'] - tag_name = 'misp-galaxy:{}="{}"'.format(type, tag_name) - try: - tag_description = data['description'] - except KeyError: - tag_description = '' - - tags.append( (tag_name, tag_description, meta) ) - - status = [] - for tag in tags: - if tag[0] in active_tag: - status.append(True) - else: - status.append(False) - - active_galaxies = r_serv_tags.smembers('active_galaxies') - if id in active_galaxies: - active = True - else: - active = False - - return render_template("edit_galaxy.html", - id = type, - name = name, - badge = badge, - description = description, - version = version, - active = active, - tags = tags, - status = status) - - - return 'INVALID GALAXY' - - -@Tags.route("/Tags/active_galaxy") -@login_required -@login_analyst -def active_galaxy(): - - id = request.args.get('galaxy') - - # verify input - try: - l_tags = list_all_tags[id] - except KeyError: - return "INCORRECT INPUT" - - r_serv_tags.sadd('active_galaxies', id) - for tag in l_tags: - r_serv_tags.sadd('active_tag_galaxies_' + id, 'misp-galaxy:{}="{}"'.format(id, tag)) - - #save synonyms - for clusters_json in list_clusters: - - #get clusters - cluster = json.loads(clusters_json) - - if cluster['type'] == id: - - val = cluster['values'] - - tags = [] - for data in val: - try: - meta = data['meta'] - synonyms = meta['synonyms'] - tag_name = data['value'] - tag_name = 'misp-galaxy:{}="{}"'.format(id, tag_name) - #save synonyms - for synonym in synonyms: - r_serv_tags.sadd('synonym_tag_' + tag_name, synonym) - - except KeyError: - pass - - break - - return redirect(url_for('Tags.galaxies')) - - -@Tags.route("/Tags/disable_galaxy") -@login_required -@login_analyst -def disable_galaxy(): - - id = request.args.get('galaxy') - - # verify input - try: - l_tags = list_all_tags[id] - except KeyError: - return "INCORRECT INPUT" - - r_serv_tags.srem('active_galaxies', id) - for tag in l_tags: - tag_name = 'misp-galaxy:{}="{}"'.format(id, tag) - r_serv_tags.srem('active_tag_galaxies_' + id, tag_name) - r_serv_tags.delete('synonym_tag_' + tag_name) - - return redirect(url_for('Tags.galaxies')) - - -@Tags.route("/Tags/edit_galaxy_tag") -@login_required -@login_analyst -def edit_galaxy_tag(): - - arg1 = request.args.getlist('tag_enabled') - arg2 = request.args.getlist('tag_disabled') - - id = request.args.get('galaxy') - - #verify input - try: - l_tags = list_all_tags[id] - except KeyError: - return "INCORRECT INPUT" - - #get full tags - list_tag = [] - for tag in l_tags: - list_tag.append('misp-galaxy:{}="{}"'.format(id, tag)) - - - #check tags validity - if ( all(elem in list_tag for elem in arg1) or (len(arg1) == 0) ) and ( all(elem in list_tag for elem in arg2) or (len(arg2) == 0) ): - - active_tag = r_serv_tags.smembers('active_tag_galaxies_' + id) - - diff = list(set(arg1) ^ set(list_tag)) - - #remove tags - for tag in diff: - r_serv_tags.srem('active_tag_galaxies_' + id, tag) - r_serv_tags.delete('synonym_tag_' + tag) - - #all tags unchecked - if len(arg1) == 0 and len(arg2) == 0: - r_serv_tags.srem('active_galaxies', id) - - #add new tags - for tag in arg2: - r_serv_tags.sadd('active_galaxies', id) - r_serv_tags.sadd('active_tag_galaxies_' + id, tag) - - #get tags synonyms - for clusters_json in list_clusters: - - #get clusters - cluster = json.loads(clusters_json) - - if cluster['type'] == id: - - val = cluster['values'] - - tags = [] - for data in val: - try: - meta = data['meta'] - synonyms = meta['synonyms'] - tag_name = data['value'] - tag_name = 'misp-galaxy:{}="{}"'.format(id, tag_name) - if tag_name in arg2: - #save synonyms - for synonym in synonyms: - r_serv_tags.sadd('synonym_tag_' + tag_name, synonym) - - except KeyError: - pass - break - - return redirect(url_for('Tags.galaxies')) - - else: - return "INCORRECT INPUT" - -@Tags.route("/Tags/tag_galaxy_info") -@login_required -@login_read_only -def tag_galaxy_info(): - - galaxy = request.args.get('galaxy') - tag = request.args.get('tag') - - full_tag = tag - title = tag.split(':')[1] - tag = tag.split('=')[1] - tag = tag[1:-1] - - #get clusters - for clusters_json in list_clusters: - cluster = json.loads(clusters_json) - - if cluster['type'] == galaxy: - val = cluster['values'] - source = cluster['source'] - - for data in val: - if tag == data['value']: - try: - description = data['description'] - except KeyError: - description = '' - if r_serv_tags.sismember('active_tag_galaxies_' + galaxy, full_tag): - active = True - else: - active = False - - synonyms = [] - metadata = [] - list_metadata = [] - try: - meta = data['meta'] - for key in meta: - if key != 'synonyms': - if type(meta[key]) is list: - for item in meta[key]: - list_metadata.append(key + ' : ' + item) - else: - list_metadata.append(key + ' : ' + meta[key]) - try: - synonyms = meta['synonyms'] - bool_synonyms = True - except KeyError: - synonyms = [] - bool_synonyms = False - except KeyError: - pass - - if synonyms: - bool_synonyms = True - else: - bool_synonyms = False - if list_metadata: - metadata = True - else: - metadata = False - - return render_template("tag_galaxy_info.html", - title = title, - description = description, - source = source, - active = active, - synonyms = synonyms, - bool_synonyms = bool_synonyms, - metadata = metadata, - list_metadata = list_metadata) - - return 'INVALID INPUT' # ========= REGISTRATION ========= app.register_blueprint(Tags, url_prefix=baseUrl) diff --git a/var/www/modules/Tags/templates/Tags.html b/var/www/modules/Tags/templates/Tags.html deleted file mode 100644 index 83ebfb3c..00000000 --- a/var/www/modules/Tags/templates/Tags.html +++ /dev/null @@ -1,407 +0,0 @@ - - - - - Tags - AIL - - - - - - - - - - - - - - - - - - - - - - - - - {% include 'nav_bar.html' %} - - - - -
-
- - {% include 'tags/menu_sidebar.html' %} - -
- -
-
-
Search Tags by date range :
-
-
- -
-
-
-
- -
-
-
-
-
- -
-
-
- -
-
- -
- -
- - - -
-
- -
- {%if all_path%} - - - - - - - - - - - - {% for path in all_path %} - - - - - - - {% endfor %} - - -
DatePath# of linesAction
{{ paste_date[loop.index0] }} -
{{ path }}
- -
- {% for tag in paste_tags[loop.index0] %} - - {{ tag[0] }} - - {% endfor %} -
- 		{{ paste_linenum[loop.index0] }}

- -

-
- -
- -
- - {%endif%} -
- -
-

- - -

- Edit Taxonomies List - - - -

- Edit Galaxies List - -
- -
-

- - -

- MISP and Hive, auto push - -
- -
-
-
- - - - - - - - - - - - diff --git a/var/www/modules/Tags/templates/tagged.html b/var/www/modules/Tags/templates/tagged.html deleted file mode 100644 index 4177e6cc..00000000 --- a/var/www/modules/Tags/templates/tagged.html +++ /dev/null @@ -1,317 +0,0 @@ - - - - - - - - Tags - AIL - - - - - - - - - - - - - - - - - - - - - {% include 'navbar.html' %} - - - - -
-
-
-

Tags

-
- -
- -
- - -
- -
- -
- - - - - - - - - - - - - - - - {% for path in all_path %} - - - - - - - - {% endfor %} - - -
#PathDate# of linesAction
{{ loop.index0 }}{{ path }} -
- {% for tag in paste_tags[loop.index0] %} - - {{ tag[0] }} - - {% endfor %} -
- 		{{ paste_date[loop.index0] }}{{ paste_linenum[loop.index0] }}

- -
-
- -
-
- - -
-
- -
-

- - -

- Edit Taxonomies List - - - -

- Edit Galaxies List - -
- -
-

- - -

- MISP and Hive, auto push - -
- -
- - - - - - - - - diff --git a/var/www/modules/dashboard/Flask_dashboard.py b/var/www/modules/dashboard/Flask_dashboard.py index 92a28772..7a9f3247 100644 --- a/var/www/modules/dashboard/Flask_dashboard.py +++ b/var/www/modules/dashboard/Flask_dashboard.py @@ -28,7 +28,6 @@ import Flask_config app = Flask_config.app config_loader = Flask_config.config_loader baseUrl = Flask_config.baseUrl -r_serv = Flask_config.r_serv r_serv_log = Flask_config.r_serv_log max_dashboard_logs = Flask_config.max_dashboard_logs @@ -53,7 +52,7 @@ def event_stream(): if msg['type'] == 'pmessage' and level != "DEBUG": yield 'data: %s\n\n' % json.dumps(msg) -def get_queues(r): +def get_queues(): # We may want to put the llen in a pipeline to do only one query. return queues_modules.get_all_modules_queues_stats() @@ -63,22 +62,22 @@ def get_date_range(date_from, num_day): for i in range(0, num_day+1): new_date = date.substract_day(i) - date_list.append(new_date[0:4] +'-'+ new_date[4:6] +'-'+ new_date[6:8]) + date_list.append(f'{new_date[0:4]}-{new_date[4:6]}-{new_date[6:8]}') return date_list def dashboard_alert(log): # check if we need to display this log - if len(log)>50: + if len(log) > 50: date = log[1:5]+log[6:8]+log[9:11] utc_str = log[1:20] log = log[46:].split(';') if len(log) == 6: - time = datetime_from_utc_to_local(utc_str) - path = url_for('objects_item.showItem',id=log[5]) + date_time = datetime_from_utc_to_local(utc_str) + path = url_for('objects_item.showItem', id=log[5]) - res = {'date': date, 'time': time, 'script': log[0], 'domain': log[1], 'date_paste': log[2], - 'paste': log[3], 'message': log[4], 'path': path} + res = {'date': date, 'time': date_time, 'script': log[0], 'domain': log[1], 'date_paste': log[2], + 'paste': log[3], 'message': log[4], 'path': path} return res else: return False @@ -116,7 +115,7 @@ def get_last_logs_json(): date_range = get_date_range(date, max_day_search) while max_day_search != day_search and warning_found != warning_to_found: - filename_warning_log = 'logs/Script_warn-'+ date_range[day_search] +'.log' + filename_warning_log = f'logs/Script_warn-{date_range[day_search]}.log' filename_log = os.path.join(os.environ['AIL_HOME'], filename_warning_log) try: @@ -147,7 +146,7 @@ def get_last_logs_json(): @login_required @login_read_only def stuff(): - return jsonify(row1=get_queues(r_serv)) + return jsonify(row1=get_queues()) # TODO: ADD UPDATE NOTE BY USER @@ -169,9 +168,11 @@ def index(): background_update = True update_message = ail_updates.get_update_background_message() - return render_template("index.html", default_minute = default_minute, threshold_stucked_module=threshold_stucked_module, - log_select=log_select, selected=max_dashboard_logs, - background_update=background_update, update_message=update_message) + return render_template("index.html", default_minute = default_minute, + threshold_stucked_module=threshold_stucked_module, + log_select=log_select, selected=max_dashboard_logs, + background_update=background_update, update_message=update_message) + # ========= REGISTRATION ========= app.register_blueprint(dashboard, url_prefix=baseUrl) diff --git a/var/www/modules/hiddenServices/Flask_hiddenServices.py b/var/www/modules/hiddenServices/Flask_hiddenServices.py index 960f3ae0..9c3e316d 100644 --- a/var/www/modules/hiddenServices/Flask_hiddenServices.py +++ b/var/www/modules/hiddenServices/Flask_hiddenServices.py @@ -17,7 +17,7 @@ from Role_Manager import login_admin, login_analyst, login_read_only, no_cache from flask_login import login_required from Date import Date -from HiddenServices import HiddenServices +sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib')) import crawlers # ============ VARIABLES ============ @@ -30,9 +30,6 @@ r_serv_onion = Flask_config.r_serv_onion r_serv_metadata = Flask_config.r_serv_metadata bootstrap_label = Flask_config.bootstrap_label -sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib')) -import crawlers - hiddenServices = Blueprint('hiddenServices', __name__, template_folder='templates') faup = Faup() diff --git a/var/www/modules/rawSkeleton/Flask_rawSkeleton.py b/var/www/modules/rawSkeleton/Flask_rawSkeleton.py deleted file mode 100644 index e0bb1e9c..00000000 --- a/var/www/modules/rawSkeleton/Flask_rawSkeleton.py +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env python3 -# -*-coding:UTF-8 -* - -''' - Flask functions and routes for the trending modules page -''' -import redis -from flask import Flask, render_template, jsonify, request, Blueprint - -from Role_Manager import login_admin, login_analyst, login_read_only -from flask_login import login_required - -# ============ VARIABLES ============ -import Flask_config - -app = Flask_config.app - -rawSkeleton = Blueprint('rawSkeleton', __name__, template_folder='templates') - -# ============ FUNCTIONS ============ -def one(): - return 1 - -# ============= ROUTES ============== - -@rawSkeleton.route("/rawSkeleton/", methods=['GET']) -@login_required -@login_read_only -def skeleton_page(): - return render_template("rawSkeleton.html") - - -# ========= REGISTRATION ========= -app.register_blueprint(rawSkeleton) diff --git a/var/www/modules/rawSkeleton/templates/header_rawSkeleton.html b/var/www/modules/rawSkeleton/templates/header_rawSkeleton.html deleted file mode 100644 index b96b89a8..00000000 --- a/var/www/modules/rawSkeleton/templates/header_rawSkeleton.html +++ /dev/null @@ -1 +0,0 @@ -
  • rawSkeleton page
    • diff --git a/var/www/modules/rawSkeleton/templates/rawSkeleton.html b/var/www/modules/rawSkeleton/templates/rawSkeleton.html deleted file mode 100644 index 7a553bd3..00000000 --- a/var/www/modules/rawSkeleton/templates/rawSkeleton.html +++ /dev/null @@ -1,47 +0,0 @@ - - - - - - - - Analysis Information Leak framework Dashboard - - - - - - - - - - - - - - - - - {% include 'navbar.html' %} - -
    -
    -
    -

    rawSkeleton

    -
    - -
    - -
    - - - - - - - diff --git a/var/www/modules/search/templates/search.html b/var/www/modules/search/templates/search.html index 3ea961f0..e01d66ce 100644 --- a/var/www/modules/search/templates/search.html +++ b/var/www/modules/search/templates/search.html @@ -311,7 +311,7 @@ // On click, donwload all paste's content $("#load-more-button").off('click.download').on("click.download", function (event) { if (complete_paste == null) { //Donwload only once - $.get("{{ url_for('showsavedpastes.getmoredata') }}"+"?paste="+$(modal).attr('data-path'), function(data, status){ + $.get("{{ url_for('objects_item.item_content_more') }}"+"?id="+$(modal).attr('data-path'), function(data, status){ complete_paste = data; update_preview(); }); diff --git a/var/www/modules/showpaste/Flask_showpaste.py b/var/www/modules/showpaste/Flask_showpaste.py deleted file mode 100644 index 6589896e..00000000 --- a/var/www/modules/showpaste/Flask_showpaste.py +++ /dev/null @@ -1,283 +0,0 @@ -#!/usr/bin/env python3 -# -*-coding:UTF-8 -* - -''' - Flask functions and routes for the trending modules page -''' -import json -import os -import sys -import flask -from flask import Flask, render_template, jsonify, request, Blueprint, make_response, Response, send_from_directory, redirect, url_for, abort - -from Role_Manager import login_admin, login_analyst, login_read_only, no_cache -from flask_login import login_required - -import difflib - -import Paste -import requests - -sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/')) -import Item - -sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) - -# ============ VARIABLES ============ -import Flask_config - -app = Flask_config.app -baseUrl = Flask_config.baseUrl -r_serv_metadata = Flask_config.r_serv_metadata -r_serv_tags = Flask_config.r_serv_tags -r_serv_statistics = Flask_config.r_serv_statistics -max_preview_char = Flask_config.max_preview_char -max_preview_modal = Flask_config.max_preview_modal -DiffMaxLineLength = Flask_config.DiffMaxLineLength -bootstrap_label = Flask_config.bootstrap_label -misp_event_url = Flask_config.misp_event_url -hive_case_url = Flask_config.hive_case_url -vt_enabled = Flask_config.vt_enabled -PASTES_FOLDER = Flask_config.PASTES_FOLDER -SCREENSHOT_FOLDER = Flask_config.SCREENSHOT_FOLDER - -showsavedpastes = Blueprint('showsavedpastes', __name__, template_folder='templates') - -# ============ FUNCTIONS ============ - -def get_item_screenshot_path(item): - screenshot = r_serv_metadata.hget('paste_metadata:{}'.format(item), 'screenshot') - if screenshot: - screenshot = os.path.join(screenshot[0:2], screenshot[2:4], screenshot[4:6], screenshot[6:8], screenshot[8:10], screenshot[10:12], screenshot[12:]) - else: - screenshot = '' - return screenshot - -def get_item_basic_info(item): - item_basic_info = {} - item_basic_info['date'] = str(item.get_p_date()) - item_basic_info['date'] = '{}/{}/{}'.format(item_basic_info['date'][0:4], item_basic_info['date'][4:6], item_basic_info['date'][6:8]) - item_basic_info['source'] = item.get_item_source() - item_basic_info['size'] = item.get_item_size() - - ## TODO: FIXME ##performance - item_basic_info['encoding'] = item._get_p_encoding() - ## TODO: FIXME ##performance - #item_basic_info['language'] = item._get_p_language() - ## TODO: FIXME ##performance - info_line = item.get_lines_info() - item_basic_info['nb_lines'] = info_line[0] - item_basic_info['max_length_line'] = info_line[1] - - return item_basic_info - -def show_item_min(requested_path , content_range=0): - relative_path = None - if PASTES_FOLDER not in requested_path: - relative_path = requested_path - requested_path = os.path.join(PASTES_FOLDER, requested_path) - else: - relative_path = requested_path.replace(PASTES_FOLDER, '', 1) - # remove old full path - # requested_path = requested_path.replace(PASTES_FOLDER, '') - # escape directory transversal - if os.path.commonprefix((os.path.realpath(requested_path),PASTES_FOLDER)) != PASTES_FOLDER: - return 'path transversal detected' - - item_info ={} - - paste = Paste.Paste(requested_path) - item_basic_info = get_item_basic_info(paste) - item_info['nb_duplictates'] = paste.get_nb_duplicate() - - ## TODO: use this for fix ? - item_content = paste.get_p_content() - char_to_display = len(item_content) - if content_range != 0: - item_content = item_content[0:content_range] - - vt_enabled = Flask_config.vt_enabled - - - p_hashtype_list = [] - - print(requested_path) - l_tags = r_serv_metadata.smembers('tag:'+relative_path) - if relative_path is not None: - l_tags.union( r_serv_metadata.smembers('tag:'+relative_path) ) - item_info['tags'] = l_tags - item_info['name'] = relative_path.replace('/', ' / ') - - - l_64 = [] - # load hash files - if r_serv_metadata.scard('hash_paste:'+relative_path) > 0: - set_b64 = r_serv_metadata.smembers('hash_paste:'+relative_path) - for hash in set_b64: - nb_in_file = r_serv_metadata.zscore('nb_seen_hash:'+hash, relative_path) - # item list not updated - if nb_in_file is None: - l_pastes = r_serv_metadata.zrange('nb_seen_hash:'+hash, 0, -1) - for paste_name in l_pastes: - # dynamic update - if PASTES_FOLDER in paste_name: - score = r_serv_metadata.zscore('nb_seen_hash:{}'.format(hash), paste_name) - r_serv_metadata.zrem('nb_seen_hash:{}'.format(hash), paste_name) - paste_name = paste_name.replace(PASTES_FOLDER, '', 1) - r_serv_metadata.zadd('nb_seen_hash:{}'.format(hash), score, paste_name) - nb_in_file = r_serv_metadata.zscore('nb_seen_hash:{}'.format(hash), relative_path) - nb_in_file = int(nb_in_file) - estimated_type = r_serv_metadata.hget('metadata_hash:'+hash, 'estimated_type') - file_type = estimated_type.split('/')[0] - # set file icon - if file_type == 'application': - file_icon = 'fa-file ' - elif file_type == 'audio': - file_icon = 'fa-file-video ' - elif file_type == 'image': - file_icon = 'fa-file-image' - elif file_type == 'text': - file_icon = 'fa-file-alt' - else: - file_icon = 'fa-file' - saved_path = r_serv_metadata.hget('metadata_hash:'+hash, 'saved_path') - if r_serv_metadata.hexists('metadata_hash:'+hash, 'vt_link'): - b64_vt = True - b64_vt_link = r_serv_metadata.hget('metadata_hash:'+hash, 'vt_link') - b64_vt_report = r_serv_metadata.hget('metadata_hash:'+hash, 'vt_report') - else: - b64_vt = False - b64_vt_link = '' - b64_vt_report = r_serv_metadata.hget('metadata_hash:'+hash, 'vt_report') - # hash never refreshed - if b64_vt_report is None: - b64_vt_report = '' - - l_64.append( (file_icon, estimated_type, hash, saved_path, nb_in_file, b64_vt, b64_vt_link, b64_vt_report) ) - - crawler_metadata = {} - if 'infoleak:submission="crawler"' in l_tags: - crawler_metadata['get_metadata'] = True - crawler_metadata['domain'] = r_serv_metadata.hget('paste_metadata:'+relative_path, 'domain') - crawler_metadata['domain'] = crawler_metadata['domain'].rsplit(':', 1)[0] - crawler_metadata['paste_father'] = r_serv_metadata.hget('paste_metadata:'+relative_path, 'father') - crawler_metadata['real_link'] = r_serv_metadata.hget('paste_metadata:'+relative_path,'real_link') - crawler_metadata['screenshot'] = get_item_screenshot_path(relative_path) - # crawler_metadata['har_file'] = Item.get_item_har(relative_path) - else: - crawler_metadata['get_metadata'] = False - - item_parent = Item.get_item_parent(requested_path) - - misp_event = r_serv_metadata.get('misp_events:' + requested_path) - if misp_event is None: - misp_eventid = False - misp_url = '' - else: - misp_eventid = True - misp_url = misp_event_url + misp_event - - hive_case = r_serv_metadata.get('hive_cases:' + requested_path) - if hive_case is None: - hive_caseid = False - hive_url = '' - else: - hive_caseid = True - hive_url = hive_case_url.replace('id_here', hive_case) - - return render_template("show_saved_item_min.html", bootstrap_label=bootstrap_label, content=item_content, - item_basic_info=item_basic_info, item_info=item_info, - item_parent=item_parent, - initsize=len(item_content), - hashtype_list = p_hashtype_list, - crawler_metadata=crawler_metadata, - l_64=l_64, vt_enabled=vt_enabled, misp_eventid=misp_eventid, misp_url=misp_url, hive_caseid=hive_caseid, hive_url=hive_url) - -# ============ ROUTES ============ - -@showsavedpastes.route("/showsaveditem_min/") #completely shows the paste in a new tab -@login_required -@login_read_only -def showsaveditem_min(): - requested_path = request.args.get('paste', '') - return show_item_min(requested_path) - -@showsavedpastes.route("/showsavedrawpaste/") #shows raw -@login_required -@login_read_only -def showsavedrawpaste(): - requested_path = request.args.get('paste', '') - paste = Paste.Paste(requested_path) - content = paste.get_p_content() - return Response(content, mimetype='text/plain') - -@showsavedpastes.route("/showpreviewpaste/") -@login_required -@login_read_only -def showpreviewpaste(): - num = request.args.get('num', '') - requested_path = request.args.get('paste', '') - return "|num|"+num+"|num|"+show_item_min(requested_path, content_range=max_preview_modal) - - -@showsavedpastes.route("/getmoredata/") -@login_required -@login_read_only -def getmoredata(): - requested_path = request.args.get('paste', '') - paste = Paste.Paste(requested_path) - p_content = paste.get_p_content() - to_return = p_content[max_preview_modal-1:] - return to_return - -@showsavedpastes.route("/showDiff/") -@login_required -@login_analyst -def showDiff(): - s1 = request.args.get('s1', '') - s2 = request.args.get('s2', '') - p1 = Paste.Paste(s1) - p2 = Paste.Paste(s2) - maxLengthLine1 = p1.get_lines_info()[1] - maxLengthLine2 = p2.get_lines_info()[1] - if maxLengthLine1 > DiffMaxLineLength or maxLengthLine2 > DiffMaxLineLength: - return "Can't make the difference as the lines are too long." - htmlD = difflib.HtmlDiff() - lines1 = p1.get_p_content().splitlines() - lines2 = p2.get_p_content().splitlines() - the_html = htmlD.make_file(lines1, lines2) - return the_html - -@showsavedpastes.route('/screenshot/') -@login_required -@login_read_only -@no_cache -def screenshot(filename): - return send_from_directory(SCREENSHOT_FOLDER, filename+'.png', as_attachment=True) - -@showsavedpastes.route('/send_file_to_vt/', methods=['POST']) -@login_required -@login_analyst -def send_file_to_vt(): - b64_path = request.form['b64_path'] - paste = request.form['paste'] - hash = request.form['hash'] - - ## TODO: # FIXME: path transversal - b64_full_path = os.path.join(os.environ['AIL_HOME'], b64_path) - b64_content = '' - with open(b64_full_path, 'rb') as f: - b64_content = f.read() - - files = {'file': (hash, b64_content)} - response = requests.post('https://www.virustotal.com/vtapi/v2/file/scan', files=files, params=vt_auth) - json_response = response.json() - print(json_response) - - vt_b64_link = json_response['permalink'].split('analysis')[0] + 'analysis/' - r_serv_metadata.hset('metadata_hash:'+hash, 'vt_link', vt_b64_link) - - return redirect(url_for('showsavedpastes.showsavedpaste', paste=paste)) - -# ========= REGISTRATION ========= -app.register_blueprint(showsavedpastes, url_prefix=baseUrl) diff --git a/var/www/modules/showpaste/templates/show_saved_paste.html b/var/www/modules/showpaste/templates/show_saved_paste.html deleted file mode 100644 index dc29ad7f..00000000 --- a/var/www/modules/showpaste/templates/show_saved_paste.html +++ /dev/null @@ -1,581 +0,0 @@ - - - - Paste information - AIL - - - - - - - - - - - - - - - - - - - - - - - -
    -
    -

    Paste: {{ request.args.get('paste') }}

    -

    -
    - - - - - {% for tag in list_tags %} - {{ tag[0] }} - - - - - {% endfor %} - -
    -

    - - - - - - - - - - - - - - - - - - - - - - - - - - -
    DateSourceEncodingLanguageSize (Kb)MimeNumber of linesMax line length
    {{ date }}{{ source }}{{ encoding }}{{ language }}{{ size }}{{ mime }}{{ lineinfo.0 }}{{ lineinfo.1 }}
    - - {% if item_parent %} -
    - {{item_parent}} -
    - {% endif %} - -
    - {% with obj_type='item', obj_id=request.args.get('paste'), obj_lvl=0%} - {% include 'import_export/block_add_user_object_to_export.html' %} - {% endwith %} - - {% if hive %} - - - - {% endif %} -
    - -
    - - - -
    - -
    - -
    - - {% if duplicate_list|length == 0 %} - {% else %} -

    Duplicate list:

    - - {% set i = 0 %} - - - - - - - - - - - {% for dup_path in duplicate_list %} - - - - - - - - {% endfor %} - -
    Hash typePaste infoDatePathAction
    {{ hashtype_list[loop.index - 1] }}Similarity: {{ simil_list[loop.index - 1] }}%{{ date_list[loop.index - 1] }}{{ dup_path }}
    - {% endif %} - - {% if l_64|length != 0 %} -

    Hash files:

    - - - - - - - - - - - {% for b64 in l_64 %} - - - - - - - {% endfor %} - -
    estimated typehashsaved_pathVirus Total
      {{ b64[1] }}{{ b64[2] }} ({{ b64[4] }}){{ b64[3] }} - {% if vt_enabled %} - {% if not b64[5] %} - - - - {% else %} - VT Report - {% endif %} - - {% else %} - Virus Total submission is disabled - {% endif %} -
    - {% endif %} - - {% if crawler_metadata['get_metadata'] %} -
    - -
    -
    -
    -
    - Crawled Paste -
    - - - - - - - - - - - - - - - - -
    Domain{{ crawler_metadata['domain'] }}
    Father{{ crawler_metadata['paste_father'] }}
    Source link{{ crawler_metadata['real_link'] }}
    - - {% if crawler_metadata['har_file'] %} - button - {% endif %} -
    -
    -
    - -
    -
    -
    -
    -
    - -
    -
    - -
    -
    -
    -
    - -
    -
    - {% endif %} - -

    Content:

    - [Raw content] - 

    {{ content }}

    -
    -
    - - - - - - - - - - - - - - - -{% if crawler_metadata['get_metadata'] %} - -{% endif %} - - diff --git a/var/www/templates/correlation/metadata_card_cryptocurrency.html b/var/www/templates/correlation/metadata_card_cryptocurrency.html index b0c39df6..7d9318e8 100644 --- a/var/www/templates/correlation/metadata_card_cryptocurrency.html +++ b/var/www/templates/correlation/metadata_card_cryptocurrency.html @@ -22,7 +22,7 @@ - {{ dict_object["metadata_card"]["icon"]["icon_text"] }} + {{ dict_object["metadata_card"]["icon"]["icon"] }} {{ dict_object["metadata"]["type_id"] }} diff --git a/var/www/templates/correlation/metadata_card_cve.html b/var/www/templates/correlation/metadata_card_cve.html index 49e6a38f..39fca4e6 100644 --- a/var/www/templates/correlation/metadata_card_cve.html +++ b/var/www/templates/correlation/metadata_card_cve.html @@ -30,7 +30,7 @@ - {{ dict_object["metadata_card"]["icon"]["icon_text"] }} + {{ dict_object["metadata_card"]["icon"]["icon"] }} diff --git a/var/www/templates/correlation/metadata_card_decoded.html b/var/www/templates/correlation/metadata_card_decoded.html index 12871036..da57cb21 100644 --- a/var/www/templates/correlation/metadata_card_decoded.html +++ b/var/www/templates/correlation/metadata_card_decoded.html @@ -32,7 +32,7 @@ - {{ dict_object["metadata_card"]["icon"]["icon_text"] }} + {{ dict_object["metadata_card"]["icon"]["icon"] }} {{ dict_object["metadata"]["file_type"] }} @@ -56,7 +56,7 @@ Tags: {% for tag in dict_object["metadata"]['tags'] %} {% endfor %} diff --git a/var/www/templates/correlation/metadata_card_domain.html b/var/www/templates/correlation/metadata_card_domain.html index 2566fb14..0e538916 100644 --- a/var/www/templates/correlation/metadata_card_domain.html +++ b/var/www/templates/correlation/metadata_card_domain.html @@ -21,7 +21,7 @@ - {{ dict_object["metadata_card"]["icon"]["icon_text"] }} + {{ dict_object["metadata_card"]["icon"]["icon"] }} {{ dict_object["metadata"]["type_id"] }} diff --git a/var/www/templates/correlation/metadata_card_pgp.html b/var/www/templates/correlation/metadata_card_pgp.html index 819c4d0f..17746183 100644 --- a/var/www/templates/correlation/metadata_card_pgp.html +++ b/var/www/templates/correlation/metadata_card_pgp.html @@ -22,7 +22,7 @@ - {{ dict_object["metadata_card"]["icon"]["icon_text"] }} + {{ dict_object["metadata_card"]["icon"]["icon"] }} {{ dict_object["metadata"]["type_id"] }} @@ -39,6 +39,16 @@ + {% if dict_object["metadata"]['tags'] %} +
  • +
    + Tags: + {% for tag in dict_object["metadata"]['tags'] %} + {{ tag }} + {% endfor %} +
    +
    • + {% endif %} {% with obj_type='pgp', obj_id=dict_object['correlation_id'], obj_subtype=dict_object["metadata"]["type_id"] %} diff --git a/var/www/templates/correlation/metadata_card_screenshot.html b/var/www/templates/correlation/metadata_card_screenshot.html index 804620e7..61b09737 100644 --- a/var/www/templates/correlation/metadata_card_screenshot.html +++ b/var/www/templates/correlation/metadata_card_screenshot.html @@ -68,7 +68,7 @@ Tags: {% for tag in dict_object["metadata"]['tags'] %} {% endfor %} @@ -99,7 +99,7 @@ img.onload = pixelate; img.addEventListener("error", img_error); var draw_img = false; -img.src = "{{ url_for('showsavedpastes.screenshot', filename=dict_object['metadata']['img']) }}"; +img.src = "{{ url_for('objects_item.screenshot', filename=dict_object['metadata']['img']) }}"; function pixelate() { /// use slider value diff --git a/var/www/templates/correlation/metadata_card_username.html b/var/www/templates/correlation/metadata_card_username.html index 46053fa5..9467eee9 100644 --- a/var/www/templates/correlation/metadata_card_username.html +++ b/var/www/templates/correlation/metadata_card_username.html @@ -22,7 +22,7 @@ - {{ dict_object["metadata_card"]["icon"]["icon_text"] }} + {{ dict_object["metadata_card"]["icon"]["icon"] }} {{ dict_object["metadata"]["type_id"] }} diff --git a/var/www/templates/correlation/show_correlation.html b/var/www/templates/correlation/show_correlation.html index c270c78d..7c6c9362 100644 --- a/var/www/templates/correlation/show_correlation.html +++ b/var/www/templates/correlation/show_correlation.html @@ -511,7 +511,7 @@ if (d.popover) { desc = desc + "" if (data["img"]) { - desc = desc + ""; + desc = desc + ""; } if (data["tags"]) { diff --git a/var/www/templates/crawler/crawler_splash/domain_explorer.html b/var/www/templates/crawler/crawler_splash/domain_explorer.html index 629cd090..c7ecc5d9 100644 --- a/var/www/templates/crawler/crawler_splash/domain_explorer.html +++ b/var/www/templates/crawler/crawler_splash/domain_explorer.html @@ -1,24 +1,24 @@ - Show Domain - AIL + Show Domain - AIL - + - - - - + + + + - + @@ -217,7 +217,7 @@ blocks.addEventListener('change', pixelate_all, false); {% for dict_domain in dict_data['list_elem'] %} {% if 'screenshot' in dict_domain %} {% if dict_domain['is_tags_safe'] %} - var screenshot_url = "{{ url_for('showsavedpastes.screenshot', filename="") }}{{dict_domain['screenshot']}}"; + var screenshot_url = "{{ url_for('objects_item.screenshot', filename="") }}{{dict_domain['screenshot']}}"; {% else %} var screenshot_url = "{{ url_for('static', filename='image/AIL.png') }}"; {% endif %} diff --git a/var/www/templates/crawler/crawler_splash/showDomain.html b/var/www/templates/crawler/crawler_splash/showDomain.html index 5fc65caa..5dd538f9 100644 --- a/var/www/templates/crawler/crawler_splash/showDomain.html +++ b/var/www/templates/crawler/crawler_splash/showDomain.html @@ -85,7 +85,7 @@
    {% include 'modals/edit_tag.html' %} {% for tag in dict_domain['tags'] %} - {% endfor %} @@ -526,7 +526,7 @@ function toggle_sidebar(){ diff --git a/var/www/templates/modals/edit_tag.html b/var/www/templates/modals/edit_tag.html index c7455631..512c8277 100644 --- a/var/www/templates/modals/edit_tag.html +++ b/var/www/templates/modals/edit_tag.html @@ -15,17 +15,17 @@
    - + + Confirm this Tag + + Delete this Tag @@ -38,12 +38,19 @@ diff --git a/var/www/templates/modals/show_min_item.html b/var/www/templates/modals/show_min_item.html index 8613d9c6..bd353bb0 100644 --- a/var/www/templates/modals/show_min_item.html +++ b/var/www/templates/modals/show_min_item.html @@ -69,13 +69,10 @@ function get_html_and_update_modal(event, truemodal) { event.preventDefault(); var modal=truemodal; - var url = " {{ url_for('showsavedpastes.showpreviewpaste') }}?paste=" + modal.attr('data-path') + "&num=" + modal.attr('data-num'); - last_clicked_paste = modal.attr('data-num'); + var url = " {{ url_for('objects_item.item_preview') }}?id=" + modal.attr('data-path'); $.get(url, function (data) { // verify that the reveived data is really the current clicked item. Otherwise, ignore it. - var received_num = parseInt(data.split("|num|")[1]); - if (received_num == last_clicked_paste && can_change_modal_content) { can_change_modal_content = false; // clear data by removing html, body, head tags. prevent dark modal background stack bug. @@ -97,7 +94,7 @@ function get_html_and_update_modal(event, truemodal) { // On click, donwload all item's content $("#load-more-button").on("click", function (event) { if (complete_item == null) { //Donwload only once - $.get("{{ url_for('showsavedpastes.getmoredata') }}"+"?paste="+$(modal).attr('data-path'), function(data, status){ + $.get("{{ url_for('objects_item.item_content_more') }}"+"?id="+$(modal).attr('data-path'), function(data, status){ complete_item = data; update_preview(); }); @@ -105,9 +102,6 @@ function get_html_and_update_modal(event, truemodal) { update_preview(); } }); - } else if (can_change_modal_content) { - $("#modal_show_min_item_body").html("Ignoring previous not finished query of item #" + received_num); - } }); } diff --git a/var/www/templates/objects/item/show_item.html b/var/www/templates/objects/item/show_item.html index 6d3afd8b..fee7c80f 100644 --- a/var/www/templates/objects/item/show_item.html +++ b/var/www/templates/objects/item/show_item.html @@ -66,11 +66,11 @@
    -
    + {% include 'modals/edit_tag.html' %} {% for tag in meta['tags'] %} @@ -80,7 +80,7 @@ -
    +
    @@ -200,7 +200,7 @@ - + {% endfor %} @@ -362,7 +362,7 @@ Others
     Raw Content -  Download +  Download
    @@ -416,7 +416,11 @@ img.addEventListener("error", img_error); var draw_img = false; - img.src = "{{ url_for('showsavedpastes.screenshot', filename=meta['crawler']['screenshot']) }}"; + {% if meta['crawler']['screenshot'] %} + img.src = "{{ url_for('objects_item.screenshot', filename=meta['crawler']['screenshot']) }}"; + {% else %} + img.src = ""; + {% endif %} function pixelate() { diff --git a/var/www/modules/showpaste/templates/show_saved_item_min.html b/var/www/templates/objects/item/show_item_min.html similarity index 60% rename from var/www/modules/showpaste/templates/show_saved_item_min.html rename to var/www/templates/objects/item/show_item_min.html index 7030868c..359627df 100644 --- a/var/www/modules/showpaste/templates/show_saved_item_min.html +++ b/var/www/templates/objects/item/show_item_min.html @@ -1,40 +1,40 @@ - Paste information - AIL - - - + Paste information - AIL + + + - - - + + + - - - - + + + + - +
    -

    {{ item_info['name'] }}

    +

    {{ meta['id'] }}

    @@ -50,19 +50,19 @@ - - - - - - + + + + + +
    {{ item_basic_info['date'] }}{{ item_basic_info['source'] }}{{ item_basic_info['encoding'] }}{{ item_basic_info['size'] }}{{ item_basic_info['nb_lines'] }}{{ item_basic_info['max_length_line'] }}{{ meta['date'] }}{{ meta['source'] }}{{ meta['mimetype'] }}{{ meta['size'] }}{{ meta['lines']['nb'] }}{{ meta['lines']['max_length'] }}
    - {% for tag in item_info['tags'] %} + {% for tag in meta['tags'] %} {{ tag }} {% endfor %}
    @@ -71,9 +71,9 @@
    - {% if item_parent %} + {% if meta['parent'] %} {% endif %} @@ -91,13 +91,13 @@
    {% endif %} - {% if item_info['nb_duplictates'] != 0 %} + {% if meta['duplicates'] != 0 %}
    duplicates   -
    {{item_info['nb_duplictates']}}
    +
    {{meta['duplicates']}}
    @@ -105,52 +105,43 @@ {% endif %} - {% if l_64|length != 0 %} -
    -
    -
    -
    -
    -
    - Decoded Files   -
    {{l_64|length}}
    -
    + {% if meta['nb_correlations'] %} +
    +
    +
    +
    +
    + {% for obj_type in meta['nb_correlations'] %} + {% if meta['nb_correlations'][obj_type] %} + + {% if obj_type=='cve' %} + + {% elif obj_type=='cryptocurrency' %} + + {% elif obj_type=='decoded' %} + + {% elif obj_type=='pgp' %} + + {% elif obj_type=='username' %} + + {% elif obj_type=='screenshot' %} + + {% else %} + + {% endif %} + {{ obj_type }}  +
    {{meta['nb_correlations'][obj_type]}}
    +     + {% endif %} + {% endfor %} +
    -
    - -
    -
    -
    - -
    -
    - - - - - - - - - - {% for b64 in l_64 %} - - - - - {% endfor %} - -
    estimated typehash
      {{ b64[1] }} {{b64[2]}} ({{ b64[4] }})
    -
    -
    {% endif %} - {% if crawler_metadata['get_metadata'] %} + {% if 'crawler' in meta %}
    @@ -175,15 +166,15 @@ Domain - {{ crawler_metadata['domain'] }} + {{ meta['crawler']['domain'] }} Father - {{ crawler_metadata['paste_father'] }} + {{ meta['parent'] }} Url - {{ crawler_metadata['real_link'] }} + {{ meta['crawler']['url'] }} @@ -229,7 +220,7 @@
    - 

    {{ content }}

    + 

    {{ meta['content'] }}

    -{% if crawler_metadata['get_metadata'] %} +{% if 'crawler' in meta %} + + + + + + + + + {% include 'nav_bar.html' %} + +
    +
    + + {% include 'tags/menu_sidebar.html' %} + +
    + +

    + Galaxies: +

    + + + + + + + + + + + + + + {% for galaxy in galaxies %} + + + + + + + + + {% endfor %} + + +
    NameDescriptionNamespaceEnabledActive Tags
    {{ galaxy['name'] }}{{ galaxy['description'] }}{{ galaxy['namespace'] }} + {% if galaxy['enebled'] %} +
    + +
    + {% else %} +
    + +
    + {% endif %} +     		+
    + + {{ galaxy['nb_active_tags'] }}/{{ galaxy['nb_tags'] }} + +
    +     		+ + + +
    + +
    +
    +
    +
    + +
    +
    +
    + + + + + + + + + diff --git a/var/www/templates/tags/galaxy.html b/var/www/templates/tags/galaxy.html new file mode 100644 index 00000000..b4498973 --- /dev/null +++ b/var/www/templates/tags/galaxy.html @@ -0,0 +1,219 @@ + + + + + Galaxy - AIL + + + + + + + + + + + + + + + + + + {% include 'nav_bar.html' %} + +
    +
    + + {% include 'tags/menu_sidebar.html' %} + +
    + +
    + + List Galaxies + +
    + +
    +
    +
    + {{ galaxy['name'] }} + + + {% if galaxy['enabled'] %} + Enabled + {% else %} + Disabled + {% endif %} + +
    +
    + +
    + + + + + + + {% if galaxy['namespace'] %} + + + + + {% endif %} + {% if galaxy['category'] %} + + + + + {% endif %} + {% if galaxy['type'] %} + + + + + {% endif %} + {% if galaxy['source'] %} + + + + + {% endif %} + + + + + +
    Description{{ galaxy['description'] }}
    Namespace{{ galaxy['namespace'] }}
    Category{{ galaxy['category'] }}
    Type{{ galaxy['type'] }}
    Source{{ galaxy['source'] }}
    Version{{ galaxy['version'] }}
    + +
    + {% if galaxy['enabled'] %} + + Disable Galaxy + + {% else %} + + Enable Galaxy + + {% endif %} +
    +
    +
    + +
    + + + + + + + + + + + + + {% for tag in galaxy['tags'] %} + + + + + + + {% endfor %} + +
    TagDescriptionEnabled
    +
    + + +
    +     		+ {{ tag['tag'] }} + {% if tag['synonyms'] %} +
    + Synonym: + {% for synonym in tag['synonyms'] %} + {{ synonym }}, + {% endfor %} + +
    + {% endif %} +     		+ {% if 'description' in tag %} + {{ tag['description'] }} + {% endif %} + +
    + {% if tag['enabled'] %} +
    + +
    + {% else %} +
    + +
    + {% endif %} +
    +
    + +
    + +
    + +
    + +



    + +
    +
    +
    + + + + + + + + + diff --git a/var/www/templates/tags/galaxy_tag.html b/var/www/templates/tags/galaxy_tag.html new file mode 100644 index 00000000..cd5a448b --- /dev/null +++ b/var/www/templates/tags/galaxy_tag.html @@ -0,0 +1,146 @@ + + + + + Galaxy - AIL + + + + + + + + + + + + + + + + + + {% include 'nav_bar.html' %} + +
    +
    + + {% include 'tags/menu_sidebar.html' %} + +
    + +
    + + {{ galaxy }} Galaxy + +
    + +
    +
    +
    + + {{ tag['value'] }} + + + + {% if tag['enabled'] %} + Enabled + {% else %} + Disabled + {% endif %} + +
    +
    + +
    +

    + {{ tag['tag'] }} +

    + + + {% if 'description' in tag %} + + + + + {% endif %} + {% if 'refs' in tag %} + + + + + {% endif %} + +
    Description{{ tag['description'] }}
    Refs{{ tag['refs'] }}
    + + {% if 'meta' in tag %} +
    + Meta + 
    {{ tag['meta'] }}
    + {% endif %} + +{#
    #} +{# {% if galaxy['enabled'] %}#} +{# #} +{# Disable Galaxy#} +{# #} +{# {% else %}#} +{# #} +{# Enable Galaxy#} +{# #} +{# {% endif %}#} +{#
    #} +
    +
    + +



    + +
    +
    +
    + + + + + + + + + diff --git a/var/www/templates/tags/menu_sidebar.html b/var/www/templates/tags/menu_sidebar.html index ccdbb70c..6a8cec37 100644 --- a/var/www/templates/tags/menu_sidebar.html +++ b/var/www/templates/tags/menu_sidebar.html @@ -29,27 +29,27 @@
  • - + - Search Images by Tags + Search Screenshots by Tags
    • -
    +
    Tags Management
    diff --git a/var/www/templates/tags/search_obj_by_tags.html b/var/www/templates/tags/search_obj_by_tags.html index 93ac9579..4652fb2c 100644 --- a/var/www/templates/tags/search_obj_by_tags.html +++ b/var/www/templates/tags/search_obj_by_tags.html @@ -37,12 +37,12 @@ {% endif %} {% with object_type=dict_tagged['object_type'], current_tags=dict_tagged['current_tags'], object_name=dict_tagged['object_name'] %} - {%if 'date' in dict_tagged%} - {% set date_from= dict_tagged['date']['date_from'] %} - {% set date_to= dict_tagged['date']['date_to'] %} - {% endif %} - {% include 'tags/block_obj_tags_search.html' %} - {% endwith %} + {%if 'date' in dict_tagged%} + {% set date_from= dict_tagged['date']['date_from'] %} + {% set date_to= dict_tagged['date']['date_to'] %} + {% endif %} + {% include 'tags/block_obj_tags_search.html' %} + {% endwith %}
    {%if 'tagged_obj' in dict_tagged%} @@ -61,7 +61,7 @@ Last seen Nb seen Size - {%elif dict_tagged["object_type"]=="image"%} + {%elif dict_tagged["object_type"]=="screenshot"%} Image {%elif dict_tagged["object_type"]=="item"%} Date @@ -106,7 +106,7 @@ {% endfor %} - {%elif dict_tagged["object_type"]=="image"%} + {%elif dict_tagged["object_type"]=="screenshot"%} {% for dict_obj in dict_tagged["tagged_obj"] %} @@ -141,7 +141,7 @@
    - diff --git a/var/www/templates/tags/taxonomies.html b/var/www/templates/tags/taxonomies.html new file mode 100644 index 00000000..eff480b5 --- /dev/null +++ b/var/www/templates/tags/taxonomies.html @@ -0,0 +1,123 @@ + + + + + Taxonomies - AIL + + + + + + + + + + + + + + + + + + {% include 'nav_bar.html' %} + +
    +
    + + {% include 'tags/menu_sidebar.html' %} + +
    + +

    + Taxonomies: +

    + + + + + + + + + + + + + + {% for taxonomy in taxonomies %} + + + + + + + + + {% endfor %} + + +
    NamespaceDescriptionVersionEnabledActive Tags
    {{ taxonomies[taxonomy]['name'] }}{{ taxonomies[taxonomy]['description'] }}{{ taxonomies[taxonomy]['version'] }} + {% if taxonomies[taxonomy]['enabled'] %} +
    + +
    + {% else %} +
    + +
    + {% endif %} +     		+
    + + {{ taxonomies[taxonomy]['nb_active_tags'] }}/{{ taxonomies[taxonomy]['nb_tags'] }} + +
    +     		+ + + +
    + +
    +
    +
    +
    + +
    +
    +
    + + + + + + + + + diff --git a/var/www/templates/tags/taxonomy.html b/var/www/templates/tags/taxonomy.html new file mode 100644 index 00000000..a5c61471 --- /dev/null +++ b/var/www/templates/tags/taxonomy.html @@ -0,0 +1,196 @@ + + + + + Taxonomy - AIL + + + + + + + + + + + + + + + + + + {% include 'nav_bar.html' %} + +
    +
    + + {% include 'tags/menu_sidebar.html' %} + +
    + +
    + + List Taxonomies + +
    + +
    +
    +
    + {{ taxonomy['name'] }} + + + {% if taxonomy['enabled'] %} + Enabled + {% else %} + Disabled + {% endif %} + +
    +
    + +
    + {% if taxonomy['expended'] %} +
    {{ taxonomy['expended'] }}
    + {% endif %} + {{ taxonomy['description'] }} +
    Version: {{ taxonomy['version'] }}
    + {% if taxonomy['predicates'] %} +
    + Predicates: + + + {% for predicate in taxonomy['predicates'] %} + + + + + + {% endfor %} + +
    {{ predicate }}{{ taxonomy['predicates'][predicate]['expanded'] }}{{ taxonomy['predicates'][predicate]['description'] }}
    + {% endif %} + {% if taxonomy['refs'] %} +
    Refs: {{ taxonomy['refs'] }}
    +

    + {% endif %} + +
    + {% if taxonomy['enabled'] %} + + Disable Taxonomy + + {% else %} + + Enable Taxonomy + + {% endif %} +
    +
    +
    + +
    + + + + + + + + + + + + + {% for tag in taxonomy['tags'] %} + + + + + + + {% endfor %} + +
    TagDescriptionEnabled
    +
    + + +
    +     		{{ tag['tag'] }} + {% if 'expanded' in tag %} + {{ tag['expanded'] }} + {% endif %} + +
    + {% if tag['enabled'] %} +
    + +
    + {% else %} +
    + +
    + {% endif %} +
    +
    + +
    + +
    + +
    + +



    + +
    +
    +
    + + + + + + + + +