chg: [custom tags] create + select custom tags

bin/lib/Tracker.py

@@ -1278,7 +1278,7 @@ def api_delete_retro_hunt_task(task_uuid):
         return (delete_retro_hunt_task(task_uuid), 200)
 
 #### DB FIX ####
-def get_trackers_custom_tags():
+def get_trackers_tags():
     tags = set()
     for tracker_uuid in get_all_tracker_uuid():
         for tag in get_tracker_tags(tracker_uuid):
@@ -1288,10 +1288,16 @@ def get_trackers_custom_tags():
             tags.add(tag)
     return tags
 
+def _fix_db_custom_tags():
+    for tag in get_trackers_tags():
+        if not Tag.is_taxonomie_tag(tag) and not Tag.is_galaxy_tag(tag):
+            print(tag)
+            Tag.create_custom_tag(tag)
+
 #### -- ####
 
 if __name__ == '__main__':
-    print(get_trackers_custom_tags())
+    _fix_db_custom_tags()
     # fix_all_tracker_uuid_list()
     # res = get_all_tracker_uuid()
     # print(len(res))

bin/packages/Tag.py

@@ -63,6 +63,14 @@ def get_galaxy_from_tag(tag):
     except IndexError:
         return None
 
+def get_taxonomies():
+    return Taxonomies().keys()
+
+def is_taxonomie(taxonomie, taxonomies=[]):
+    if not taxonomies:
+        taxonomies = get_taxonomies()
+    return taxonomie in taxonomies
+
 def get_active_taxonomies(r_set=False):
     res = r_serv_tags.smembers('active_taxonomies')
     if r_set:
@@ -81,6 +89,9 @@ def get_all_taxonomies_tags(): # # TODO: add + REMOVE + Update
 def get_all_galaxies_tags(): # # TODO: add + REMOVE + Update
     return r_serv_tags.smembers('active_galaxies_tags')
 
+def get_all_custom_tags():
+    return r_serv_tags.smembers('tags:custom')
+
 def get_taxonomies_enabled_tags(r_list=False):
     l_tag_keys = []
     for taxonomie in get_active_taxonomies():
@@ -89,6 +100,9 @@ def get_taxonomies_enabled_tags(r_list=False):
         res = r_serv_tags.sunion(l_tag_keys[0], *l_tag_keys[1:])
     elif l_tag_keys:
         res = r_serv_tags.smembers(l_tag_keys[0])
+    #### # WARNING: # TODO: DIRTY FIX, REPLACE WITH LOCAL TAGS ####
+
+
     if r_list:
         return list(res)
     else:
@@ -105,6 +119,19 @@ def get_galaxies_enabled_tags():
     else:
         return []
 
+def get_custom_enabled_tags(r_list=False):
+    res = r_serv_tags.smembers('tags:custom:enabled_tags')
+    if r_list:
+        return list(res)
+    else:
+        return res
+
+def get_taxonomies_customs_tags(r_list=False):
+    tags = get_custom_enabled_tags().union(get_taxonomies_enabled_tags())
+    if r_list:
+        tags = list(tags)
+    return tags
+
 def get_taxonomie_enabled_tags(taxonomie, r_list=False):
     res = r_serv_tags.smembers(f'active_tag_{taxonomie}')
     if r_list:
@@ -131,6 +158,9 @@ def is_galaxy_tag_enabled(galaxy, tag):
     else:
         return False
 
+def is_custom_tag_enabled(tag):
+    return r_serv_tags.sismember('tags:custom:enabled_tags', tag)
+
 def enable_taxonomy(taxonomie, enable_tags=True):
     '''
     Enable a taxonomy. (UI)
@@ -184,7 +214,7 @@ def is_taxonomie_tag(tag, namespace=None):
     if not namespace:
         namespace = tag.split(':')[0]
     if namespace != 'misp-galaxy':
-        return True
+        return is_taxonomie(namespace)
     else:
         return False
 
@@ -196,6 +226,9 @@ def is_galaxy_tag(tag, namespace=None):
     else:
         return False
 
+def is_custom_tag(tag):
+    return r_serv_tags.sismember('tags:custom', tag)
+
 # # TODO:
 # def is_valid_tag(tag):
 #     pass
@@ -317,6 +350,10 @@ def get_modal_add_tags(item_id, object_type='item'):
             "object_id": item_id, "object_type": object_type}
 
 ######## NEW VERSION ########
+def create_custom_tag(tag):
+    r_serv_tags.sadd('tags:custom', tag)
+    r_serv_tags.sadd('tags:custom:enabled_tags', tag)
+
 def get_tag_first_seen(tag, r_int=False):
     '''
     Get tag first seen (current: item only)
@@ -341,6 +378,7 @@ def get_tag_last_seen(tag, r_int=False):
             return int(res)
     return res
 
+# # TODO: ADD color
 def get_tag_metadata(tag, r_int=False):
     '''
     Get tag metadata (current: item only)
@@ -427,6 +465,8 @@ def update_tag_last_seen(tag, tag_first_seen, tag_last_seen):
             #update_tag_last_seen(tag, tag_first_seen, tag_last_seen)
             pass
 
+## Objects tags ##
+
 def update_tag_metadata(tag, tag_date, object_type=None, add_tag=True):
     '''
     Update tag metadata (current: item only)
@@ -742,6 +782,3 @@ def get_list_of_solo_tags_to_export_by_type(export_type): # by type
     else:
         return None
     #r_serv_db.smembers('whitelist_hive')
-
-
-#### -- ####

configs/core.cfg.sample

@@ -211,7 +211,7 @@ host = localhost
 port = 6382
 db = 10
 
-[Kvrocks_Meta]
+[Kvrocks_DB]
 host = localhost
 port = 6383
 db = 0

var/www/blueprints/tags_ui.py

@@ -84,6 +84,12 @@ def delete_tag():
 def get_all_tags():
     return jsonify(Tag.get_all_tags())
 
+@tags_ui.route('/tag/get_taxonomies_customs_tags')
+@login_required
+@login_read_only
+def get_all_taxonomies_customs_tags():
+    return jsonify(Tag.get_taxonomies_customs_tags(r_list=True))
+
 @tags_ui.route('/tag/get_all_obj_tags')
 @login_required
 @login_read_only
@@ -107,6 +113,12 @@ def tag_galaxies_tags_enabled_json():
     tags = Tag.get_galaxies_enabled_tags()
     return jsonify(Tag.get_tags_selector_dict(tags))
 
+@tags_ui.route('/tag/custum/tags/enabled/json')
+@login_required
+@login_read_only
+def tag_custum_tags_enabled_json():
+    return jsonify(Tag.get_custom_enabled_tags(r_list=True))
+
 @tags_ui.route('/tag/taxonomie/tags/enabled/json')
 @login_required
 @login_read_only

var/www/modules/hunter/Flask_hunter.py

@@ -23,6 +23,9 @@ import Term
 import Tracker
 import item_basic
 
+sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))
+import Tag
+
 # ============ VARIABLES ============
 import Flask_config
 
@@ -98,10 +101,26 @@ def add_tracked_menu():
         description = request.form.get("description", '')
         webhook = request.form.get("webhook", '')
         level = request.form.get("level", 0)
-        tags = request.form.get("tags", [])
         mails = request.form.get("mails", [])
         sources = request.form.get("sources", [])
 
+        tags = request.form.get("tags", [])
+        taxonomies_tags = request.form.get('taxonomies_tags')
+        if taxonomies_tags:
+            try:
+                taxonomies_tags = json.loads(taxonomies_tags)
+            except Exception:
+                taxonomies_tags = []
+        else:
+            taxonomies_tags = []
+        galaxies_tags = request.form.get('galaxies_tags')
+        if galaxies_tags:
+            try:
+                galaxies_tags = json.loads(galaxies_tags)
+            except Exception:
+                galaxies_tags = []
+        
+
         # YARA #
         if tracker_type == 'yara':
             yara_default_rule = request.form.get("yara_default_rule")
@@ -141,6 +160,7 @@ def add_tracked_menu():
     else:
         return render_template("edit_tracker.html",
                                 all_sources=item_basic.get_all_items_sources(r_list=True),
+                                tags_selector_data=Tag.get_tags_selector_data(),
                                 all_yara_files=Tracker.get_all_default_yara_files())
 
 @hunter.route("/tracker/edit", methods=['GET', 'POST'])

var/www/modules/hunter/templates/edit_tracker.html

@@ -42,12 +42,6 @@
 
 									<div class="row">
 										<div class="col-12 col-xl-9">
-										  <div class="input-group mb-2 mr-sm-2">
-										    <div class="input-group-prepend">
-										      <div class="input-group-text bg-danger text-white"><i class="fas fa-tag"></i></div>
-										    </div>
-												<input id="tags" name="tags" class="form-control" placeholder="Tags (optional, space separated)" type="text" {%if dict_tracker%}{%if dict_tracker['tags']%}value="{{dict_tracker['tags']}}"{%endif%}{%endif%}>
-										  </div>
 											<div class="input-group mb-2 mr-sm-2">
 										    <div class="input-group-prepend">
 										      <div class="input-group-text bg-secondary text-white"><i class="fas fa-at"></i></div>
@@ -74,6 +68,22 @@
 												<input id="sources" class="form-control" type="text" name="sources" placeholder="Sources to track (ALL IF EMPTY)" autocomplete="off">
 											</div>
 
+
+											<div class="card my-4">
+											  <div class="card-header bg-secondary text-white">
+													<b>Tags</b>
+												</div>
+											  <div class="card-body">
+													<div class="input-group mb-2 mr-sm-2">
+														<div class="input-group-prepend">
+													  	<div class="input-group-text bg-danger text-white"><i class="fas fa-tag"></i></div>
+													  </div>
+														<input id="tags" name="tags" class="form-control" placeholder="Custom Tags (optional, space separated)" type="text" {%if dict_tracker%}{%if dict_tracker['tags']%}value="{{dict_tracker['tags']}}"{%endif%}{%endif%}>
+													</div>
+													{% include 'tags/block_tags_selector.html' %}
+												</div>
+											</div>
+
 										</div>
 										<div class="col-12 col-xl-3">
 											<div class="custom-control custom-switch mt-1">

var/www/templates/tags/block_tags_selector.html

@@ -9,6 +9,7 @@
   <div class="dropdown-menu dropdown-menu-right" aria-labelledby="dropdown-taxonomie"> <!-- TODO: make dropdown-scrollable -->
     <h6 class="dropdown-header">Taxonomie Tags</h6>
     <button class="dropdown-item" type="button" id="all-tags-taxonomies">All Tags <i class="fas fa-tags"></i></button>
+    <button class="dropdown-item" type="button" id="all-tags-customs">Custom Tags <i class="fas fa-tag"></i></button>
     <div class="dropdown-divider"></div>
     {% for taxo in tags_selector_data['active_taxonomies'] %}
       <button class="dropdown-item" type="button" id="{{ taxo }}-id{{ loop.index0 }}">{{ taxo }}</button>
@@ -48,7 +49,7 @@
 var ltags;
 var ltagsgalaxies;
 
-$.getJSON("{{ url_for('Tags.get_all_tags') }}",
+$.getJSON("{{ url_for('tags_ui.get_all_taxonomies_customs_tags') }}",
     function(data) {
       {% if 'taxonomies_tags' in tags_selector_data %}
         var valueData = [
@@ -91,7 +92,15 @@ $.getJSON("{{ url_for('tags_ui.tag_galaxies_tags_enabled_json') }}",
 
 jQuery("#all-tags-taxonomies").click(function(e){
         //change input tags list
-      $.getJSON("{{ url_for('tags_ui.tag_taxonomies_tags_enabled_json') }}",
+      $.getJSON("{{ url_for('tags_ui.get_all_taxonomies_customs_tags') }}",
+        function(data) {
+          ltags.setData(data)
+      });
+    });
+
+jQuery("#all-tags-customs").click(function(e){
+        //change input tags list
+      $.getJSON("{{ url_for('tags_ui.tag_custum_tags_enabled_json') }}",
         function(data) {
           ltags.setData(data)
       });