From 1457575dda5a846a702199bc02f6d516f8cd0627 Mon Sep 17 00:00:00 2001 From: David Cruciani Date: Thu, 16 May 2024 14:32:49 +0200 Subject: [PATCH] new: [functionality] flowintel + multiple entry --- website/app/__init__.py | 1 + website/app/db_class/db.py | 4 +- website/app/history/history_core.py | 8 +- website/app/home.py | 51 ++- website/app/home_core.py | 2 +- website/app/session_class.py | 51 +-- .../static/js/history/history_tree_query.js | 2 +- website/app/static/js/history/history_view.js | 4 +- website/app/static/js/mispParser.js | 30 +- website/app/templates/history.html | 2 +- website/app/templates/history_session.html | 4 +- website/app/templates/home.html | 62 +++- website/app/templates/query.html | 310 ++++++++++-------- website/app/utils/utils.py | 1 + website/conf/config.py | 1 + 15 files changed, 316 insertions(+), 217 deletions(-) diff --git a/website/app/__init__.py b/website/app/__init__.py index 6cf0a3cf..29967261 100644 --- a/website/app/__init__.py +++ b/website/app/__init__.py @@ -37,6 +37,7 @@ def create_app(): app.register_blueprint(home_blueprint, url_prefix="/") app.register_blueprint(history_blueprint, url_prefix="/") app.register_blueprint(account_blueprint, url_prefix="/") + csrf.exempt(home_blueprint) return app diff --git a/website/app/db_class/db.py b/website/app/db_class/db.py index 924b0fc6..89ec6ee2 100644 --- a/website/app/db_class/db.py +++ b/website/app/db_class/db.py @@ -38,7 +38,7 @@ class Session_db(db.Model): "id": self.id, "uuid": self.uuid, "modules": json.loads(self.modules_list), - "query_enter": self.query_enter, + "query_enter": json.loads(self.query_enter), "input_query": self.input_query, "config_module": json.loads(self.config_module), "result": json.loads(self.result), @@ -51,7 +51,7 @@ class Session_db(db.Model): json_dict = { "uuid": self.uuid, "modules": json.loads(self.modules_list), - "query": self.query_enter, + "query": json.loads(self.query_enter), "input": self.input_query, "query_date": self.query_date.strftime('%Y-%m-%d %H:%M') } diff --git a/website/app/history/history_core.py b/website/app/history/history_core.py index baa2b1ef..200a96b1 100644 --- a/website/app/history/history_core.py +++ b/website/app/history/history_core.py @@ -146,8 +146,8 @@ def util_remove_node_session(node_uuid, parent, parent_path): child = parent["children"][i] if child["uuid"] == node_uuid: del parent_path["children"][i] - return - elif child["children"]: + return True + elif "children" in child and child["children"]: return util_remove_node_session(node_uuid, child, parent_path["children"][i]) def remove_node_session(node_uuid): @@ -160,7 +160,9 @@ def remove_node_session(node_uuid): loc = i break elif q_value["children"]: - return util_remove_node_session(node_uuid, q_value, sess[keys_list[i]]) + if util_remove_node_session(node_uuid, q_value, sess[keys_list[i]]): + loc = i + break if loc: del sess[keys_list[i]] diff --git a/website/app/home.py b/website/app/home.py index 6c4e669d..7a675cb1 100644 --- a/website/app/home.py +++ b/website/app/home.py @@ -1,9 +1,10 @@ +import ast import json -from flask import Blueprint, render_template, request, jsonify, session as sess +from flask import Blueprint, redirect, render_template, request, jsonify, session as sess from flask_login import current_user from . import session_class as SessionModel from . import home_core as HomeModel -from .utils.utils import admin_user_active +from .utils.utils import admin_user_active, FLOWINTEL_URL home_blueprint = Blueprint( 'home', @@ -13,18 +14,35 @@ home_blueprint = Blueprint( ) -@home_blueprint.route("/") +@home_blueprint.route("/", methods=["GET", "POST"]) def home(): + try: + del sess["query"] + except: + pass sess["admin_user"] = bool(admin_user_active()) if "query" in request.args: - return render_template("home.html", query=request.args.get("query")) + sess["query"] = ast.literal_eval(request.args.get("query")) + if "query" in request.form: + sess["query"] = json.loads(request.form.get("query")) return render_template("home.html") +@home_blueprint.route("/get_query", methods=['GET', 'POST']) +def get_query(): + """Get result from flowintel""" + if "query" in sess: + return {"query": sess.get("query")} + return {"message": "No query"} + @home_blueprint.route("/home/", methods=["GET", "POST"]) def home_query(sid): + try: + del sess["query"] + except: + pass sess["admin_user"] = admin_user_active() if "query" in request.args: - query = request.args.get("query") + sess["query"] = [request.args.get("query")] return render_template("home.html", query=query, sid=sid) return render_template("404.html") @@ -33,21 +51,28 @@ def query(sid): sess["admin_user"] = admin_user_active() session = HomeModel.get_session(sid) flag=False + modules_list = [] if session: flag = True - query_loc = session.query_enter + query_loc = json.loads(session.query_enter) + modules_list = json.loads(session.modules_list) else: for s in SessionModel.sessions: if s.uuid == sid: flag = True query_loc = s.query session=s + modules_list = session.modules_list + query_str = ", ".join(query_loc) + if len(query_str) > 40: + query_str = query_str[0:40] + "..." if flag: return render_template("query.html", query=query_loc, + query_str=query_str, sid=sid, input_query=session.input_query, - modules=json.loads(session.modules_list), + modules=modules_list, query_date=session.query_date.strftime('%Y-%m-%d %H:%M')) return render_template("404.html") @@ -60,18 +85,20 @@ def get_query_info(sid): flag=False if session: flag = True - query_loc = session.query_enter + query_loc = json.loads(session.query_enter) + modules_list = json.loads(session.modules_list) else: for s in SessionModel.sessions: if s.uuid == sid: flag = True query_loc = s.query + modules_list = s.modules_list session=s if flag: loc_dict = { "query": query_loc, "input_query": session.input_query, - "modules": json.loads(session.modules_list), + "modules": modules_list, "query_date": session.query_date.strftime('%Y-%m-%d %H:%M') } return loc_dict @@ -227,3 +254,9 @@ def change_status(): return {'message': 'Something went wrong', 'toast_class': "danger-subtle"}, 400 return {'message': 'Need to pass "module_id"', 'toast_class': "warning-subtle"}, 400 return {'message': 'Permission denied', 'toast_class': "danger-subtle"}, 403 + + +@home_blueprint.route("/flowintel_url") +def flowintel_url(): + """send result to flowintel-cm""" + return {"url": f"{FLOWINTEL_URL}/analyzer/recieve_result"}, 200 diff --git a/website/app/home_core.py b/website/app/home_core.py index 1221fe54..2b13088b 100644 --- a/website/app/home_core.py +++ b/website/app/home_core.py @@ -163,7 +163,7 @@ def create_new_session_tree(current_session, parent_id): loc_json = { "uuid": loc_session.uuid, "modules": json.loads(loc_session.modules_list), - "query": loc_session.query_enter, + "query": json.loads(loc_session.query_enter), "input": loc_session.input_query, "query_date": loc_session.query_date.strftime('%Y-%m-%d %H:%M'), "config": json.loads(loc_session.config_module), diff --git a/website/app/session_class.py b/website/app/session_class.py index 2e32cab2..10b07f25 100644 --- a/website/app/session_class.py +++ b/website/app/session_class.py @@ -64,9 +64,12 @@ class Session_class: def start(self): """Start all worker""" - for i in range(len(self.modules_list)): - #need the index and the url in each queue item. - self.jobs.put((i, self.modules_list[i])) + cp = 0 + for i in self.query: + for j in self.modules_list: + self.jobs.put((cp, i, j)) + cp += 1 + #need the index and the url in each queue item. for _ in range(self.thread_count): worker = Thread(target=self.process) worker.daemon = True @@ -111,44 +114,44 @@ class Session_class: modules = query_get_module() loc_query = {} + self.result[work[1]] = dict() # If Misp format for module in modules: - if module["name"] == work[1]: + if module["name"] == work[2]: if "format" in module["mispattributes"]: loc_query = { "type": self.input_query, - "value": self.query, + "value": work[1], "uuid": str(uuid.uuid4()) } break loc_config = {} - if work[1] in self.config_module: - loc_config = self.config_module[work[1]] + if work[2] in self.config_module: + loc_config = self.config_module[work[2]] if loc_query: - send_to = {"module": work[1], "attribute": loc_query, "config": loc_config} + send_to = {"module": work[2], "attribute": loc_query, "config": loc_config} else: - send_to = {"module": work[1], self.input_query: self.query, "config": loc_config} + send_to = {"module": work[2], self.input_query: work[1], "config": loc_config} res = query_post_query(send_to) ## Sort attr in object by ui-priority - if "results" in res: - if "Object" in res["results"]: - for obj in res["results"]["Object"]: - loc_obj = get_object(obj["name"]) - if loc_obj: - for attr in obj["Attribute"]: - attr["ui-priority"] = loc_obj["attributes"][attr["object_relation"]]["ui-priority"] - - # After adding 'ui-priority' - obj["Attribute"].sort(key=lambda x: x["ui-priority"], reverse=True) + if res: + if "results" in res: + if "Object" in res["results"]: + for obj in res["results"]["Object"]: + loc_obj = get_object(obj["name"]) + if loc_obj: + for attr in obj["Attribute"]: + attr["ui-priority"] = loc_obj["attributes"][attr["object_relation"]]["ui-priority"] + + # After adding 'ui-priority' + obj["Attribute"].sort(key=lambda x: x["ui-priority"], reverse=True) - - # print(res) - if "error" in res: + if res and "error" in res: self.nb_errors += 1 - self.result[work[1]] = res + self.result[work[1]][work[2]] = res self.jobs.task_done() return True @@ -161,7 +164,7 @@ class Session_class: s = Session_db( uuid=str(self.uuid), modules_list=json.dumps(self.modules_list), - query_enter=self.query, + query_enter=json.dumps(self.query), input_query=self.input_query, config_module=json.dumps(self.config_module), result=json.dumps(self.result), diff --git a/website/app/static/js/history/history_tree_query.js b/website/app/static/js/history/history_tree_query.js index 022a3b56..8140c0c5 100644 --- a/website/app/static/js/history/history_tree_query.js +++ b/website/app/static/js/history/history_tree_query.js @@ -6,7 +6,7 @@ export default { }, template: ` -
  • [[history.query]]
  • +
  • [[history.query.join(", ")]]