Merge 7b050c956d into 7c59af5eb5

.github/workflows/codeql.yml

@@ -0,0 +1,42 @@
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+  schedule:
+    - cron: "54 22 * * 2"
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ python, javascript ]
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v2
+        with:
+          languages: ${{ matrix.language }}
+          queries: +security-and-quality
+
+      - name: Autobuild
+        uses: github/codeql-action/autobuild@v2
+        if: ${{ matrix.language == 'python' || matrix.language == 'javascript' }}
+
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v2
+        with:
+          category: "/language:${{ matrix.language }}"

.gitignore

@@ -21,3 +21,4 @@ venv*
 #vscode
 .vscode*
 *.sqlite
+website/conf/config.cfg

website/README.md

@@ -35,14 +35,20 @@ Edit `config.py`
 
 - `ADMIN_PASSWORD`: Password for Admin user if `ADMIN_USER` is True
 
+
+
+Rename `config.cfg.sample` to `config.cfg` then edit it:
+
+- `ADMIN_USER`: If True, config page will not be accessible
+
+- `ADMIN_PASSWORD`: Password for Admin user if `ADMIN_USER` is True
+
 ## Launch
 
 ```bash
 ./launch.sh -l
 ```
 
-
-
 ## Admin user
 
 If admin user is active, type `/login` in url to access a login page and type the password wrote in `config.py` in `ADMIN_PASSOWRD`.

website/app.py

@@ -7,6 +7,7 @@ from app.utils.init_modules import create_modules_db
 import signal
 import sys
 import subprocess
+from app.utils.utils import gen_admin_password
 
 def signal_handler(sig, frame):
     path = os.path.join(os.getcwd(), "launch.sh")
@@ -47,4 +48,5 @@ elif args.create_module:
     with app.app_context():
         create_modules_db()
 else:
+    gen_admin_password()
     app.run(host=app.config.get("FLASK_URL"), port=app.config.get("FLASK_PORT"))

website/app/__init__.py

@@ -5,7 +5,7 @@ from flask_migrate import Migrate
 from flask_session import Session
 from flask_login import LoginManager
 
-from config import config as Config
+from conf.config import config as Config
 import os
 
 

website/app/history/history_core.py

@@ -31,15 +31,14 @@ def get_history_session():
         # It's already save in history
         # Only parent-child tree structure is in flask session
         current_query_value = sess.get(current_query)
-        if current_query_value and current_query_value["children"]:
+        if current_query_value:
             loc_list.append(current_query_value)
     for q in sess:
         if isUUID(q):
             # If query have no children then don't display it
             q_value = sess.get(q)
-            if q_value["children"]:
-                if not q == current_query:
-                    loc_list.append(q_value)
+            if not q == current_query:
+                loc_list.append(q_value)
 
     return loc_list
 
@@ -47,7 +46,7 @@ def get_current_query_history():
     current_query = sess.get("current_query")
     if current_query:
         current_query_value = sess.get(current_query)
-        if current_query_value and current_query_value["children"]:
+        if current_query_value:
             return current_query_value
     return {}
 
@@ -82,25 +81,22 @@ def save_history_core(sid):
         # Doesn't already exist
         history_tree_db = History_Tree.query.filter_by(session_uuid=session["uuid"]).first()
         if not history_tree_db:
-            if "children" in session and session["children"]:
-                # Get all children before add to db
-                loc_dict = util_save_history(session)
-                h = History_Tree(
-                    session_uuid = session["uuid"],
-                    tree=json.dumps(loc_dict)
-                )
-                db.session.add(h)
-                db.session.commit()
-                return {"message": "History Save", 'toast_class': "success-subtle"}
-            return {"message": "No children", 'toast_class': "warning-subtle"}
+            # Get all children before add to db
+            loc_dict = util_save_history(session)
+            h = History_Tree(
+                session_uuid = session["uuid"],
+                tree=json.dumps(loc_dict)
+            )
+            db.session.add(h)
+            db.session.commit()
+            return {"message": "History Save", 'toast_class': "success-subtle"}
         # Save same session but with new value
         elif not json.loads(history_tree_db.tree) == session:
-            if "children" in session and session["children"]:
-                # Get all children before add to db
-                loc_dict = util_save_history(session)
-                history_tree_db.tree = json.dumps(loc_dict)
-                db.session.commit()
-                return {"message": "History updated", 'toast_class': "success-subtle"}
+            # Get all children before add to db
+            loc_dict = util_save_history(session)
+            history_tree_db.tree = json.dumps(loc_dict)
+            db.session.commit()
+            return {"message": "History updated", 'toast_class': "success-subtle"}
         return {"message": "History already saved", 'toast_class': "warning-subtle"}
     return {"message": "Session not found", 'toast_class': "danger-subtle"}
 
@@ -163,9 +159,8 @@ def remove_node_session(node_uuid):
             if q_value["uuid"] == node_uuid:
                 loc = i
                 break
-            else:
-                if q_value["children"]:
-                    return util_remove_node_session(node_uuid, q_value, sess[keys_list[i]])
+            elif q_value["children"]:
+                return util_remove_node_session(node_uuid, q_value, sess[keys_list[i]])
     if loc:
         del sess[keys_list[i]]
 

website/app/home.py

@@ -15,7 +15,7 @@ home_blueprint = Blueprint(
 
 @home_blueprint.route("/")
 def home():
-    sess["admin_user"] = admin_user_active()
+    sess["admin_user"] = bool(admin_user_active())
     if "query" in request.args:
         return render_template("home.html", query=request.args.get("query"))
     return render_template("home.html")
@@ -104,6 +104,8 @@ def run_modules():
             if "modules" in request.json:
                 if "query_as_same" in request.json:
                     session = SessionModel.Session_class(request.json, query_as_same=True, parent_id=request.json["parent_id"])
+                elif "query_as_params" in request.json:
+                    session = SessionModel.Session_class(request.json, query_as_same=True, parent_id=request.json["same_query_id"])
                 else:
                     session = SessionModel.Session_class(request.json)
                 HomeModel.set_flask_session(session, request.json["parent_id"])
@@ -168,46 +170,60 @@ def download(sid):
 def modules_config():
     """List all modules for configuration"""
     sess["admin_user"] = admin_user_active()
+    flag = True
     if sess.get("admin_user"):
-        if current_user.is_authenticated:
-            return render_template("modules_config.html")
+        if not current_user.is_authenticated:
+            flag = False
+    if flag:
+        return render_template("modules_config.html")
     return render_template("404.html")
-
+    
+    
 @home_blueprint.route("/modules_config_data")
 def modules_config_data():
     """List all modules for configuration"""
     sess["admin_user"] = admin_user_active()
+    flag = True
     if sess.get("admin_user"):
-        if current_user.is_authenticated:
-            modules_config = HomeModel.get_modules_config()
-            return modules_config, 200
+        if not current_user.is_authenticated:
+            flag = False
+    if flag:
+        modules_config = HomeModel.get_modules_config()
+        return modules_config, 200
     return {"message": "Permission denied"}, 403
-
+    
 
 @home_blueprint.route("/change_config", methods=["POST"])
 def change_config():
     """Change configuation for a module"""
     sess["admin_user"] = admin_user_active()
+    flag = True
     if sess.get("admin_user"):
-        if current_user.is_authenticated:
-            if "module_name" in request.json["result_dict"]:
-                res = HomeModel.change_config_core(request.json["result_dict"])
-                if res:
-                    return {'message': 'Config changed', 'toast_class': "success-subtle"}, 200
-                return {'message': 'Something went wrong', 'toast_class': "danger-subtle"}, 400
-            return {'message': 'Need to pass "module_name"', 'toast_class': "warning-subtle"}, 400
+        if not current_user.is_authenticated:
+            flag = False  
+    if flag:
+        if "module_name" in request.json["result_dict"]:
+            res = HomeModel.change_config_core(request.json["result_dict"])
+            if res:
+                return {'message': 'Config changed', 'toast_class': "success-subtle"}, 200
+            return {'message': 'Something went wrong', 'toast_class': "danger-subtle"}, 400
+        return {'message': 'Need to pass "module_name"', 'toast_class': "warning-subtle"}, 400
     return {'message': 'Permission denied', 'toast_class': "danger-subtle"}, 403
 
 @home_blueprint.route("/change_status", methods=["GET"])
 def change_status():
     """Change the status of a module, active or unactive"""
     sess["admin_user"] = admin_user_active()
+    flag = True
     if sess.get("admin_user"):
-        if current_user.is_authenticated:
-            if "module_id" in request.args:
-                res = HomeModel.change_status_core(request.args.get("module_id"))
-                if res:
-                    return {'message': 'Module status changed', 'toast_class': "success-subtle"}, 200
-                return {'message': 'Something went wrong', 'toast_class': "danger-subtle"}, 400
-            return {'message': 'Need to pass "module_id"', 'toast_class': "warning-subtle"}, 400
+        if not current_user.is_authenticated:
+            flag = False
+    # if admin is active and user is logon or if admin is not active
+    if flag:
+        if "module_id" in request.args:
+            res = HomeModel.change_status_core(request.args.get("module_id"))
+            if res:
+                return {'message': 'Module status changed', 'toast_class': "success-subtle"}, 200
+            return {'message': 'Something went wrong', 'toast_class': "danger-subtle"}, 400
+        return {'message': 'Need to pass "module_id"', 'toast_class': "warning-subtle"}, 400
     return {'message': 'Permission denied', 'toast_class': "danger-subtle"}, 403

website/app/home_core.py

@@ -1,5 +1,5 @@
 import json
-from .utils.utils import query_get_module
+from .utils.utils import isUUID, query_get_module
 from . import db
 from .db_class.db import History, Module, Config, Module_Config, Session_db, History_Tree
 from flask import session as sess
@@ -150,14 +150,7 @@ def get_history():
 
 def create_new_session_tree(current_session, parent_id):
     loc_session = get_session(parent_id)
-    loc_json = {
-        "uuid": loc_session.uuid,
-        "modules": json.loads(loc_session.modules_list),
-        "query": loc_session.query_enter,
-        "input": loc_session.input_query,
-        "query_date": loc_session.query_date.strftime('%Y-%m-%d %H:%M'),
-        "config": json.loads(loc_session.config_module)
-    }
+    
     loc_json_child = {
         "uuid": current_session.uuid,
         "modules": current_session.modules_list,
@@ -165,12 +158,20 @@ def create_new_session_tree(current_session, parent_id):
         "input": current_session.input_query,
         "query_date": current_session.query_date.strftime('%Y-%m-%d'),
         "config": current_session.config_module,
-        "children": []
+        "children": list()
+    }
+    loc_json = {
+        "uuid": loc_session.uuid,
+        "modules": json.loads(loc_session.modules_list),
+        "query": loc_session.query_enter,
+        "input": loc_session.input_query,
+        "query_date": loc_session.query_date.strftime('%Y-%m-%d %H:%M'),
+        "config": json.loads(loc_session.config_module),
+        "children" : [loc_json_child]
     }
 
     sess["current_query"] = loc_session.uuid
     sess[sess.get("current_query")] = loc_json
-    sess[sess.get("current_query")]["children"] = [loc_json_child]
 
 def util_set_flask_session(parent_id, loc_session, current_session):
     if parent_id == loc_session["uuid"]:
@@ -201,9 +202,34 @@ def set_flask_session(current_session, parent_id):
         if not current_query or current_query not in sess:
             create_new_session_tree(current_session, parent_id)
         else:
-            # sess["uuid"]
-            loc_session = sess.get(sess.get("current_query"))
+            ## Check in current query
+            loc_session = sess.get(current_query)
             if not "children" in loc_session:
                 loc_session["children"] = list()
+            ## If not in current query, current query change for an other one
             if not util_set_flask_session(parent_id, loc_session, current_session):
-                create_new_session_tree(current_session, parent_id)
+                # sess["uuid"]
+                for q in sess:
+                    if isUUID(q) and not q == current_query:
+                        loc_session = sess.get(q)
+                        if not "children" in loc_session:
+                            loc_session["children"] = list()
+                        if util_set_flask_session(parent_id, loc_session, current_session):
+                            sess["current_query"] = q
+                            flag = False
+                            break
+                if flag:
+                    create_new_session_tree(current_session, parent_id)
+    else:
+        loc_json = {
+            "uuid": current_session.uuid,
+            "modules": current_session.modules_list,
+            "query": current_session.query,
+            "input": current_session.input_query,
+            "query_date": current_session.query_date.strftime('%Y-%m-%d %H:%M'),
+            "config": current_session.config_module,
+            "children": list()
+        }
+
+        sess["current_query"] = current_session.uuid
+        sess[sess.get("current_query")] = loc_json

website/app/session_class.py

@@ -3,7 +3,7 @@ import json
 from queue import Queue
 from threading import Thread
 from uuid import uuid4
-from .utils.utils import query_post_query, query_get_module, get_object
+from .utils.utils import query_post_query, query_get_module, get_object, get_limit_queries
 from . import home_core as HomeModel
 import uuid
 from . import db
@@ -179,7 +179,7 @@ class Session_class:
 
         histories = History.query.all()
         
-        while len(histories) > 200:
+        while len(histories) > get_limit_queries():
             history = History.query.order_by(History.id).all()
             session = Session_db.query.filter_by(id=history[0].session_id)
             if not History_Tree.query.filter_by(session_uuid=session.uuid):

website/app/templates/query.html

@@ -14,7 +14,21 @@
     <div class="btn-group" style="float: right;" role="group" aria-label="Basic mixed styles example">
         <a style="float: right;" class="btn btn-primary" href="/" title="Do a new query with no relation with this one">New query</a>
         <a style="float: right;" class="btn btn-secondary" href="/?query={{query}}" title="New query with same name">Query</a>
-      </div>
+        <div class="dropdown" style="float: right;">
+            <button class="btn btn-primary dropdown-toggle" title="New query with same parameters" style="border-radius: 0;" data-bs-toggle="dropdown" aria-expanded="false" data-bs-auto-close="outside">
+                Query as same
+            </button>
+            <div class="dropdown-menu p-4" style="min-width: 200px;">
+                <div class="mb-3">
+                    <label for="query_as_params" class="form-label">To query:</label>
+                    <input type="email" class="form-control" id="query_as_params" placeholder="{{query}}">
+                    <div id="query_as_params_error" style="color:brown"></div>
+                </div>
+                <button type="submit" class="btn btn-primary btn-sm" @click="query_as_params()" style="border-radius: 50px;">Query</button>
+            </div>
+        </div>
+    </div>
+    
     
 
     <div class="card card-body">
@@ -341,6 +355,7 @@ Value: [[misp_attr]]
                 history.value = loc
             }
 
+            // query 'value' with same parameters with a parent
             async function query_as_same(value){
                 let result_dict = {"modules": query_info.value["modules"], 
                                     "input": query_info.value["input_query"],
@@ -361,6 +376,33 @@ Value: [[misp_attr]]
                 }
             }
 
+            // query 'value' with same parameters without a parent
+             async function query_as_params(){
+                let loc = $("#query_as_params").val()
+                $("#query_as_params_error").text("")
+                if(loc){
+                    let result_dict = {"modules": query_info.value["modules"], 
+                                        "input": query_info.value["input_query"],
+                                        "query": loc,
+                                        "config": {},
+                                        "same_query_id": sid.value,
+                                        "parent_id": ""
+                                    }
+                    const res = await fetch('/run_modules',{
+                                    headers: { "X-CSRFToken": $("#csrf_token").val(), "Content-Type": "application/json" },
+                                    method: "POST",
+                                    body: JSON.stringify(result_dict)
+                                })
+                    if(await res.status == 201){
+                        let loc = await res.json()
+                        await nextTick()
+                        window.location.href="/query/" + loc['id']
+                    }
+                }else{
+                    $("#query_as_params_error").text("Please give value")
+                }
+            }
+
             onMounted(() => {
                 queryInfo()
                 actionQuery()
@@ -382,7 +424,8 @@ Value: [[misp_attr]]
                 parseMispObject,
                 parseMispAttr,
                 active_tab,
-                query_as_same
+                query_as_same,
+                query_as_params
             }
         }
     }).mount('.container-fluid')

website/app/utils/utils.py

@@ -1,10 +1,15 @@
 import os
+import random
 import uuid
 import json
 import requests
 # import jsonschema
-from config import Config
+from conf.config import Config
 from pathlib import Path
+import configparser
+config = configparser.ConfigParser()
+CONF_PATH = os.path.join(os.getcwd(), "conf", "config.cfg")
+config.read(CONF_PATH)
 
 MODULES = []
 
@@ -51,9 +56,22 @@ def get_object(obj_name):
 
 
 def admin_user_active():
-    return Config.ADMIN_USER
+    config.read(CONF_PATH)
+    return config.getboolean("ADMIN", "ADMIN_USER")
 
 def admin_password():
-    return Config.ADMIN_PASSWORD
+    return config["ADMIN"]["ADMIN_PASSWORD"]
 
+def gen_admin_password():
+    if not admin_password():
+        chars = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ!@$%#[]+-:;_&*().,?0123456789'
+        password = ''
+        for _ in range(20):
+            password += random.choice(chars)
+        print(f"##########################\n##    Admin password    ##\n## {password} ##\n##########################")
+        config["ADMIN"]["ADMIN_PASSWORD"] = password
+        with open(CONF_PATH, "w") as conffile:
+            config.write(conffile)
 
+def get_limit_queries():
+    return Config.QUERIES_LIMIT

website/conf/config.cfg.sample

@@ -0,0 +1,4 @@
+[ADMIN]
+admin_user = False
+admin_password = 
+

website/conf/config.py

@@ -4,8 +4,8 @@ class Config:
     FLASK_URL = '127.0.0.1'
     FLASK_PORT = 7008
     MISP_MODULE = '127.0.0.1:6666'
-    ADMIN_USER = False
-    ADMIN_PASSWORD = "Password1234"
+    
+    QUERIES_LIMIT = 200
 
 class DevelopmentConfig(Config):
     DEBUG = True