new: [feature] external tools config and use

pull/670/head
David Cruciani 2024-06-28 11:32:21 +02:00
parent 555cfa807c
commit 35aa27ee51
No known key found for this signature in database
GPG Key ID: 8690CDE1E3994B9B
13 changed files with 514 additions and 41 deletions

View File

@ -34,9 +34,11 @@ def create_app():
from .home import home_blueprint
from .history.history import history_blueprint
from .account.account import account_blueprint
from .external_tools.external_tools import external_tools_blueprint
app.register_blueprint(home_blueprint, url_prefix="/")
app.register_blueprint(history_blueprint, url_prefix="/")
app.register_blueprint(account_blueprint, url_prefix="/")
app.register_blueprint(external_tools_blueprint, url_prefix="/")
csrf.exempt(home_blueprint)
return app

View File

@ -91,6 +91,20 @@ class User(UserMixin, db.Model):
"last_name": self.last_name,
"email": self.email
}
class ExternalTools(db.Model):
id = db.Column(db.Integer, primary_key=True, autoincrement=True)
name = db.Column(db.String(64), index=True)
url = db.Column(db.String)
is_active = db.Column(db.Boolean)
def to_json(self):
return {
"id": self.id,
"url": self.url,
"name": self.name,
"is_active": self.is_active
}
class AnonymousUser(AnonymousUserMixin):
def is_admin(self):

View File

@ -0,0 +1,72 @@
import json
from flask import Blueprint, render_template, request, jsonify, redirect, session as sess
from ..utils.utils import admin_user_active
from . import external_tools_core as ToolModel
from .form import ExternalToolForm
external_tools_blueprint = Blueprint(
'external_tools',
__name__,
template_folder='templates',
static_folder='static'
)
@external_tools_blueprint.route("/external_tools", methods=["GET"])
def external_tools():
"""View config page for external tools"""
sess["admin_user"] = admin_user_active()
return render_template("external_tools/external_tools_index.html")
@external_tools_blueprint.route("/external_tools/list", methods=['GET'])
def analyzers_data():
"""List all tools"""
return [tool.to_json() for tool in ToolModel.get_tools()]
@external_tools_blueprint.route("/add_external_tool", methods=['GET', 'POST'])
def add_external_tool():
"""Add a new tool"""
form = ExternalToolForm()
if form.validate_on_submit():
if ToolModel.add_tool_core(ToolModel.form_to_dict(form)):
return redirect("/external_tools")
return render_template("external_tools/add_external_tool.html", form=form)
@external_tools_blueprint.route("/external_tools/<tid>/delete_tool", methods=['GET', 'POST'])
def delete_tool(tid):
"""Delete a tool"""
if ToolModel.get_tool(tid):
if ToolModel.delete_tool(tid):
return {"message": "Tool deleted", "toast_class": "success-subtle"}, 200
return {"message": "Error tool deleted", 'toast_class': "danger-subtle"}, 400
return {"message": "Tool not found", 'toast_class': "danger-subtle"}, 404
@external_tools_blueprint.route("/external_tools/change_status", methods=['GET', 'POST'])
def change_status():
"""Active or disabled a tool"""
if "tool_id" in request.args:
res = ToolModel.change_status_core(request.args.get("tool_id"))
if res:
return {'message': 'Tool status changed', 'toast_class': "success-subtle"}, 200
return {'message': 'Something went wrong', 'toast_class': "danger-subtle"}, 400
return {'message': 'Need to pass "tool_id"', 'toast_class': "warning-subtle"}, 400
@external_tools_blueprint.route("/external_tools/change_config", methods=['GET', 'POST'])
def change_config():
"""Change configuration for a tool"""
if "tool_id" in request.json["result_dict"] and request.json["result_dict"]["tool_id"]:
if "tool_name" in request.json["result_dict"] and request.json["result_dict"]["tool_name"]:
if "tool_url" in request.json["result_dict"] and request.json["result_dict"]["tool_url"]:
res = ToolModel.change_config_core(request.json["result_dict"])
if res:
return {'message': 'Config changed', 'toast_class': "success-subtle"}, 200
return {'message': 'Something went wrong', 'toast_class': "danger-subtle"}, 400
return {'message': 'Need to pass "tool_url"', 'toast_class': "warning-subtle"}, 400
return {'message': 'Need to pass "tool_name"', 'toast_class': "warning-subtle"}, 400
return {'message': 'Need to pass "tool_id"', 'toast_class': "warning-subtle"}, 400

View File

@ -0,0 +1,59 @@
from .. import db
from ..db_class.db import *
def get_tool(tool_id):
"""Return a tool by id"""
return ExternalTools.query.get(tool_id)
def get_tools():
"""Return all External tools"""
return ExternalTools.query.all()
def change_status_core(tool_id):
"""Active or disabled a tool"""
an = get_tool(tool_id)
if an:
an.is_active = not an.is_active
db.session.commit()
return True
return False
def change_config_core(request_json):
"""Change config for a tool"""
tool = get_tool(request_json["tool_id"])
if tool:
tool.name = request_json["tool_name"]
tool.url = request_json["tool_url"]
db.session.commit()
return True
return False
def add_tool_core(form_dict):
tool = ExternalTools(
name=form_dict["name"],
url = form_dict["url"],
is_active=True
)
db.session.add(tool)
db.session.commit()
return True
def delete_tool(tool_id):
tool = get_tool(tool_id)
if tool:
db.session.delete(tool)
return True
return False
def form_to_dict(form):
loc_dict = dict()
for field in form._fields:
if field == "files_upload":
loc_dict[field] = dict()
loc_dict[field]["data"] = form._fields[field].data
loc_dict[field]["name"] = form._fields[field].name
elif not field == "submit" and not field == "csrf_token":
loc_dict[field] = form._fields[field].data
return loc_dict

View File

@ -0,0 +1,12 @@
from flask_wtf import FlaskForm
from wtforms.fields import (
StringField,
SubmitField,
)
from wtforms.validators import InputRequired, Length
class ExternalToolForm(FlaskForm):
name = StringField('Name', validators=[InputRequired(), Length(1, 64)])
url = StringField('Url', validators=[InputRequired()])
submit = SubmitField('Create')

View File

@ -1,10 +1,10 @@
import ast
import json
from flask import Blueprint, redirect, render_template, request, jsonify, session as sess
from flask import Blueprint, render_template, request, jsonify, session as sess
from flask_login import current_user
from . import session_class as SessionModel
from . import home_core as HomeModel
from .utils.utils import admin_user_active, FLOWINTEL_URL
from .utils.utils import admin_user_active
home_blueprint = Blueprint(
'home',
@ -255,8 +255,3 @@ def change_status():
return {'message': 'Need to pass "module_id"', 'toast_class': "warning-subtle"}, 400
return {'message': 'Permission denied', 'toast_class': "danger-subtle"}, 403
@home_blueprint.route("/flowintel_url")
def flowintel_url():
"""send result to flowintel-cm"""
return {"url": f"{FLOWINTEL_URL}/analyzer/recieve_result"}, 200

View File

@ -0,0 +1,32 @@
<!--
Author: David Cruciani
-->
{% extends 'base.html' %}
{% block content %}
<form action="" method="post">
{{ form.hidden_tag() }}
<h2>Add External tool</h2>
<hr>
<div class="row">
<div class="mb-3 w-50">
{{form.name.label(class_="col-form-label")}}:
{{form.name(class_="form-control")}}
{% if form.name.errors %}
<div style="color: red;">{{form.name.errors[0] | safe}}</div>
{%endif%}
</div>
<div class="mb-3 w-50">
{{form.url.label(class_="col-form-label")}}:
{{form.url(class_="form-control")}}
{% if form.url.errors %}
<div style="color: red;">{{form.url.errors[0] | safe}}</div>
{%endif%}
</div>
</div>
{{form.submit(class='btn btn-primary')}}
</form>
{% endblock %}

View File

@ -0,0 +1,189 @@
<!--
Author: David Cruciani
-->
{% extends 'base.html' %}
{% block content %}
<div style="display: flex;">
<h2>External tools</h2>
<span style="margin-top: 4px; margin-left: 7px;">
<a class="btn btn-primary btn-sm" href="/add_external_tool" title="Add a new external tool"><i class="fa-solid fa-plus"></i></a>
</span>
</div>
<hr>
<div id="top"></div>
<div style="width:50%; transform: translate(50%, 0);">
<div>
<input type="search" @input="onInput" placeholder="Search tools" autofocus class="form-control" style="border-radius: 5px;" />
</div>
</div>
<br>
<div class="row" style="margin-bottom: 100px;">
<div class="col" style="flex: 0 0 50%">
<div class="list-group">
<div v-for="tool in tools_config" style="display:flex; ">
<input v-if="tool.is_active || tool.is_active == null" type="checkbox" style="margin-right: 5px;" checked @click="change_status(tool)">
<input v-else type="checkbox" style="margin-right: 5px;" @click="change_status(tool)">
<a class="list-group-item list-group-item-action" style="border-radius: 10px;" :title="tool.description" @click="display_config(tool)">
[[tool.name]]
</a>
</div>
</div>
</div>
<!-- Right panel -->
<div v-if="Object.keys(current_config).length" class="side-panel-config">
<div class="round-button" title="close" style="margin-top: 3px;">
<div class="round-button-circle">
<a @click="close_panel()" class="round-button">x</a>
</div>
</div>
<br>
<h4>[[ current_config.tool_name ]]</h4>
<div class="mb-3">
<label :for="'form-name-'+current_config.tool_id" class="form-label">Name: </label>
<input type="text" class="form-control" :id="'form-name-'+current_config.tool_id" :value="current_config.tool_name">
<span style="color: brown" :id="'error-name-'+current_config.tool_id"></span>
</div>
<div class="mb-3">
<label :for="'form-url-'+current_config.tool_id" class="form-label">Url: </label>
<input type="text" class="form-control" :id="'form-url-'+current_config.tool_id" :value="current_config.tool_url">
<span style="color: brown" :id="'error-url-'+current_config.tool_id"></span>
</div>
<button class="btn btn-primary" @click="change_config()">Save</button>
<button class="btn btn-danger" @click="delete_tool()" style="margin-left: 5px;">Delete</button>
</div>
</div>
<span id="goTop">[<a href="#top">Go Back Top</a>]</span>
{% endblock %}
{% block script %}
<script type="module">
const { createApp, ref, onMounted, nextTick } = Vue
import {display_toast, message_list} from '/static/js/toaster.js'
createApp({
delimiters: ['[[', ']]'],
setup() {
const tools_config = ref({})
const current_config = ref({})
let loc_tools = {}
async function query_tools(){
let res = await fetch("/external_tools/list")
let loc = await res.json()
tools_config.value = loc
loc_tools = tools_config.value
}
query_tools()
async function display_config(tool){
current_config.value = {}
current_config.value["tool_name"] = tool.name
current_config.value["tool_url"] = tool.url
current_config.value["tool_id"] = tool.id
}
function close_panel(){
current_config.value = {}
}
async function change_config(){
$("#error-name-"+current_config.value["tool_id"]).text("")
$("#error-url-"+current_config.value["tool_id"]).text("")
let result_dict = {}
result_dict["tool_id"] = current_config.value["tool_id"]
let loc_name = $("#form-name-"+current_config.value["tool_id"]).val()
if(!loc_name){
$("#error-name-"+current_config.value["tool_id"]).text('Cannot be empty')
return
}
let loc_url = $("#form-url-"+current_config.value["tool_id"]).val()
if(!loc_url){
$("#error-url-"+current_config.value["tool_id"]).text('Cannot be empty')
return
}
// Update result_dict and current_config
result_dict["tool_name"] = loc_name
current_config.value["tool_name"] = loc_name
result_dict["tool_url"] = loc_url
current_config.value["tool_url"] = loc_url
// Update list of tools with new value for current tool
for(let i in tools_config.value){
if(tools_config.value[i].id == current_config.value["tool_id"] ){
tools_config.value[i].name = loc_name
tools_config.value[i].url = loc_url
}
}
const res = await fetch('/external_tools/change_config',{
headers: { "X-CSRFToken": $("#csrf_token").val(), "Content-Type": "application/json" },
method: "POST",
body: JSON.stringify({
result_dict
})
})
display_toast(res)
}
async function change_status(tool){
let res = await fetch("/external_tools/change_status?tool_id="+tool.id)
if(await res.status_code == 200){
tool.is_active = !tool.is_active
}
display_toast(res)
}
function onInput(e){
tools_config.value = []
if(e.target.value){
tools_config.value = loc_tools.filter((tool) => {
return tool.name.toLowerCase().includes(e.target.value.toLowerCase())
})
}else{
tools_config.value = loc_tools
}
}
async function delete_tool(){
let loc_id = current_config.value["tool_id"]
let res = await fetch("/external_tools/"+loc_id+"/delete_tool")
if(await res.status_code == 200){
current_config.value = {}
let index
for(let i in tools_config.value){
if(tools_config.value[i].id == loc_id){
index = i
break
}
}
delete tools_config.value[index]
}
display_toast(res)
}
return {
message_list,
tools_config,
current_config,
display_config,
close_panel,
change_config,
change_status,
onInput,
delete_tool
}
}
}).mount('.container-fluid')
</script>
{% endblock %}

View File

@ -55,7 +55,35 @@
</div>
<span v-if="status_site" style="margin-left: 5px; font-size: 13px; float: right;">[[status_site]]</span>
<br>
<button class="btn btn-secondary btn-sm" @click="send_flowintel_cm_all()" title="sendd all result to Flowintel-cm" style="float: right;">Flowintel-cm all</button>
<button type="button" class="btn btn-secondary btn-sm" @click="send_all = -1" title="Send all results to an external tool" style="float: right;" data-bs-toggle="modal" data-bs-target="#Send_to_modal">
External tools
<i class="fa-solid fa-share-from-square"></i>
</button>
<!-- Modal send to -->
<div class="modal fade" id="Send_to_modal" tabindex="-1" aria-labelledby="Send_to_modalLabel" aria-hidden="true">
<div class="modal-dialog modal-lg">
<div class="modal-content">
<div class="modal-header">
<h1 class="modal-title fs-5" id="Send_to_modalLabel">Send to external tools</h1>
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
</div>
<div class="modal-body">
<label for="tools_select">Tools:</label>
<select data-placeholder="Tools" class="select2-select form-control" name="tools_select" id="tools_select" >
<option value="None">--</option>
<template v-for="tool, key in external_tools">
<option v-if="tool.is_active" :value="[[key]]">[[tool.name]]</option>
</template>
</select>
<div id="tools_errors" class="invalid-feedback"></div>
</div>
<div class="modal-footer">
<button type="button" @click="submit_external_tool()" class="btn btn-primary">Submit</button>
</div>
</div>
</div>
</div>
<br/>
@ -108,8 +136,11 @@
</button>
</h2>
<div :id="'panelsStayOpenMain-'+key_query" class="accordion-collapse collapse show">
<button class="btn btn-secondary" @click="send_flowintel_cm(key_query)" title="Send this result to Flowintel-cm" style="margin-top: 10px;margin-left: 10px;">Flowintel-cm</button>
<div class="accordion" style="padding: 25px">
<button type="button" class="btn btn-secondary btn-sm" @click="send_all = key_query" title="Send this result to an external tool" data-bs-toggle="modal" data-bs-target="#Send_to_modal" style="margin-top: 5px; margin-left:5px">
External tools
<i class="fa-solid fa-share-from-square"></i>
</button>
<div class="accordion" style="padding: 20px">
<div class="accordion-item" :id="'list-item-'+key_query+'-'+key" v-for="result, key in ele">
<template v-if="!('error' in result)">
<h2 class="accordion-header">
@ -260,7 +291,7 @@ Value: [[misp_attr]]
{% block script %}
<script type="module">
const { createApp, ref, onMounted, nextTick, defineComponent} = Vue
const { createApp, ref, onMounted, nextTick} = Vue
import {message_list} from '/static/js/toaster.js'
import history_view from '/static/js/history/history_tree_query.js'
createApp({
@ -280,6 +311,9 @@ Value: [[misp_attr]]
const history = ref({})
const query_info = ref({})
const external_tools = ref({})
const send_all = ref()
function actionQuery(){
is_searching.value = true
@ -382,13 +416,14 @@ Value: [[misp_attr]]
loc = loc.split(', ')
$("#query_as_params_error").text("")
if(loc){
let result_dict = {"modules": query_info.value["modules"],
"input": query_info.value["input_query"],
"query": loc,
"config": {},
"same_query_id": sid.value,
"parent_id": ""
}
let result_dict = {
"modules": query_info.value["modules"],
"input": query_info.value["input_query"],
"query": loc,
"config": {},
"same_query_id": sid.value,
"parent_id": ""
}
const res = await fetch('/run_modules',{
headers: { "X-CSRFToken": $("#csrf_token").val(), "Content-Type": "application/json" },
method: "POST",
@ -404,27 +439,36 @@ Value: [[misp_attr]]
}
}
async function sender_flowintel(to_return){
const res = await fetch('/flowintel_url')
let loc = await res.json()
const flowintel_cm_url = loc["url"]
$('#insert_form').append(
$('<form>').attr({"action": flowintel_cm_url, "name": "flowintel", "method": "post", "style": "display:none"}).append(
$("<input>").attr({"type": "text", "name": "result"}).val(JSON.stringify(to_return))
)
);
document.forms['flowintel'].submit();
async function fetch_external_tools(){
const res = await fetch("/external_tools/list")
if(await res.status==400 ){
display_toast(res)
}else{
let loc = await res.json()
external_tools.value = loc
}
}
fetch_external_tools()
async function send_flowintel_cm(key){
let to_return = {}
to_return[key] = modules_res.value[key]
sender_flowintel(to_return)
}
async function submit_external_tool(){
let tool_selected = $("#tools_select").val()
if(tool_selected != 'None'){
let to_return = {}
if(send_all.value != -1){
to_return[send_all.value] = modules_res.value[send_all.value]
}else{
to_return = modules_res.value
}
async function send_flowintel_cm_all(){
sender_flowintel(modules_res.value)
$('#insert_form').append(
$('<form>').attr(
{"action": external_tools.value[tool_selected].url, "name": "external_tools_form", "method": "post", "style": "display:none"}).append(
$("<input>").attr({"type": "text", "name": "results"}).val(JSON.stringify(to_return)
)
)
);
document.forms['external_tools_form'].submit();
}
}
onMounted(() => {
@ -432,6 +476,11 @@ Value: [[misp_attr]]
actionQuery()
get_history_session()
window._query_as_same = query_as_same
$('.select2-select').select2({
theme: 'bootstrap-5',
width: '50%'
})
})
return {
@ -447,11 +496,12 @@ Value: [[misp_attr]]
generateCoreFormatUI,
parseMispObject,
parseMispAttr,
external_tools,
send_all,
active_tab,
query_as_same,
query_as_params,
send_flowintel_cm,
send_flowintel_cm_all
submit_external_tool
}
}
}).mount('.container-fluid')

View File

@ -39,6 +39,17 @@
<i class="fa-solid fa-gear fa-fw me-3"></i><span>Config</span>
</a>
{%endif%}
{% if session.admin_user %}
{% if current_user.is_authenticated %}
<a style="margin-top: 30px;" href="/external_tools" class="list-group-item list-group-item-action text-nowrap">
<i class="fa-solid fa-link fa-fw me-3"></i><span>External tools</span>
</a>
{%endif%}
{%else%}
<a style="margin-top: 30px;" href="/external_tools" class="list-group-item list-group-item-action text-nowrap">
<i class="fa-solid fa-link fa-fw me-3"></i><span>External tools</span>
</a>
{%endif%}
</div>
</div>
</div>

View File

@ -3,7 +3,6 @@ import random
import uuid
import json
import requests
# import jsonschema
from conf.config import Config
from pathlib import Path
import configparser
@ -12,7 +11,6 @@ CONF_PATH = os.path.join(os.getcwd(), "conf", "config.cfg")
config.read(CONF_PATH)
MODULES = []
FLOWINTEL_URL = Config.FLOWINTEL_URL
def query_get_module(headers={'Content-type': 'application/json'}):
global MODULES

View File

@ -4,7 +4,6 @@ class Config:
FLASK_URL = '127.0.0.1'
FLASK_PORT = 7008
MISP_MODULE = '127.0.0.1:6666'
FLOWINTEL_URL = 'http://localhost:7006'
QUERIES_LIMIT = 200

View File

@ -0,0 +1,40 @@
"""empty message
Revision ID: 3a631d400f60
Revises: 183bf8fa2b87
Create Date: 2024-06-27 11:15:52.165895
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '3a631d400f60'
down_revision = '183bf8fa2b87'
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.create_table('external_tools',
sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
sa.Column('name', sa.String(length=64), nullable=True),
sa.Column('url', sa.String(), nullable=True),
sa.Column('is_active', sa.Boolean(), nullable=True),
sa.PrimaryKeyConstraint('id')
)
with op.batch_alter_table('external_tools', schema=None) as batch_op:
batch_op.create_index(batch_op.f('ix_external_tools_name'), ['name'], unique=False)
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
with op.batch_alter_table('external_tools', schema=None) as batch_op:
batch_op.drop_index(batch_op.f('ix_external_tools_name'))
op.drop_table('external_tools')
# ### end Alembic commands ###