CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

chg: [search] disable whoosh index and search

otp
terrtia 2024-09-16 10:09:56 +02:00
parent 09bd748253
commit 907f370b29
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
3 changed files with 256 additions and 258 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
bin/LAUNCH.sh
View File

@ -210,8 +210,6 @@ function launching_scripts {
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Categ" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Categ.py; read x" screen -S "Script_AIL" -X screen -t "Categ" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Categ.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Indexer" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Indexer.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "Tags" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Tags.py; read x" screen -S "Script_AIL" -X screen -t "Tags" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Tags.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "SubmitPaste" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./SubmitPaste.py; read x" screen -S "Script_AIL" -X screen -t "SubmitPaste" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./SubmitPaste.py; read x"
@ -270,6 +268,8 @@ function launching_scripts {
# screen -S "Script_AIL" -X screen -t "LibInjection" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./LibInjection.py; read x" # screen -S "Script_AIL" -X screen -t "LibInjection" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./LibInjection.py; read x"
# sleep 0.1 # sleep 0.1
# screen -S "Script_AIL" -X screen -t "Pasties" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Pasties.py; read x" # screen -S "Script_AIL" -X screen -t "Pasties" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Pasties.py; read x"
# sleep 0.1
# screen -S "Script_AIL" -X screen -t "Indexer" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Indexer.py; read x"
# sleep 0.1 # sleep 0.1
screen -S "Script_AIL" -X screen -t "MISP_Thehive_Auto_Push" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./MISP_Thehive_Auto_Push.py; read x" screen -S "Script_AIL" -X screen -t "MISP_Thehive_Auto_Push" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./MISP_Thehive_Auto_Push.py; read x"

492
var/www/modules/search/Flask_search.py
View File

@ -1,251 +1,249 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*-coding:UTF-8 -* # -*-coding:UTF-8 -*
'''
Flask functions and routes for the trending modules page
'''
import os
import sys
import datetime
import flask
from flask import Flask, render_template, jsonify, request, Blueprint
from Role_Manager import login_admin, login_user_no_api # import os
from flask_login import login_required # import sys
# import datetime
from whoosh import index # import flask
from whoosh.fields import Schema, TEXT, ID # from flask import Flask, render_template, jsonify, request, Blueprint
from whoosh.qparser import QueryParser #
# from Role_Manager import login_admin, login_user_no_api
sys.path.append(os.environ['AIL_BIN']) # from flask_login import login_required
################################## #
# Import Project packages # from whoosh import index
################################## # from whoosh.fields import Schema, TEXT, ID
from lib.objects.Items import Item # from whoosh.qparser import QueryParser
#
import time # sys.path.append(os.environ['AIL_BIN'])
# ##################################
# ============ VARIABLES ============ # # Import Project packages
import Flask_config # ##################################
# from lib.objects.Items import Item
app = Flask_config.app #
config_loader = Flask_config.config_loader # import time
baseUrl = Flask_config.baseUrl #
max_preview_char = Flask_config.max_preview_char # # ============ VARIABLES ============
max_preview_modal = Flask_config.max_preview_modal # import Flask_config
bootstrap_label = Flask_config.bootstrap_label #
PASTES_FOLDER = Flask_config.PASTES_FOLDER # app = Flask_config.app
# config_loader = Flask_config.config_loader
baseindexpath = os.path.join(os.environ['AIL_HOME'], config_loader.get_config_str("Indexer", "path")) # baseUrl = Flask_config.baseUrl
indexRegister_path = os.path.join(os.environ['AIL_HOME'], config_loader.get_config_str("Indexer", "register")) # max_preview_char = Flask_config.max_preview_char
# max_preview_modal = Flask_config.max_preview_modal
searches = Blueprint('searches', __name__, template_folder='templates') # bootstrap_label = Flask_config.bootstrap_label
# PASTES_FOLDER = Flask_config.PASTES_FOLDER
# ============ FUNCTIONS ============ #
def get_current_index(): # baseindexpath = os.path.join(os.environ['AIL_HOME'], config_loader.get_config_str("Indexer", "path"))
with open(indexRegister_path, "r") as f: # indexRegister_path = os.path.join(os.environ['AIL_HOME'], config_loader.get_config_str("Indexer", "register"))
allIndex = f.read() #
allIndex = allIndex.split() # format [time1\ntime2] # searches = Blueprint('searches', __name__, template_folder='templates')
allIndex.sort() #
try: # # ============ FUNCTIONS ============
indexname = allIndex[-1].strip('\n\r') # def get_current_index():
except IndexError as e: # with open(indexRegister_path, "r") as f:
indexname = "no-index" # allIndex = f.read()
indexpath = os.path.join(baseindexpath, indexname) # allIndex = allIndex.split() # format [time1\ntime2]
return indexpath # allIndex.sort()
# try:
def get_index_list(selected_index=""): # indexname = allIndex[-1].strip('\n\r')
temp = [] # except IndexError as e:
index_list = [] # indexname = "no-index"
for dirs in os.listdir(baseindexpath): # indexpath = os.path.join(baseindexpath, indexname)
if os.path.isdir(os.path.join(baseindexpath, dirs)): # return indexpath
value = dirs #
name = to_iso_date(dirs) + " - " + \ # def get_index_list(selected_index=""):
str(get_dir_size(dirs) / (1000*1000)) + " Mb " #+ \ # temp = []
#"(" + str(get_item_count(dirs))''' + " Items" + ")" # index_list = []
flag = dirs==selected_index.split('/')[-1] # for dirs in os.listdir(baseindexpath):
if dirs == "old_index": # if os.path.isdir(os.path.join(baseindexpath, dirs)):
temp = [value, name, flag] # value = dirs
else: # name = to_iso_date(dirs) + " - " + \
index_list.append([value, name, flag]) # str(get_dir_size(dirs) / (1000*1000)) + " Mb " #+ \
# #"(" + str(get_item_count(dirs))''' + " Items" + ")"
index_list.sort(reverse=True, key=lambda x: x[0]) # flag = dirs==selected_index.split('/')[-1]
if len(temp) != 0: # if dirs == "old_index":
index_list.append(temp) # temp = [value, name, flag]
# else:
return index_list # index_list.append([value, name, flag])
#
def get_dir_size(directory): # index_list.sort(reverse=True, key=lambda x: x[0])
cur_sum = 0 # if len(temp) != 0:
for directory, subdirs, files in os.walk(os.path.join(baseindexpath,directory)): # index_list.append(temp)
try: #
cur_sum += sum(os.path.getsize(os.path.join(directory, name)) for name in files) # return index_list
except OSError as e: #File disappeared #
pass # def get_dir_size(directory):
return cur_sum # cur_sum = 0
# for directory, subdirs, files in os.walk(os.path.join(baseindexpath,directory)):
def get_item_count(dirs): # try:
ix = index.open_dir(os.path.join(baseindexpath, dirs)) # cur_sum += sum(os.path.getsize(os.path.join(directory, name)) for name in files)
return ix.doc_count_all() # except OSError as e: #File disappeared
# pass
def to_iso_date(timestamp): # return cur_sum
if timestamp == "old_index": #
return "old_index" # def get_item_count(dirs):
return str(datetime.datetime.fromtimestamp(int(timestamp))).split()[0] # ix = index.open_dir(os.path.join(baseindexpath, dirs))
# return ix.doc_count_all()
#
# ============ ROUTES ============ # def to_iso_date(timestamp):
# if timestamp == "old_index":
@searches.route("/search", methods=['POST']) # return "old_index"
@login_required # return str(datetime.datetime.fromtimestamp(int(timestamp))).split()[0]
@login_user_no_api #
def search(): #
query = request.form['query'] # # ============ ROUTES ============
q = [] #
q.append(query) # @searches.route("/search", methods=['POST'])
r = [] #complete path # @login_required
c = [] #preview of the paste content # @login_user_no_api
paste_date = [] # def search():
paste_size = [] # query = request.form['query']
paste_tags = [] # q = []
index_name = request.form['index_name'] # q.append(query)
num_elem_to_get = 50 # r = [] #complete path
# c = [] #preview of the paste content
# select correct index # paste_date = []
if index_name is None or index_name == "0": # paste_size = []
selected_index = get_current_index() # paste_tags = []
else: # index_name = request.form['index_name']
selected_index = os.path.join(baseindexpath, index_name) # num_elem_to_get = 50
#
''' temporary disabled # # select correct index
# # TODO: search by filename/item id # if index_name is None or index_name == "0":
''' # selected_index = get_current_index()
# else:
# Search full line # selected_index = os.path.join(baseindexpath, index_name)
schema = Schema(title=TEXT(stored=True), path=ID(stored=True), content=TEXT) #
# ''' temporary disabled
ix = index.open_dir(selected_index) # # # TODO: search by filename/item id
with ix.searcher() as searcher: # '''
query = QueryParser("content", ix.schema).parse("".join(q)) #
results = searcher.search_page(query, 1, pagelen=num_elem_to_get) # # Search full line
for x in results: # schema = Schema(title=TEXT(stored=True), path=ID(stored=True), content=TEXT)
r.append(x.items()[0][1].replace(PASTES_FOLDER, '', 1)) #
path = x.items()[0][1].replace(PASTES_FOLDER, '', 1) # ix = index.open_dir(selected_index)
item = Item(path) # with ix.searcher() as searcher:
content = item.get_content() # query = QueryParser("content", ix.schema).parse("".join(q))
content_range = max_preview_char if len(content)>max_preview_char else len(content)-1 # results = searcher.search_page(query, 1, pagelen=num_elem_to_get)
c.append(content[0:content_range]) # for x in results:
curr_date = item.get_date(separator=True) # r.append(x.items()[0][1].replace(PASTES_FOLDER, '', 1))
paste_date.append(curr_date) # path = x.items()[0][1].replace(PASTES_FOLDER, '', 1)
paste_size.append(item.get_size()) # item = Item(path)
p_tags = item.get_tags() # content = item.get_content()
l_tags = [] # content_range = max_preview_char if len(content)>max_preview_char else len(content)-1
for tag in p_tags: # c.append(content[0:content_range])
complete_tag = tag # curr_date = item.get_date(separator=True)
tag = tag.split('=') # paste_date.append(curr_date)
if len(tag) > 1: # paste_size.append(item.get_size())
if tag[1] != '': # p_tags = item.get_tags()
tag = tag[1][1:-1] # l_tags = []
# no value # for tag in p_tags:
else: # complete_tag = tag
tag = tag[0][1:-1] # tag = tag.split('=')
# use for custom tags # if len(tag) > 1:
else: # if tag[1] != '':
tag = tag[0] # tag = tag[1][1:-1]
# # no value
l_tags.append( (tag, complete_tag) ) # else:
# tag = tag[0][1:-1]
paste_tags.append(l_tags) # # use for custom tags
results = searcher.search(query) # else:
num_res = len(results) # tag = tag[0]
#
index_list = get_index_list() # l_tags.append( (tag, complete_tag) )
#
index_min = 1 # paste_tags.append(l_tags)
index_max = len(index_list) # results = searcher.search(query)
# num_res = len(results)
return render_template("search.html", r=r, c=c, #
query=request.form['query'], paste_date=paste_date, # index_list = get_index_list()
paste_size=paste_size, char_to_display=max_preview_modal, #
num_res=num_res, index_min=index_min, index_max=index_max, # index_min = 1
bootstrap_label=bootstrap_label, # index_max = len(index_list)
paste_tags=paste_tags, #
index_list=index_list # return render_template("search.html", r=r, c=c,
) # query=request.form['query'], paste_date=paste_date,
# paste_size=paste_size, char_to_display=max_preview_modal,
# num_res=num_res, index_min=index_min, index_max=index_max,
@searches.route("/get_more_search_result", methods=['POST']) # bootstrap_label=bootstrap_label,
@login_required # paste_tags=paste_tags,
@login_user_no_api # index_list=index_list
def get_more_search_result(): # )
query = request.form['query'] #
q = [] #
q.append(query) # @searches.route("/get_more_search_result", methods=['POST'])
page_offset = int(request.form['page_offset']) # @login_required
index_name = request.form['index_name'] # @login_user_no_api
num_elem_to_get = 50 # def get_more_search_result():
# query = request.form['query']
# select correct index # q = []
if index_name is None or index_name == "0": # q.append(query)
selected_index = get_current_index() # page_offset = int(request.form['page_offset'])
else: # index_name = request.form['index_name']
selected_index = os.path.join(baseindexpath, index_name) # num_elem_to_get = 50
#
path_array = [] # # select correct index
preview_array = [] # if index_name is None or index_name == "0":
date_array = [] # selected_index = get_current_index()
size_array = [] # else:
list_tags = [] # selected_index = os.path.join(baseindexpath, index_name)
#
schema = Schema(title=TEXT(stored=True), path=ID(stored=True), content=TEXT) # path_array = []
# preview_array = []
ix = index.open_dir(selected_index) # date_array = []
with ix.searcher() as searcher: # size_array = []
query = QueryParser("content", ix.schema).parse(" ".join(q)) # list_tags = []
results = searcher.search_page(query, page_offset, num_elem_to_get) #
for x in results: # schema = Schema(title=TEXT(stored=True), path=ID(stored=True), content=TEXT)
path = x.items()[0][1] #
path = path.replace(PASTES_FOLDER, '', 1) # ix = index.open_dir(selected_index)
path_array.append(path) # with ix.searcher() as searcher:
item = Item(path) # query = QueryParser("content", ix.schema).parse(" ".join(q))
content = item.get_content() # results = searcher.search_page(query, page_offset, num_elem_to_get)
content_range = max_preview_char if len(content)>max_preview_char else len(content)-1 # for x in results:
preview_array.append(content[0:content_range]) # path = x.items()[0][1]
curr_date = item.get_date(separator=True) # path = path.replace(PASTES_FOLDER, '', 1)
date_array.append(curr_date) # path_array.append(path)
size_array.append(item.get_size()) # item = Item(path)
p_tags = item.get_tags() # content = item.get_content()
l_tags = [] # content_range = max_preview_char if len(content)>max_preview_char else len(content)-1
for tag in p_tags: # preview_array.append(content[0:content_range])
complete_tag = tag # curr_date = item.get_date(separator=True)
tag = tag.split('=') # date_array.append(curr_date)
if len(tag) > 1: # size_array.append(item.get_size())
if tag[1] != '': # p_tags = item.get_tags()
tag = tag[1][1:-1] # l_tags = []
# no value # for tag in p_tags:
else: # complete_tag = tag
tag = tag[0][1:-1] # tag = tag.split('=')
# use for custom tags # if len(tag) > 1:
else: # if tag[1] != '':
tag = tag[0] # tag = tag[1][1:-1]
# # no value
l_tags.append( (tag, complete_tag) ) # else:
list_tags.append(l_tags) # tag = tag[0][1:-1]
# # use for custom tags
to_return = {} # else:
to_return["path_array"] = path_array # tag = tag[0]
to_return["preview_array"] = preview_array #
to_return["date_array"] = date_array # l_tags.append( (tag, complete_tag) )
to_return["size_array"] = size_array # list_tags.append(l_tags)
to_return["list_tags"] = list_tags #
to_return["bootstrap_label"] = bootstrap_label # to_return = {}
if len(path_array) < num_elem_to_get: #pagelength # to_return["path_array"] = path_array
to_return["moreData"] = False # to_return["preview_array"] = preview_array
else: # to_return["date_array"] = date_array
to_return["moreData"] = True # to_return["size_array"] = size_array
# to_return["list_tags"] = list_tags
return jsonify(to_return) # to_return["bootstrap_label"] = bootstrap_label
# if len(path_array) < num_elem_to_get: #pagelength
# to_return["moreData"] = False
# ========= REGISTRATION ========= # else:
app.register_blueprint(searches, url_prefix=baseUrl) # to_return["moreData"] = True
#
# return jsonify(to_return)
#
#
# # ========= REGISTRATION =========
# app.register_blueprint(searches, url_prefix=baseUrl)

18
var/www/templates/nav_bar.html
View File

@ -38,15 +38,15 @@
</li> </li>
</ul> </ul>
<form class="form-inline my-2 my-lg-0 ml-auto justify-content-center" action="{{ url_for('searches.search') }}" method=POST> {# <form class="form-inline my-2 my-lg-0 ml-auto justify-content-center" action="{{ url_for('searches.search') }}" method=POST>#}
<div class="d-flex flex-column"> {# <div class="d-flex flex-column">#}
<div> {# <div>#}
<input class="form-control mr-sm-2" type="search" id="global_search" name="query" placeholder="Search" aria-label="Search" aria-describedby="advanced_search"> {# <input class="form-control mr-sm-2" type="search" id="global_search" name="query" placeholder="Search" aria-label="Search" aria-describedby="advanced_search">#}
<input type="hidden" name="index_name" class="form-control" value="0" placeholder="Index Name"> {# <input type="hidden" name="index_name" class="form-control" value="0" placeholder="Index Name">#}
<button class="btn btn-outline-info my-2 my-sm-0" type="submit"><i class="fas fa-search"></i></button> {# <button class="btn btn-outline-info my-2 my-sm-0" type="submit"><i class="fas fa-search"></i></button>#}
</div> {# </div>#}
{#<small id="advanced_search" class="form-text"><a class="nav text-muted" href="#" aria-disabled="true">Advanced Search</a></small>#} {#<small id="advanced_search" class="form-text"><a class="nav text-muted" href="#" aria-disabled="true">Advanced Search</a></small>#}
</div> {# </div>#}
</form> {# </form>#}
</div> </div>
</nav> </nav>