CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

chg: [lib ConfigLoader] add configLoader class

pull/418/head
Terrtia 2019-10-28 13:48:43 +01:00
parent 48abb89d28
commit f1def65c89
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
19 changed files with 97 additions and 145 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
bin/Helper.py
View File

@ -58,7 +58,6 @@ class PubSub(object):
for address in addresses.split(','):
new_sub = context.socket(zmq.SUB)
new_sub.connect(address)
# bytes64 encode bytes to ascii only bytes
new_sub.setsockopt_string(zmq.SUBSCRIBE, channel)
self.subscribers.append(new_sub)

48
bin/lib/ConfigLoader.py Executable file
View File

@ -0,0 +1,48 @@
#!/usr/bin/python3
"""
The ``Domain``
===================
"""
import os
import sys
import time
import redis
import configparser
# Get Config file
config_dir = os.path.join(os.environ['AIL_HOME'], 'configs')
config_file = os.path.join(config_dir, 'core.cfg')
if not os.path.exists(config_file):
raise Exception('Unable to find the configuration file. \
Did you set environment variables? \
Or activate the virtualenv.')
# # TODO: create sphinx doc
# # TODO: add config_field to reload
class ConfigLoader(object):
"""docstring for Config_Loader."""
def __init__(self):
self.cfg = configparser.ConfigParser()
self.cfg.read(config_file)
def get_redis_conn(self, redis_name, decode_responses=True): ## TODO: verify redis name
return redis.StrictRedis( host=self.cfg.get(redis_name, "host"),
port=self.cfg.getint(redis_name, "port"),
db=self.cfg.getint(redis_name, "db"),
decode_responses=decode_responses )
def get_config_str(self, section, key_name):
return self.cfg.get(section, key_name)
def get_config_int(self, section, key_name):
return self.cfg.getint(section, key_name)
def get_config_boolean(self, section, key_name):
return self.cfg.getboolean(section, key_name)

31
var/www/Flask_server.py
View File

@ -24,6 +24,10 @@ sys.path.append('./modules/')
from User import User
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader
from pytaxonomies import Taxonomies
# Import config
@ -31,33 +35,21 @@ import Flask_config
# Import Blueprint
from blueprints.root import root
from blueprints.crawler_splash import crawler_splash
Flask_dir = os.environ['AIL_FLASK']
# CONFIG #
cfg = Flask_config.cfg
baseUrl = cfg.get("Flask", "baseurl")
config_loader = ConfigLoader.ConfigLoader()
baseUrl = config_loader.get_config_str("Flask", "baseurl")
baseUrl = baseUrl.replace('/', '')
if baseUrl != '':
baseUrl = '/'+baseUrl
# ========= REDIS =========#
r_serv_db = redis.StrictRedis(
host=cfg.get("ARDB_DB", "host"),
port=cfg.getint("ARDB_DB", "port"),
db=cfg.getint("ARDB_DB", "db"),
decode_responses=True)
r_serv_tags = redis.StrictRedis(
host=cfg.get("ARDB_Tags", "host"),
port=cfg.getint("ARDB_Tags", "port"),
db=cfg.getint("ARDB_Tags", "db"),
decode_responses=True)
r_cache = redis.StrictRedis(
host=cfg.get("Redis_Cache", "host"),
port=cfg.getint("Redis_Cache", "port"),
db=cfg.getint("Redis_Cache", "db"),
decode_responses=True)
r_serv_db = config_loader.get_redis_conn("ARDB_DB")
r_serv_tags = config_loader.get_redis_conn("ARDB_Tags")
r_cache = config_loader.get_redis_conn("Redis_Cache")
# logs
log_dir = os.path.join(os.environ['AIL_HOME'], 'logs')
@ -88,6 +80,7 @@ app.config['MAX_CONTENT_LENGTH'] = 900 * 1024 * 1024
# ========= BLUEPRINT =========#
app.register_blueprint(root, url_prefix=baseUrl)
app.register_blueprint(crawler_splash, url_prefix=baseUrl)
# ========= =========#
# ========= session ========
@ -199,7 +192,7 @@ def add_header(response):
@app.errorhandler(405)
def _handle_client_error(e):
if request.path.startswith('/api/'): ## # TODO: add baseUrl
if request.path.startswith('/api/'): ## # TODO: add baseUrl
res_dict = {"status": "error", "reason": "Method Not Allowed: The method is not allowed for the requested URL"}
anchor_id = request.path[8:]
anchor_id = anchor_id.replace('/', '_')

129
var/www/modules/Flask_config.py
View File

@ -4,109 +4,34 @@
'''
Flask global variables shared accross modules
'''
import configparser
import redis
import os
import re
import sys
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader
# FLASK #
app = None
# CONFIG #
configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg')
if not os.path.exists(configfile):
raise Exception('Unable to find the configuration file. \
Did you set environment variables? \
Or activate the virtualenv.')
cfg = configparser.ConfigParser()
cfg.read(configfile)
config_loader = ConfigLoader.ConfigLoader()
# REDIS #
r_serv = redis.StrictRedis(
host=cfg.get("Redis_Queues", "host"),
port=cfg.getint("Redis_Queues", "port"),
db=cfg.getint("Redis_Queues", "db"),
decode_responses=True)
r_cache = redis.StrictRedis(
host=cfg.get("Redis_Cache", "host"),
port=cfg.getint("Redis_Cache", "port"),
db=cfg.getint("Redis_Cache", "db"),
decode_responses=True)
r_serv_log = redis.StrictRedis(
host=cfg.get("Redis_Log", "host"),
port=cfg.getint("Redis_Log", "port"),
db=cfg.getint("Redis_Log", "db"),
decode_responses=True)
r_serv_log_submit = redis.StrictRedis(
host=cfg.get("Redis_Log_submit", "host"),
port=cfg.getint("Redis_Log_submit", "port"),
db=cfg.getint("Redis_Log_submit", "db"),
decode_responses=True)
r_serv_charts = redis.StrictRedis(
host=cfg.get("ARDB_Trending", "host"),
port=cfg.getint("ARDB_Trending", "port"),
db=cfg.getint("ARDB_Trending", "db"),
decode_responses=True)
r_serv_sentiment = redis.StrictRedis(
host=cfg.get("ARDB_Sentiment", "host"),
port=cfg.getint("ARDB_Sentiment", "port"),
db=cfg.getint("ARDB_Sentiment", "db"),
decode_responses=True)
r_serv_term = redis.StrictRedis(
host=cfg.get("ARDB_Tracker", "host"),
port=cfg.getint("ARDB_Tracker", "port"),
db=cfg.getint("ARDB_Tracker", "db"),
decode_responses=True)
r_serv_cred = redis.StrictRedis(
host=cfg.get("ARDB_TermCred", "host"),
port=cfg.getint("ARDB_TermCred", "port"),
db=cfg.getint("ARDB_TermCred", "db"),
decode_responses=True)
r_serv_pasteName = redis.StrictRedis(
host=cfg.get("Redis_Paste_Name", "host"),
port=cfg.getint("Redis_Paste_Name", "port"),
db=cfg.getint("Redis_Paste_Name", "db"),
decode_responses=True)
r_serv_tags = redis.StrictRedis(
host=cfg.get("ARDB_Tags", "host"),
port=cfg.getint("ARDB_Tags", "port"),
db=cfg.getint("ARDB_Tags", "db"),
decode_responses=True)
r_serv_metadata = redis.StrictRedis(
host=cfg.get("ARDB_Metadata", "host"),
port=cfg.getint("ARDB_Metadata", "port"),
db=cfg.getint("ARDB_Metadata", "db"),
decode_responses=True)
r_serv_db = redis.StrictRedis(
host=cfg.get("ARDB_DB", "host"),
port=cfg.getint("ARDB_DB", "port"),
db=cfg.getint("ARDB_DB", "db"),
decode_responses=True)
r_serv_statistics = redis.StrictRedis(
host=cfg.get("ARDB_Statistics", "host"),
port=cfg.getint("ARDB_Statistics", "port"),
db=cfg.getint("ARDB_Statistics", "db"),
decode_responses=True)
r_serv_onion = redis.StrictRedis(
host=cfg.get("ARDB_Onion", "host"),
port=cfg.getint("ARDB_Onion", "port"),
db=cfg.getint("ARDB_Onion", "db"),
decode_responses=True)
r_serv = config_loader.get_redis_conn("Redis_Queues")
r_cache = config_loader.get_redis_conn("Redis_Cache")
r_serv_log = config_loader.get_redis_conn("Redis_Log")
r_serv_log_submit = config_loader.get_redis_conn("Redis_Log_submit")
r_serv_charts = config_loader.get_redis_conn("ARDB_Trending")
r_serv_sentiment = config_loader.get_redis_conn("ARDB_Sentiment")
r_serv_term = config_loader.get_redis_conn("ARDB_Tracker")
r_serv_cred = config_loader.get_redis_conn("ARDB_TermCred")
r_serv_pasteName = config_loader.get_redis_conn("Redis_Paste_Name")
r_serv_tags = config_loader.get_redis_conn("ARDB_Tags")
r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
r_serv_db = config_loader.get_redis_conn("ARDB_DB")
r_serv_statistics = config_loader.get_redis_conn("ARDB_Statistics")
r_serv_onion = config_loader.get_redis_conn("ARDB_Onion")
sys.path.append('../../configs/keys')
# MISP #
@ -146,17 +71,17 @@ if HiveApi != False:
print('The Hive not connected')
#### VARIABLES ####
baseUrl = cfg.get("Flask", "baseurl")
baseUrl = config_loader.get_config_str("Flask", "baseurl")
baseUrl = baseUrl.replace('/', '')
if baseUrl != '':
baseUrl = '/'+baseUrl
max_preview_char = int(cfg.get("Flask", "max_preview_char")) # Maximum number of character to display in the tooltip
max_preview_modal = int(cfg.get("Flask", "max_preview_modal")) # Maximum number of character to display in the modal
max_preview_char = int(config_loader.get_config_str("Flask", "max_preview_char")) # Maximum number of character to display in the tooltip
max_preview_modal = int(config_loader.get_config_str("Flask", "max_preview_modal")) # Maximum number of character to display in the modal
max_tags_result = 50
DiffMaxLineLength = int(cfg.get("Flask", "DiffMaxLineLength"))#Use to display the estimated percentage instead of a raw value
DiffMaxLineLength = int(config_loader.get_config_str("Flask", "DiffMaxLineLength"))#Use to display the estimated percentage instead of a raw value
bootstrap_label = ['primary', 'success', 'danger', 'warning', 'info']
@ -166,14 +91,14 @@ dict_update_description = {'v1.5':{'nb_background_update': 5, 'update_warning_me
UPLOAD_FOLDER = os.path.join(os.environ['AIL_FLASK'], 'submitted')
PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], cfg.get("Directories", "pastes")) + '/'
SCREENSHOT_FOLDER = os.path.join(os.environ['AIL_HOME'], cfg.get("Directories", "crawled_screenshot"), 'screenshot')
PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], config_loader.get_config_str("Directories", "pastes")) + '/'
SCREENSHOT_FOLDER = os.path.join(os.environ['AIL_HOME'], config_loader.get_config_str("Directories", "crawled_screenshot"), 'screenshot')
REPO_ORIGIN = 'https://github.com/CIRCL/AIL-framework.git'
max_dashboard_logs = int(cfg.get("Flask", "max_dashboard_logs"))
max_dashboard_logs = int(config_loader.get_config_str("Flask", "max_dashboard_logs"))
crawler_enabled = cfg.getboolean("Crawler", "activate_crawler")
crawler_enabled = config_loader.get_config_boolean("Crawler", "activate_crawler")
email_regex = r'[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,6}'
email_regex = re.compile(email_regex)
@ -191,6 +116,6 @@ try:
vt_enabled = False
print('VT submission is disabled')
except:
vt_auth = {'apikey': cfg.get("Flask", "max_preview_char")}
vt_auth = {'apikey': config_loader.get_config_str("Flask", "max_preview_char")}
vt_enabled = False
print('VT submission is disabled')

1
var/www/modules/PasteSubmit/Flask_PasteSubmit.py
View File

@ -44,7 +44,6 @@ except:
import Flask_config
app = Flask_config.app
cfg = Flask_config.cfg
baseUrl = Flask_config.baseUrl
r_serv_tags = Flask_config.r_serv_tags
r_serv_metadata = Flask_config.r_serv_metadata

1
var/www/modules/Tags/Flask_Tags.py
View File

@ -23,7 +23,6 @@ import Flask_config
import Tag
app = Flask_config.app
cfg = Flask_config.cfg
baseUrl = Flask_config.baseUrl
r_serv_tags = Flask_config.r_serv_tags
r_serv_metadata = Flask_config.r_serv_metadata

6
var/www/modules/dashboard/Flask_dashboard.py
View File

@ -21,7 +21,7 @@ from flask_login import login_required
import Flask_config
app = Flask_config.app
cfg = Flask_config.cfg
config_loader = Flask_config.config_loader
baseUrl = Flask_config.baseUrl
r_serv = Flask_config.r_serv
r_serv_log = Flask_config.r_serv_log
@ -171,8 +171,8 @@ def stuff():
@login_required
@login_analyst
def index():
default_minute = cfg.get("Flask", "minute_processed_paste")
threshold_stucked_module = cfg.getint("Module_ModuleInformation", "threshold_stucked_module")
default_minute = config_loader.get_config_str("Flask", "minute_processed_paste")
threshold_stucked_module = config_loader.get_config_int("Module_ModuleInformation", "threshold_stucked_module")
log_select = {10, 25, 50, 100}
log_select.add(max_dashboard_logs)
log_select = list(log_select)

1
var/www/modules/hashDecoded/Flask_hashDecoded.py
View File

@ -24,7 +24,6 @@ from flask_login import login_required
import Flask_config
app = Flask_config.app
cfg = Flask_config.cfg
baseUrl = Flask_config.baseUrl
r_serv_metadata = Flask_config.r_serv_metadata
vt_enabled = Flask_config.vt_enabled

1
var/www/modules/hiddenServices/Flask_hiddenServices.py
View File

@ -23,7 +23,6 @@ from HiddenServices import HiddenServices
import Flask_config
app = Flask_config.app
cfg = Flask_config.cfg
baseUrl = Flask_config.baseUrl
r_cache = Flask_config.r_cache
r_serv_onion = Flask_config.r_serv_onion

1
var/www/modules/hunter/Flask_hunter.py
View File

@ -27,7 +27,6 @@ import Term
import Flask_config
app = Flask_config.app
cfg = Flask_config.cfg
baseUrl = Flask_config.baseUrl
r_serv_term = Flask_config.r_serv_term
r_serv_cred = Flask_config.r_serv_cred

1
var/www/modules/rawSkeleton/Flask_rawSkeleton.py
View File

@ -14,7 +14,6 @@ from flask_login import login_required
import Flask_config
app = Flask_config.app
cfg = Flask_config.cfg
rawSkeleton = Blueprint('rawSkeleton', __name__, template_folder='templates')

1
var/www/modules/restApi/Flask_restApi.py
View File

@ -31,7 +31,6 @@ import Flask_config
app = Flask_config.app
cfg = Flask_config.cfg
baseUrl = Flask_config.baseUrl
r_cache = Flask_config.r_cache
r_serv_db = Flask_config.r_serv_db

7
var/www/modules/search/Flask_search.py
View File

@ -25,7 +25,7 @@ import time
import Flask_config
app = Flask_config.app
cfg = Flask_config.cfg
config_loader = Flask_config.config_loader
baseUrl = Flask_config.baseUrl
r_serv_pasteName = Flask_config.r_serv_pasteName
r_serv_metadata = Flask_config.r_serv_metadata
@ -34,9 +34,8 @@ max_preview_modal = Flask_config.max_preview_modal
bootstrap_label = Flask_config.bootstrap_label
PASTES_FOLDER = Flask_config.PASTES_FOLDER
baseindexpath = os.path.join(os.environ['AIL_HOME'], cfg.get("Indexer", "path"))
indexRegister_path = os.path.join(os.environ['AIL_HOME'],
cfg.get("Indexer", "register"))
baseindexpath = os.path.join(os.environ['AIL_HOME'], config_loader.get_config_str("Indexer", "path"))
indexRegister_path = os.path.join(os.environ['AIL_HOME'], config_loader.get_config_str("Indexer", "register"))
searches = Blueprint('searches', __name__, template_folder='templates')

1
var/www/modules/sentiment/Flask_sentiment.py
View File

@ -20,7 +20,6 @@ import Paste
import Flask_config
app = Flask_config.app
cfg = Flask_config.cfg
baseUrl = Flask_config.baseUrl
r_serv_charts = Flask_config.r_serv_charts
r_serv_sentiment = Flask_config.r_serv_sentiment

1
var/www/modules/settings/Flask_settings.py
View File

@ -19,7 +19,6 @@ import git_status
import Flask_config
app = Flask_config.app
cfg = Flask_config.cfg
baseUrl = Flask_config.baseUrl
r_serv_db = Flask_config.r_serv_db
max_preview_char = Flask_config.max_preview_char

1
var/www/modules/showpaste/Flask_showpaste.py
View File

@ -23,7 +23,6 @@ import requests
import Flask_config
app = Flask_config.app
cfg = Flask_config.cfg
baseUrl = Flask_config.baseUrl
r_serv_pasteName = Flask_config.r_serv_pasteName
r_serv_metadata = Flask_config.r_serv_metadata

1
var/www/modules/terms/Flask_terms.py
View File

@ -29,7 +29,6 @@ import Term
import Flask_config
app = Flask_config.app
cfg = Flask_config.cfg
baseUrl = Flask_config.baseUrl
r_serv_term = Flask_config.r_serv_term
r_serv_cred = Flask_config.r_serv_cred

8
var/www/modules/trendingcharts/Flask_trendingcharts.py
View File

@ -17,7 +17,7 @@ from flask_login import login_required
import Flask_config
app = Flask_config.app
cfg = Flask_config.cfg
config_loader = Flask_config.config_loader
baseUrl = Flask_config.baseUrl
r_serv_charts = Flask_config.r_serv_charts
@ -69,7 +69,7 @@ def progressionCharts():
@login_required
@login_analyst
def wordstrending():
default_display = cfg.get("Flask", "default_display")
default_display = config_loader.get_config_str("Flask", "default_display")
return render_template("Wordstrending.html", default_display = default_display)
@ -77,7 +77,7 @@ def wordstrending():
@login_required
@login_analyst
def protocolstrending():
default_display = cfg.get("Flask", "default_display")
default_display = config_loader.get_config_str("Flask", "default_display")
return render_template("Protocolstrending.html", default_display = default_display)
@ -85,7 +85,7 @@ def protocolstrending():
@login_required
@login_analyst
def trending():
default_display = cfg.get("Flask", "default_display")
default_display = config_loader.get_config_str("Flask", "default_display")
return render_template("Trending.html", default_display = default_display)

1
var/www/modules/trendingmodules/Flask_trendingmodules.py
View File

@ -17,7 +17,6 @@ from flask_login import login_required
import Flask_config
app = Flask_config.app
cfg = Flask_config.cfg
baseUrl = Flask_config.baseUrl
r_serv_charts = Flask_config.r_serv_charts