Merge pull request #422 from CIRCL/misp_modules

Misp modules

OVERVIEW.md

@@ -97,12 +97,24 @@ Redis and ARDB overview
 | ------ | ------ | ------ |
 | ail:all_role | **role** | **int, role priority (1=admin)** |
 
+##### MISP Modules:
+
+| Set Key | Value |
+| ------ | ------ |
+| enabled_misp_modules | **module name** |
+
+| Key | Value |
+| ------ | ------ |
+| misp_module:**module name** | **module dict** |
+
 ##### Item Import:
 | Key | Value |
 | ------ | ------ |
 | **uuid**:isfile   | **boolean** |
 | **uuid**:paste_content | **item_content** |
 
+## DB2 - TermFreq:
+
 | Set Key | Value |
 | ------ | ------ |
 | submitted:uuid | **uuid** |

bin/lib/MispModules.py

@@ -0,0 +1,125 @@
+#!/usr/bin/python3
+
+import os
+import json
+import redis
+import requests
+import configparser
+
+misp_module_url = 'http://localhost:6666'
+
+default_config_path = os.path.join(os.environ['AIL_HOME'], 'configs', 'misp_modules.cfg')
+
+sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
+import ConfigLoader
+
+config_loader = ConfigLoader.ConfigLoader()
+r_serv = config_loader.get_redis_conn("ARDB_DB")
+config_loader = None
+
+def init_config(config_path=default_config_path):
+    config = configparser.ConfigParser()
+    if os.path.isfile(config_path):
+        config.read(config_path)
+    else:
+        config.add_section('misp_modules')
+        config.set('misp_modules', 'url', 'http://localhost')
+        config.set('misp_modules', 'port', '6666')
+    return config
+
+def init_module_config(module_json, config, config_path=default_config_path):
+    if 'config' in module_json['meta']:
+        if module_json['meta']['config']:
+            if module_json['name'] not in config:
+                config.add_section(module_json['name'])
+            for config_var in module_json['meta']['config']:
+                if config_var not in config[module_json['name']]:
+                    config.set(module_json['name'], config_var, '')
+    return config
+
+def load_modules_list():
+    req = requests.get('{}/modules'.format(misp_module_url))
+    if req.status_code == 200:
+        all_misp_modules = req.json()
+        all_modules = []
+        for module_json in all_misp_modules:
+
+            #filter module-types
+            if 'hover' in module_json['meta']['module-type'] or 'expansion' in module_json['meta']['module-type']:
+                all_modules.append(module_json)
+
+            # # TODO: handle import/export modules
+
+        config = init_config()
+        r_serv.delete('misp_modules')
+        for module_json in all_modules:
+            config = init_module_config(module_json, config, config_path=default_config_path)
+            r_serv.hset('misp_modules', module_json['name'], json.dumps(module_json))
+
+        with open(default_config_path, 'w') as f:
+            config.write(f)
+
+    else:
+        print('Error: Module service not reachable.')
+
+
+def build_config_json(module_name):
+    misp_module_config = configparser.ConfigParser()
+    misp_module_config.read(default_config_path)
+    dict_config = {}
+    if module_name in misp_module_config:
+        for config_key in misp_module_config[module_name]:
+            config_value = misp_module_config[module_name][config_key]
+            if config_value:
+                dict_config[config_key] = config_value
+    return dict_config
+
+def build_enrichment_request_json(module_name, var_name, var_value):
+    # # TODO: add error handler
+    request_dict = {'module': module_name, var_name: var_value}
+    # add config
+    config_json = build_config_json(module_name)
+    if config_json:
+        request_dict['config'] = config_json
+    return json.dumps(request_dict)
+
+def misp_module_enrichment_request(misp_module_url, misp_module_port, request_content):
+    # # TODO: check if module is enabled
+    endpoint_url = '{}:{}/query'.format(misp_module_url, misp_module_port)
+    req = requests.post(endpoint_url, headers={'Content-Type': 'application/json'}, data=request_content)
+    if req.status_code == 200:
+        response = req.json()
+        if response:
+            return parse_module_enrichment_response(response)
+    else:
+        print('error: {} Enrichment service not reachable.'.format(req.status_code,))
+        return ''
+
+def parse_module_enrichment_response(misp_module_response):
+    print(misp_module_response)
+    response_values = []
+    if 'results' in misp_module_response:
+        # # TODO: handle misp_format (Attribute, Object, Tags)
+        response_types = []
+        for result in misp_module_response['results']:
+            # get all types
+            for resp_type in result['types']:
+                response_types.append(resp_type)
+            # get all values
+            for resp_value in result['values']:
+                response_values.append(resp_value)
+        # TODO: handle / verify / use response types
+        #print(response_types)
+    return response_values
+
+if __name__ == "__main__":
+
+    load_modules_list()
+
+    misp_module_url = 'http://localhost'
+    misp_module_port = 6666
+
+    bitcoin_address = 'bitcoin_address'
+    test_content = build_enrichment_request_json('btc_steroids', 'btc', bitcoin_address)
+    print(test_content)
+    misp_module_enrichment_request(misp_module_url, misp_module_port, test_content)

update/v2.5/Update.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+import os
+import re
+import sys
+import time
+import redis
+import datetime
+
+sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
+import ConfigLoader
+
+new_version = 'v2.5'
+
+if __name__ == '__main__':
+
+    start_deb = time.time()
+
+    config_loader = ConfigLoader.ConfigLoader()
+    r_serv = config_loader.get_redis_conn("ARDB_DB")
+    config_loader = None
+
+    r_serv.zadd('ail:all_role', 3, 'user')
+    r_serv.zadd('ail:all_role', 4, 'user_no_api')
+    r_serv.zadd('ail:all_role', 5, 'read_only')
+
+    for user in r_serv.hkeys(user:all):
+        r_serv.sadd('user_role:user', user)
+        r_serv.sadd('user_role:user_no_api', user)
+        r_serv.sadd('user_role:read_only', user)
+
+    #Set current ail version
+    r_serv.set('ail:version', new_version)
+
+    #Set current ail version
+    r_serv.hset('ail:update_date', new_version, datetime.datetime.now().strftime("%Y%m%d"))

update/v2.5/Update.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1;
+[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1;
+[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
+[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
+[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1;
+
+export PATH=$AIL_HOME:$PATH
+export PATH=$AIL_REDIS:$PATH
+export PATH=$AIL_ARDB:$PATH
+export PATH=$AIL_BIN:$PATH
+export PATH=$AIL_FLASK:$PATH
+
+GREEN="\\033[1;32m"
+DEFAULT="\\033[0;39m"
+
+echo -e $GREEN"Shutting down AIL ..."$DEFAULT
+bash ${AIL_BIN}/LAUNCH.sh -ks
+wait
+
+bash ${AIL_BIN}/LAUNCH.sh -lav &
+wait
+echo ""
+
+echo ""
+echo -e $GREEN"Updating AIL VERSION ..."$DEFAULT
+echo ""
+python ${AIL_HOME}/update/v2.5/Update.py
+wait
+echo ""
+echo ""
+
+echo ""
+echo -e $GREEN"Shutting down ARDB ..."$DEFAULT
+bash ${AIL_BIN}/LAUNCH.sh -ks
+wait
+
+exit 0

var/www/blueprints/correlation.py

@@ -18,7 +18,7 @@ import Flask_config
 
 # Import Role_Manager
 from Role_Manager import create_user_db, check_password_strength, check_user_role_integrity
-from Role_Manager import login_admin, login_analyst
+from Role_Manager import login_admin, login_analyst, login_read_only
 
 sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
 import Correlate_object
@@ -121,7 +121,7 @@ def get_card_metadata(object_type, correlation_id, type_id=None, expand_card=Fal
 # ============= ROUTES ==============
 @correlation.route('/correlation/show_correlation', methods=['GET', 'POST']) # GET + POST
 @login_required
-@login_analyst
+@login_read_only
 def show_correlation():
     if request.method == 'POST':
         object_type = request.form.get('object_type')
@@ -197,7 +197,7 @@ def show_correlation():
 
 @correlation.route('/correlation/graph_node_json')
 @login_required
-@login_analyst
+@login_read_only
 def graph_node_json(): # # TODO: use post
     correlation_id = request.args.get('correlation_id')
     type_id = request.args.get('type_id')

var/www/blueprints/crawler_splash.py

@@ -18,7 +18,7 @@ import Flask_config
 
 # Import Role_Manager
 from Role_Manager import create_user_db, check_password_strength, check_user_role_integrity
-from Role_Manager import login_admin, login_analyst
+from Role_Manager import login_admin, login_analyst, login_read_only
 
 sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))
 import Tag
@@ -47,7 +47,7 @@ def api_validator(api_response):
 # add route : /crawlers/show_domain
 @crawler_splash.route('/crawlers/showDomain')
 @login_required
-@login_analyst
+@login_read_only
 def showDomain():
     domain_name = request.args.get('domain')
     epoch = request.args.get('epoch')

var/www/create_default_user.py

@@ -23,6 +23,9 @@ if __name__ == "__main__":
     if not r_serv.exists('ail:all_role'):
         r_serv.zadd('ail:all_role', 1, 'admin')
         r_serv.zadd('ail:all_role', 2, 'analyst')
+        r_serv.zadd('ail:all_role', 3, 'user')
+        r_serv.zadd('ail:all_role', 4, 'user_no_api')
+        r_serv.zadd('ail:all_role', 5, 'read_only')
 
     username = 'admin@admin.test'
     password = gen_password()

var/www/modules/Role_Manager.py

@@ -16,7 +16,7 @@ from flask_login import LoginManager, current_user, login_user, logout_user, log
 from flask import request, make_response, current_app
 
 login_manager = LoginManager()
-login_manager.login_view = 'role'
+login_manager.login_view = 'root.role'
 
 # CONFIG #
 config_loader = ConfigLoader.ConfigLoader()
@@ -68,7 +68,35 @@ def login_analyst(func):
         return func(*args, **kwargs)
     return decorated_view
 
+def login_user(func):
+    @wraps(func)
+    def decorated_view(*args, **kwargs):
+        if not current_user.is_authenticated:
+            return login_manager.unauthorized()
+        elif (not current_user.is_in_role('user')):
+            return login_manager.unauthorized()
+        return func(*args, **kwargs)
+    return decorated_view
 
+def login_user_no_api(func):
+    @wraps(func)
+    def decorated_view(*args, **kwargs):
+        if not current_user.is_authenticated:
+            return login_manager.unauthorized()
+        elif (not current_user.is_in_role('user_no_api')):
+            return login_manager.unauthorized()
+        return func(*args, **kwargs)
+    return decorated_view
+
+def login_read_only(func):
+    @wraps(func)
+    def decorated_view(*args, **kwargs):
+        if not current_user.is_authenticated:
+            return login_manager.unauthorized()
+        elif (not current_user.is_in_role('read_only')):
+            return login_manager.unauthorized()
+        return func(*args, **kwargs)
+    return decorated_view
 
 ###############################################################
 ###############################################################
@@ -107,11 +135,15 @@ def create_user_db(username_id , password, default=False, role=None, update=Fals
     # create user token
     generate_new_token(username_id)
 
+    if not role:
+        role = 'read_only'
+
     if update:
         r_serv_db.hdel('user_metadata:{}'.format(username_id), 'change_passwd')
         # remove default user password file
         if username_id=='admin@admin.test':
             os.remove(default_passwd_file)
+        r_serv_db.hset('user:all', username_id, password_hash)
     else:
         if default:
             r_serv_db.hset('user_metadata:{}'.format(username_id), 'change_passwd', True)
@@ -121,7 +153,7 @@ def create_user_db(username_id , password, default=False, role=None, update=Fals
                     r_serv_db.sadd('user_role:{}'.format(role_to_add), username_id)
                 r_serv_db.hset('user_metadata:{}'.format(username_id), 'role', role)
 
-    r_serv_db.hset('user:all', username_id, password_hash)
+        r_serv_db.hset('user:all', username_id, password_hash)
 
 def edit_user_db(user_id, role, password=None):
     if password:

var/www/modules/Tags/Flask_Tags.py

@@ -7,7 +7,7 @@
 import redis
 from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for
 
-from Role_Manager import login_admin, login_analyst
+from Role_Manager import login_admin, login_analyst, login_read_only
 from flask_login import login_required
 
 import json
@@ -125,7 +125,7 @@ def get_last_seen_from_tags_list(list_tags):
 
 @Tags.route("/tags/", methods=['GET'])
 @login_required
-@login_analyst
+@login_read_only
 def Tags_page():
     date_from = request.args.get('date_from')
     date_to = request.args.get('date_to')
@@ -260,7 +260,7 @@ def Tags_page():
 
 @Tags.route("/Tags/get_all_tags")
 @login_required
-@login_analyst
+@login_read_only
 def get_all_tags():
 
     all_tags = r_serv_tags.smembers('list_tags')
@@ -284,7 +284,7 @@ def get_all_tags():
 
 @Tags.route("/Tags/get_all_tags_taxonomies")
 @login_required
-@login_analyst
+@login_read_only
 def get_all_tags_taxonomies():
 
     taxonomies = Taxonomies()
@@ -303,7 +303,7 @@ def get_all_tags_taxonomies():
 
 @Tags.route("/Tags/get_all_tags_galaxies")
 @login_required
-@login_analyst
+@login_read_only
 def get_all_tags_galaxy():
 
     active_galaxies = r_serv_tags.smembers('active_galaxies')
@@ -318,7 +318,7 @@ def get_all_tags_galaxy():
 
 @Tags.route("/Tags/get_tags_taxonomie")
 @login_required
-@login_analyst
+@login_read_only
 def get_tags_taxonomie():
 
     taxonomie = request.args.get('taxonomie')
@@ -346,7 +346,7 @@ def get_tags_taxonomie():
 
 @Tags.route("/Tags/get_tags_galaxy")
 @login_required
-@login_analyst
+@login_read_only
 def get_tags_galaxy():
 
     galaxy = request.args.get('galaxy')
@@ -467,7 +467,7 @@ def add_item_tags():
 
 @Tags.route("/Tags/taxonomies")
 @login_required
-@login_analyst
+@login_read_only
 def taxonomies():
 
     active_taxonomies = r_serv_tags.smembers('active_taxonomies')
@@ -642,7 +642,7 @@ def edit_taxonomie_tag():
 
 @Tags.route("/Tags/galaxies")
 @login_required
-@login_analyst
+@login_read_only
 def galaxies():
 
     active_galaxies = r_serv_tags.smembers('active_galaxies')
@@ -901,7 +901,7 @@ def edit_galaxy_tag():
 
 @Tags.route("/Tags/tag_galaxy_info")
 @login_required
-@login_analyst
+@login_read_only
 def tag_galaxy_info():
 
     galaxy = request.args.get('galaxy')

var/www/modules/dashboard/Flask_dashboard.py

@@ -14,7 +14,7 @@ from Date import Date
 
 from flask import Flask, render_template, jsonify, request, Blueprint, url_for
 
-from Role_Manager import login_admin, login_analyst
+from Role_Manager import login_admin, login_analyst, login_read_only
 from flask_login import login_required
 
 # ============ VARIABLES ============
@@ -113,13 +113,13 @@ def datetime_from_utc_to_local(utc_str):
 
 @dashboard.route("/_logs")
 @login_required
-@login_analyst
+@login_read_only
 def logs():
     return flask.Response(event_stream(), mimetype="text/event-stream")
 
 @dashboard.route("/_get_last_logs_json")
 @login_required
-@login_analyst
+@login_read_only
 def get_last_logs_json():
     date = datetime.datetime.now().strftime("%Y%m%d")
 
@@ -162,14 +162,14 @@ def get_last_logs_json():
 
 @dashboard.route("/_stuff", methods=['GET'])
 @login_required
-@login_analyst
+@login_read_only
 def stuff():
     return jsonify(row1=get_queues(r_serv))
 
 
 @dashboard.route("/")
 @login_required
-@login_analyst
+@login_read_only
 def index():
     default_minute = config_loader.get_config_str("Flask", "minute_processed_paste")
     threshold_stucked_module = config_loader.get_config_int("Module_ModuleInformation", "threshold_stucked_module")

var/www/modules/hashDecoded/Flask_hashDecoded.py

@@ -17,7 +17,7 @@ from hashlib import sha256
 
 import requests
 from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, send_file
-from Role_Manager import login_admin, login_analyst
+from Role_Manager import login_admin, login_analyst, login_read_only
 from flask_login import login_required
 
 # ============ VARIABLES ============
@@ -475,7 +475,7 @@ def correlation_graph_node_json(correlation_type, type_id, key_id):
 # ============= ROUTES ==============
 @hashDecoded.route("/hashDecoded/all_hash_search", methods=['POST'])
 @login_required
-@login_analyst
+@login_read_only
 def all_hash_search():
     date_from = request.form.get('date_from')
     date_to = request.form.get('date_to')
@@ -486,7 +486,7 @@ def all_hash_search():
 
 @hashDecoded.route("/hashDecoded/", methods=['GET'])
 @login_required
-@login_analyst
+@login_read_only
 def hashDecoded_page():
     date_from = request.args.get('date_from')
     date_to = request.args.get('date_to')
@@ -605,7 +605,7 @@ def hashDecoded_page():
 
 @hashDecoded.route('/hashDecoded/hash_by_type')
 @login_required
-@login_analyst
+@login_read_only
 def hash_by_type():
     type = request.args.get('type')
     type = 'text/plain'
@@ -614,7 +614,7 @@ def hash_by_type():
 
 @hashDecoded.route('/hashDecoded/hash_hash')
 @login_required
-@login_analyst
+@login_read_only
 def hash_hash():
     hash = request.args.get('hash')
     return render_template('hash_hash.html')
@@ -677,7 +677,7 @@ def hash_hash():
 
 @hashDecoded.route('/hashDecoded/downloadHash')
 @login_required
-@login_analyst
+@login_read_only
 def downloadHash():
     hash = request.args.get('hash')
     # sanitize hash
@@ -715,7 +715,7 @@ def downloadHash():
 
 @hashDecoded.route('/hashDecoded/hash_by_type_json')
 @login_required
-@login_analyst
+@login_read_only
 def hash_by_type_json():
     type = request.args.get('type')
 
@@ -750,7 +750,7 @@ def hash_by_type_json():
 
 @hashDecoded.route('/hashDecoded/decoder_type_json')
 @login_required
-@login_analyst
+@login_read_only
 def decoder_type_json():
     date_from = request.args.get('date_from')
     date_to = request.args.get('date_to')
@@ -807,7 +807,7 @@ def decoder_type_json():
 
 @hashDecoded.route('/hashDecoded/top5_type_json')
 @login_required
-@login_analyst
+@login_read_only
 def top5_type_json():
     date_from = request.args.get('date_from')
     date_to = request.args.get('date_to')
@@ -867,7 +867,7 @@ def top5_type_json():
 
 @hashDecoded.route('/hashDecoded/daily_type_json')
 @login_required
-@login_analyst
+@login_read_only
 def daily_type_json():
     date = request.args.get('date')
 
@@ -888,7 +888,7 @@ def daily_type_json():
 
 @hashDecoded.route('/hashDecoded/range_type_json')
 @login_required
-@login_analyst
+@login_read_only
 def range_type_json():
     date_from = request.args.get('date_from')
     date_to = request.args.get('date_to')
@@ -946,7 +946,7 @@ def range_type_json():
 
 @hashDecoded.route('/hashDecoded/hash_graph_line_json')
 @login_required
-@login_analyst
+@login_read_only
 def hash_graph_line_json():
     hash = request.args.get('hash')
     date_from = request.args.get('date_from')
@@ -977,7 +977,7 @@ def hash_graph_line_json():
 
 @hashDecoded.route('/hashDecoded/hash_graph_node_json')
 @login_required
-@login_analyst
+@login_read_only
 def hash_graph_node_json():
     hash = request.args.get('hash')
 
@@ -1046,7 +1046,7 @@ def hash_graph_node_json():
 
 @hashDecoded.route('/hashDecoded/hash_types')
 @login_required
-@login_analyst
+@login_read_only
 def hash_types():
     date_from = 20180701
     date_to = 20180706
@@ -1118,7 +1118,7 @@ def update_vt_result():
 
 @hashDecoded.route('/decoded/pgp_by_type_json') ## TODO: REFRACTOR
 @login_required
-@login_analyst
+@login_read_only
 def pgp_by_type_json():
     type_id = request.args.get('type_id')
     date_from = request.args.get('date_from')
@@ -1164,7 +1164,7 @@ def pgp_by_type_json():
 ############################ Correlation ############################
 @hashDecoded.route("/correlation/pgpdump", methods=['GET'])
 @login_required
-@login_analyst
+@login_read_only
 def pgpdump_page():
     date_from = request.args.get('date_from')
     date_to = request.args.get('date_to')
@@ -1176,7 +1176,7 @@ def pgpdump_page():
 
 @hashDecoded.route("/correlation/cryptocurrency", methods=['GET'])
 @login_required
-@login_analyst
+@login_read_only
 def cryptocurrency_page():
     date_from = request.args.get('date_from')
     date_to = request.args.get('date_to')
@@ -1188,7 +1188,7 @@ def cryptocurrency_page():
 
 @hashDecoded.route("/correlation/all_pgpdump_search", methods=['POST'])
 @login_required
-@login_analyst
+@login_read_only
 def all_pgpdump_search():
     date_from = request.form.get('date_from')
     date_to = request.form.get('date_to')
@@ -1198,7 +1198,7 @@ def all_pgpdump_search():
 
 @hashDecoded.route("/correlation/all_cryptocurrency_search", methods=['POST'])
 @login_required
-@login_analyst
+@login_read_only
 def all_cryptocurrency_search():
     date_from = request.form.get('date_from')
     date_to = request.form.get('date_to')
@@ -1225,7 +1225,7 @@ def all_cryptocurrency_search():
 
 @hashDecoded.route('/correlation/cryptocurrency_range_type_json')
 @login_required
-@login_analyst
+@login_read_only
 def cryptocurrency_range_type_json():
     date_from = request.args.get('date_from')
     date_to = request.args.get('date_to')
@@ -1233,7 +1233,7 @@ def cryptocurrency_range_type_json():
 
 @hashDecoded.route('/correlation/pgpdump_range_type_json')
 @login_required
-@login_analyst
+@login_read_only
 def pgpdump_range_type_json():
     date_from = request.args.get('date_from')
     date_to = request.args.get('date_to')
@@ -1241,7 +1241,7 @@ def pgpdump_range_type_json():
 
 @hashDecoded.route('/correlation/pgpdump_graph_node_json')
 @login_required
-@login_analyst
+@login_read_only
 def pgpdump_graph_node_json():
     type_id = request.args.get('type_id')
     key_id = request.args.get('key_id')
@@ -1250,7 +1250,7 @@ def pgpdump_graph_node_json():
 # # TODO: REFRACTOR
 @hashDecoded.route('/correlation/cryptocurrency_graph_node_json')
 @login_required
-@login_analyst
+@login_read_only
 def cryptocurrency_graph_node_json():
     type_id = request.args.get('type_id')
     key_id = request.args.get('key_id')
@@ -1259,7 +1259,7 @@ def cryptocurrency_graph_node_json():
 # # TODO: REFRACTOR
 @hashDecoded.route('/correlation/pgpdump_graph_line_json')
 @login_required
-@login_analyst
+@login_read_only
 def pgpdump_graph_line_json():
     type_id = request.args.get('type_id')
     key_id = request.args.get('key_id')
@@ -1293,7 +1293,7 @@ def correlation_graph_line_json(correlation_type, type_id, key_id, date_from, da
 
 @hashDecoded.route('/correlation/cryptocurrency_graph_line_json')
 @login_required
-@login_analyst
+@login_read_only
 def cryptocurrency_graph_line_json():
     type_id = request.args.get('type_id')
     key_id = request.args.get('key_id')

var/www/modules/hiddenServices/Flask_hiddenServices.py

@@ -13,7 +13,7 @@ import json
 from pyfaup.faup import Faup
 from flask import Flask, render_template, jsonify, request, send_file, Blueprint, redirect, url_for
 
-from Role_Manager import login_admin, login_analyst, no_cache
+from Role_Manager import login_admin, login_analyst, login_read_only, no_cache
 from flask_login import login_required
 
 from Date import Date
@@ -242,7 +242,7 @@ def delete_auto_crawler(url):
 
 @hiddenServices.route("/crawlers/", methods=['GET'])
 @login_required
-@login_analyst
+@login_read_only
 def dashboard():
     crawler_metadata_onion = get_crawler_splash_status('onion')
     crawler_metadata_regular = get_crawler_splash_status('regular')
@@ -259,13 +259,13 @@ def dashboard():
 
 @hiddenServices.route("/crawlers/manual", methods=['GET'])
 @login_required
-@login_analyst
+@login_read_only
 def manual():
     return render_template("Crawler_Splash_manual.html", crawler_enabled=crawler_enabled)
 
 @hiddenServices.route("/crawlers/crawler_splash_onion", methods=['GET'])
 @login_required
-@login_analyst
+@login_read_only
 def crawler_splash_onion():
     type = 'onion'
     last_onions = get_last_domains_crawled(type)
@@ -284,7 +284,7 @@ def crawler_splash_onion():
 
 @hiddenServices.route("/crawlers/Crawler_Splash_last_by_type", methods=['GET'])
 @login_required
-@login_analyst
+@login_read_only
 def Crawler_Splash_last_by_type():
     type = request.args.get('type')
     # verify user input
@@ -309,7 +309,7 @@ def Crawler_Splash_last_by_type():
 
 @hiddenServices.route("/crawlers/blacklisted_domains", methods=['GET'])
 @login_required
-@login_analyst
+@login_read_only
 def blacklisted_domains():
     blacklist_domain = request.args.get('blacklist_domain')
     unblacklist_domain = request.args.get('unblacklist_domain')
@@ -479,7 +479,7 @@ def create_spider_splash():
 
 @hiddenServices.route("/crawlers/auto_crawler", methods=['GET'])
 @login_required
-@login_analyst
+@login_read_only
 def auto_crawler():
     nb_element_to_display = 100
     try:
@@ -544,7 +544,7 @@ def remove_auto_crawler():
 
 @hiddenServices.route("/crawlers/crawler_dashboard_json", methods=['GET'])
 @login_required
-@login_analyst
+@login_read_only
 def crawler_dashboard_json():
 
     crawler_metadata_onion = get_crawler_splash_status('onion')
@@ -562,7 +562,7 @@ def crawler_dashboard_json():
 # # TODO: refractor
 @hiddenServices.route("/hiddenServices/last_crawled_domains_with_stats_json", methods=['GET'])
 @login_required
-@login_analyst
+@login_read_only
 def last_crawled_domains_with_stats_json():
     last_onions = r_serv_onion.lrange('last_onion', 0 ,-1)
     list_onion = []
@@ -613,7 +613,7 @@ def last_crawled_domains_with_stats_json():
 
 @hiddenServices.route("/hiddenServices/get_onions_by_daterange", methods=['POST'])
 @login_required
-@login_analyst
+@login_read_only
 def get_onions_by_daterange():
     date_from = request.form.get('date_from')
     date_to = request.form.get('date_to')
@@ -626,7 +626,7 @@ def get_onions_by_daterange():
 
 @hiddenServices.route("/hiddenServices/show_domains_by_daterange", methods=['GET'])
 @login_required
-@login_analyst
+@login_read_only
 def show_domains_by_daterange():
     date_from = request.args.get('date_from')
     date_to = request.args.get('date_to')
@@ -732,7 +732,7 @@ def show_domains_by_daterange():
 
 @hiddenServices.route("/crawlers/download_domain", methods=['GET'])
 @login_required
-@login_analyst
+@login_read_only
 @no_cache
 def download_domain():
     domain = request.args.get('domain')
@@ -798,7 +798,7 @@ def onion_son():
 # ============= JSON ==============
 @hiddenServices.route("/hiddenServices/domain_crawled_7days_json", methods=['GET'])
 @login_required
-@login_analyst
+@login_read_only
 def domain_crawled_7days_json():
     type = 'onion'
         ## TODO: # FIXME: 404 error
@@ -818,7 +818,7 @@ def domain_crawled_7days_json():
 
 @hiddenServices.route('/hiddenServices/domain_crawled_by_type_json')
 @login_required
-@login_analyst
+@login_read_only
 def domain_crawled_by_type_json():
     current_date = request.args.get('date')
     type = request.args.get('type')

var/www/modules/hunter/Flask_hunter.py

@@ -11,7 +11,7 @@ import calendar
 import flask
 from flask import Flask, render_template, jsonify, request, Blueprint, url_for, redirect, Response, escape
 
-from Role_Manager import login_admin, login_analyst
+from Role_Manager import login_admin, login_analyst, login_read_only
 from flask_login import login_required, current_user
 
 import re
@@ -41,7 +41,7 @@ hunter = Blueprint('hunter', __name__, template_folder='templates')
 
 @hunter.route("/trackers")
 @login_required
-@login_analyst
+@login_read_only
 def tracked_menu():
     user_id = current_user.get_id()
     user_term = Term.get_all_user_tracked_terms(user_id)
@@ -50,7 +50,7 @@ def tracked_menu():
 
 @hunter.route("/trackers/word")
 @login_required
-@login_analyst
+@login_read_only
 def tracked_menu_word():
     filter_type = 'word'
     user_id = current_user.get_id()
@@ -60,7 +60,7 @@ def tracked_menu_word():
 
 @hunter.route("/trackers/set")
 @login_required
-@login_analyst
+@login_read_only
 def tracked_menu_set():
     filter_type = 'set'
     user_id = current_user.get_id()
@@ -70,7 +70,7 @@ def tracked_menu_set():
 
 @hunter.route("/trackers/regex")
 @login_required
-@login_analyst
+@login_read_only
 def tracked_menu_regex():
     filter_type = 'regex'
     user_id = current_user.get_id()
@@ -113,7 +113,7 @@ def add_tracked_menu():
 
 @hunter.route("/tracker/show_tracker")
 @login_required
-@login_analyst
+@login_read_only
 def show_tracker():
     user_id = current_user.get_id()
     term_uuid = request.args.get('uuid', None)
@@ -207,7 +207,7 @@ def delete_tracker():
 
 @hunter.route("/tracker/get_json_tracker_stats", methods=['GET'])
 @login_required
-@login_analyst
+@login_read_only
 def get_json_tracker_stats():
     date_from = request.args.get('date_from')
     date_to = request.args.get('date_to')

var/www/modules/rawSkeleton/Flask_rawSkeleton.py

@@ -7,7 +7,7 @@
 import redis
 from flask import Flask, render_template, jsonify, request, Blueprint
 
-from Role_Manager import login_admin, login_analyst
+from Role_Manager import login_admin, login_analyst, login_read_only
 from flask_login import login_required
 
 # ============ VARIABLES ============
@@ -25,7 +25,7 @@ def one():
 
 @rawSkeleton.route("/rawSkeleton/", methods=['GET'])
 @login_required
-@login_analyst
+@login_read_only
 def skeleton_page():
     return render_template("rawSkeleton.html")
 

var/www/modules/restApi/Flask_restApi.py

@@ -172,14 +172,14 @@ def one():
 #
 # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
 @restApi.route("api/v1/get/item", methods=['POST'])
-@token_required('analyst')
+@token_required('user')
 def get_item_id():
     data = request.get_json()
     res = Item.get_item(data)
     return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
 
 @restApi.route("api/v1/get/item/default", methods=['POST'])
-@token_required('analyst')
+@token_required('user')
 def get_item_id_basic():
 
     data = request.get_json()
@@ -202,7 +202,7 @@ def get_item_id_basic():
 #
 # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
 @restApi.route("api/v1/get/item/tag", methods=['POST'])
-@token_required('analyst')
+@token_required('user')
 def get_item_tag():
 
     data = request.get_json()
@@ -283,7 +283,7 @@ def delete_item_tags():
 #
 # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
 @restApi.route("api/v1/get/item/content", methods=['POST'])
-@token_required('analyst')
+@token_required('user')
 def get_item_content():
 
     data = request.get_json()
@@ -298,7 +298,7 @@ def get_item_content():
 # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
 
 @restApi.route("api/v1/get/tag/metadata", methods=['POST'])
-@token_required('analyst')
+@token_required('user')
 def get_tag_metadata():
     data = request.get_json()
     tag = data.get('tag', None)
@@ -308,7 +308,7 @@ def get_tag_metadata():
     return Response(json.dumps(metadata, indent=2, sort_keys=True), mimetype='application/json'), 200
 
 @restApi.route("api/v1/get/tag/all", methods=['GET'])
-@token_required('analyst')
+@token_required('user')
 def get_all_tags():
     res = {'tags': Tag.get_all_tags()}
     return Response(json.dumps(res, indent=2, sort_keys=True), mimetype='application/json'), 200
@@ -335,7 +335,7 @@ def delete_tracker_term():
     return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
 
 @restApi.route("api/v1/get/tracker/item", methods=['POST'])
-@token_required('analyst')
+@token_required('user')
 def get_tracker_term_item():
     data = request.get_json()
     user_token = get_auth_from_header()
@@ -348,7 +348,7 @@ def get_tracker_term_item():
 # # # # # # # # # # # #        CRYPTOCURRENCY       # # # # # # # # # # # # # #
 # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
 @restApi.route("api/v1/get/cryptocurrency/bitcoin/metadata", methods=['POST'])
-@token_required('analyst')
+@token_required('user')
 def get_cryptocurrency_bitcoin_metadata():
     data = request.get_json()
     crypto_address = data.get('bitcoin', None)
@@ -357,7 +357,7 @@ def get_cryptocurrency_bitcoin_metadata():
     return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
 
 @restApi.route("api/v1/get/cryptocurrency/bitcoin/item", methods=['POST'])
-@token_required('analyst')
+@token_required('user')
 def get_cryptocurrency_bitcoin_item():
     data = request.get_json()
     bitcoin_address = data.get('bitcoin', None)
@@ -369,7 +369,7 @@ def get_cryptocurrency_bitcoin_item():
 # # # # # # # # # # # # # # #       PGP       # # # # # # # # # # # # # # # # #
 # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
 @restApi.route("api/v1/get/pgp/key/metadata", methods=['POST'])
-@token_required('analyst')
+@token_required('user')
 def get_pgp_key_metadata():
     data = request.get_json()
     pgp_field = data.get('key', None)
@@ -378,7 +378,7 @@ def get_pgp_key_metadata():
     return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
 
 @restApi.route("api/v1/get/pgp/mail/metadata", methods=['POST'])
-@token_required('analyst')
+@token_required('user')
 def get_pgp_mail_metadata():
     data = request.get_json()
     pgp_field = data.get('mail', None)
@@ -387,7 +387,7 @@ def get_pgp_mail_metadata():
     return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
 
 @restApi.route("api/v1/get/pgp/name/metadata", methods=['POST'])
-@token_required('analyst')
+@token_required('user')
 def get_pgp_name_metadata():
     data = request.get_json()
     pgp_field = data.get('name', None)
@@ -396,7 +396,7 @@ def get_pgp_name_metadata():
     return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
 
 @restApi.route("api/v1/get/pgp/key/item", methods=['POST'])
-@token_required('analyst')
+@token_required('user')
 def get_pgp_key_item():
     data = request.get_json()
     pgp_field = data.get('key', None)
@@ -405,7 +405,7 @@ def get_pgp_key_item():
     return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
 
 @restApi.route("api/v1/get/pgp/mail/item", methods=['POST'])
-@token_required('analyst')
+@token_required('user')
 def get_pgp_mail_item():
     data = request.get_json()
     pgp_mail = data.get('mail', None)
@@ -414,7 +414,7 @@ def get_pgp_mail_item():
     return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
 
 @restApi.route("api/v1/get/pgp/name/item", methods=['POST'])
-@token_required('analyst')
+@token_required('user')
 def get_pgp_name_item():
     data = request.get_json()
     pgp_name = data.get('name', None)

var/www/modules/sentiment/Flask_sentiment.py

@@ -11,7 +11,7 @@ from Date import Date
 import flask
 from flask import Flask, render_template, jsonify, request, Blueprint
 
-from Role_Manager import login_admin, login_analyst
+from Role_Manager import login_admin, login_analyst, login_read_only
 from flask_login import login_required
 
 import Paste
@@ -42,14 +42,14 @@ def get_date_range(num_day):
 
 @sentiments.route("/sentiment_analysis_trending/")
 @login_required
-@login_analyst
+@login_read_only
 def sentiment_analysis_trending():
     return render_template("sentiment_analysis_trending.html")
 
 
 @sentiments.route("/sentiment_analysis_getplotdata/", methods=['GET'])
 @login_required
-@login_analyst
+@login_read_only
 def sentiment_analysis_getplotdata():
     # Get the top providers based on number of pastes
     oneHour = 60*60
@@ -101,7 +101,7 @@ def sentiment_analysis_getplotdata():
 
 @sentiments.route("/sentiment_analysis_plot_tool/")
 @login_required
-@login_analyst
+@login_read_only
 def sentiment_analysis_plot_tool():
     return render_template("sentiment_analysis_plot_tool.html")
 
@@ -109,7 +109,7 @@ def sentiment_analysis_plot_tool():
 
 @sentiments.route("/sentiment_analysis_plot_tool_getdata/", methods=['GET'])
 @login_required
-@login_analyst
+@login_read_only
 def sentiment_analysis_plot_tool_getdata():
     getProviders = request.args.get('getProviders')
 

var/www/modules/settings/Flask_settings.py

@@ -7,7 +7,7 @@
 from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for
 from flask_login import login_required, current_user
 
-from Role_Manager import login_admin, login_analyst
+from Role_Manager import login_admin, login_analyst, login_user, login_read_only
 from Role_Manager import create_user_db, edit_user_db, delete_user_db, check_password_strength, generate_new_token, gen_password
 
 import json
@@ -103,7 +103,7 @@ def get_all_roles():
 
 @settings.route("/settings/", methods=['GET'])
 @login_required
-@login_analyst
+@login_read_only
 def settings_page():
     git_metadata = get_git_metadata()
     current_version = r_serv_db.get('ail:version')
@@ -117,7 +117,7 @@ def settings_page():
 
 @settings.route("/settings/edit_profile", methods=['GET'])
 @login_required
-@login_analyst
+@login_read_only
 def edit_profile():
     user_metadata = get_user_metadata(current_user.get_id())
     admin_level = current_user.is_in_role('admin')
@@ -126,7 +126,7 @@ def edit_profile():
 
 @settings.route("/settings/new_token", methods=['GET'])
 @login_required
-@login_analyst
+@login_user
 def new_token():
     generate_new_token(current_user.get_id())
     return redirect(url_for('settings.edit_profile'))
@@ -233,7 +233,7 @@ def delete_user():
 
 @settings.route("/settings/get_background_update_stats_json", methods=['GET'])
 @login_required
-@login_analyst
+@login_read_only
 def get_background_update_stats_json():
     # handle :end, error
     update_stats = {}

var/www/modules/showpaste/Flask_showpaste.py

@@ -10,7 +10,7 @@ import os
 import flask
 from flask import Flask, render_template, jsonify, request, Blueprint, make_response, Response, send_from_directory, redirect, url_for
 
-from Role_Manager import login_admin, login_analyst, no_cache
+from Role_Manager import login_admin, login_analyst, login_read_only, no_cache
 from flask_login import login_required
 
 import difflib
@@ -384,21 +384,21 @@ def show_item_min(requested_path , content_range=0):
 
 @showsavedpastes.route("/showsavedpaste/") #completely shows the paste in a new tab
 @login_required
-@login_analyst
+@login_read_only
 def showsavedpaste():
     requested_path = request.args.get('paste', '')
     return showpaste(0, requested_path)
 
 @showsavedpastes.route("/showsaveditem_min/") #completely shows the paste in a new tab
 @login_required
-@login_analyst
+@login_read_only
 def showsaveditem_min():
     requested_path = request.args.get('paste', '')
     return show_item_min(requested_path)
 
 @showsavedpastes.route("/showsavedrawpaste/") #shows raw
 @login_required
-@login_analyst
+@login_read_only
 def showsavedrawpaste():
     requested_path = request.args.get('paste', '')
     paste = Paste.Paste(requested_path)
@@ -407,7 +407,7 @@ def showsavedrawpaste():
 
 @showsavedpastes.route("/showpreviewpaste/")
 @login_required
-@login_analyst
+@login_read_only
 def showpreviewpaste():
     num = request.args.get('num', '')
     requested_path = request.args.get('paste', '')
@@ -416,7 +416,7 @@ def showpreviewpaste():
 
 @showsavedpastes.route("/getmoredata/")
 @login_required
-@login_analyst
+@login_read_only
 def getmoredata():
     requested_path = request.args.get('paste', '')
     paste = Paste.Paste(requested_path)
@@ -444,7 +444,7 @@ def showDiff():
 
 @showsavedpastes.route('/screenshot/<path:filename>')
 @login_required
-@login_analyst
+@login_read_only
 @no_cache
 def screenshot(filename):
     return send_from_directory(SCREENSHOT_FOLDER, filename+'.png', as_attachment=True)

var/www/modules/terms/Flask_terms.py

@@ -13,7 +13,7 @@ import calendar
 import flask
 from flask import Flask, render_template, jsonify, request, Blueprint, url_for, redirect, Response
 
-from Role_Manager import login_admin, login_analyst
+from Role_Manager import login_admin, login_analyst, login_user_no_api, login_read_only
 from flask_login import login_required, current_user
 
 import re
@@ -153,7 +153,7 @@ def save_tag_to_auto_push(list_tag):
 
 @terms.route("/terms_plot_tool/")
 @login_required
-@login_analyst
+@login_read_only
 def terms_plot_tool():
     term =  request.args.get('term')
     if term is not None:
@@ -164,7 +164,7 @@ def terms_plot_tool():
 
 @terms.route("/terms_plot_tool_data/")
 @login_required
-@login_analyst
+@login_read_only
 def terms_plot_tool_data():
     oneDay = 60*60*24
     range_start =  datetime.datetime.utcfromtimestamp(int(float(request.args.get('range_start')))) if request.args.get('range_start') is not None else 0;
@@ -196,7 +196,7 @@ def terms_plot_tool_data():
 
 @terms.route("/terms_plot_top/")
 @login_required
-@login_analyst
+@login_read_only
 def terms_plot_top():
     per_paste = request.args.get('per_paste')
     per_paste = per_paste if per_paste is not None else 1
@@ -205,7 +205,7 @@ def terms_plot_top():
 
 @terms.route("/terms_plot_top_data/")
 @login_required
-@login_analyst
+@login_read_only
 def terms_plot_top_data():
     oneDay = 60*60*24
     today = datetime.datetime.now()
@@ -253,13 +253,13 @@ def terms_plot_top_data():
 
 @terms.route("/credentials_tracker/")
 @login_required
-@login_analyst
+@login_read_only
 def credentials_tracker():
     return render_template("credentials_tracker.html")
 
 @terms.route("/credentials_management_query_paste/", methods=['GET', 'POST'])
 @login_required
-@login_analyst
+@login_user_no_api
 def credentials_management_query_paste():
     cred =  request.args.get('cred')
     allPath = request.json['allPath']
@@ -284,7 +284,7 @@ def credentials_management_query_paste():
 
 @terms.route("/credentials_management_action/", methods=['GET'])
 @login_required
-@login_analyst
+@login_user_no_api
 def cred_management_action():
 
     supplied =  request.args.get('term')

var/www/modules/trendingcharts/Flask_trendingcharts.py

@@ -10,7 +10,7 @@ from Date import Date
 import flask
 from flask import Flask, render_template, jsonify, request, Blueprint
 
-from Role_Manager import login_admin, login_analyst
+from Role_Manager import login_admin, login_analyst, login_read_only
 from flask_login import login_required
 
 # ============ VARIABLES ============
@@ -40,7 +40,7 @@ def get_date_range(num_day):
 
 @trendings.route("/_progressionCharts", methods=['GET'])
 @login_required
-@login_analyst
+@login_read_only
 def progressionCharts():
     attribute_name = request.args.get('attributeName')
     trending_name = request.args.get('trendingName')
@@ -67,7 +67,7 @@ def progressionCharts():
 
 @trendings.route("/wordstrending/")
 @login_required
-@login_analyst
+@login_read_only
 def wordstrending():
     default_display = config_loader.get_config_str("Flask", "default_display")
     return render_template("Wordstrending.html", default_display = default_display)
@@ -75,7 +75,7 @@ def wordstrending():
 
 @trendings.route("/protocolstrending/")
 @login_required
-@login_analyst
+@login_read_only
 def protocolstrending():
     default_display = config_loader.get_config_str("Flask", "default_display")
     return render_template("Protocolstrending.html", default_display = default_display)
@@ -83,7 +83,7 @@ def protocolstrending():
 
 @trendings.route("/trending/")
 @login_required
-@login_analyst
+@login_read_only
 def trending():
     default_display = config_loader.get_config_str("Flask", "default_display")
     return render_template("Trending.html", default_display = default_display)

var/www/modules/trendingmodules/Flask_trendingmodules.py

@@ -10,7 +10,7 @@ from Date import Date
 import flask
 from flask import Flask, render_template, jsonify, request, Blueprint
 
-from Role_Manager import login_admin, login_analyst
+from Role_Manager import login_admin, login_analyst, login_read_only
 from flask_login import login_required
 
 # ============ VARIABLES ============
@@ -52,7 +52,7 @@ def get_date_range(num_day):
 
 @trendingmodules.route("/_moduleCharts", methods=['GET'])
 @login_required
-@login_analyst
+@login_read_only
 def modulesCharts():
     keyword_name = request.args.get('keywordName')
     module_name = request.args.get('moduleName')
@@ -80,7 +80,7 @@ def modulesCharts():
 
 @trendingmodules.route("/_providersChart", methods=['GET'])
 @login_required
-@login_analyst
+@login_read_only
 def providersChart():
     keyword_name = request.args.get('keywordName')
     module_name = request.args.get('moduleName')
@@ -128,7 +128,7 @@ def providersChart():
 
 @trendingmodules.route("/moduletrending/")
 @login_required
-@login_analyst
+@login_read_only
 def moduletrending():
     return render_template("Moduletrending.html")