chg: [restapi] add rest api authentification + create default user

OVERVIEW.md

@@ -56,6 +56,10 @@ Redis and ARDB overview
 | Key | Field | Value |
 | ------ | ------ | ------ |
 | user:all | **user id** | **password hash**  |
+| | | |
+| user:tokens | **token** | **user id** |
+| | | |
+| user_metadata:**user id** | **user token** | **token**  |
 
 | Key | Value |
 | ------ | ------ |

var/www/create_default_user.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+import os
+import sys
+import redis
+import configparser
+
+import bcrypt
+import secrets
+
+# Import config
+sys.path.append('./modules/')
+
+def hashing_password(bytes_password):
+    hashed = bcrypt.hashpw(bytes_password, bcrypt.gensalt())
+    return hashed
+
+def create_user_db(username_id , password, default=False):
+    password = password.encode()
+    password_hash = hashing_password(password)
+    r_serv_db.hset('user:all', username_id, password_hash)
+    if default:
+        r_serv_db.set('user:request_password_change', username_id)
+
+
+if __name__ == "__main__":
+    configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg')
+    if not os.path.exists(configfile):
+        raise Exception('Unable to find the configuration file. \
+                        Did you set environment variables? \
+                        Or activate the virtualenv.')
+
+    cfg = configparser.ConfigParser()
+    cfg.read(configfile)
+
+    r_serv_db = redis.StrictRedis(
+        host=cfg.get("ARDB_DB", "host"),
+        port=cfg.getint("ARDB_DB", "port"),
+        db=cfg.getint("ARDB_DB", "db"),
+        decode_responses=True)
+
+    username = 'admin@admin.test'
+    # # TODO: create random password
+    password = 'admin'
+    create_user_db(username, password, default=True)
+
+    # create user token
+    token = secrets.token_urlsafe(41)
+    r_serv_db.hset('user:tokens', token, username)
+
+    print('new user created: {}'.format(username))
+    print('password: {}'.format(password))

var/www/modules/Tags/Flask_Tags.py

@@ -5,7 +5,7 @@
     Flask functions and routes for the trending modules page
 '''
 import redis
-from flask import Flask, render_template, jsonify, request, Blueprint, current_app, redirect, url_for
+from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for
 
 from Role_Manager import login_admin, login_analyst
 from flask_login import login_required

var/www/modules/restApi/Flask_restApi.py

@@ -0,0 +1,109 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+'''
+    Flask functions and routes for the rest api
+'''
+
+import os
+import re
+import sys
+import json
+import redis
+import datetime
+
+from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response
+from flask_login import login_required
+
+from functools import wraps
+
+# ============ VARIABLES ============
+import Flask_config
+
+app = Flask_config.app
+cfg = Flask_config.cfg
+baseUrl = Flask_config.baseUrl
+r_cache = Flask_config.r_cache
+r_serv_db = Flask_config.r_serv_db
+r_serv_onion = Flask_config.r_serv_onion
+r_serv_metadata = Flask_config.r_serv_metadata
+
+restApi = Blueprint('restApi', __name__, template_folder='templates')
+
+# ============ AUTH FUNCTIONS ============
+
+def check_token_format(strg, search=re.compile(r'[^a-zA-Z0-9_-]').search):
+    return not bool(search(strg))
+
+def verify_token(token):
+    if len(token) != 55:
+        return False
+
+    if not check_token_format(token):
+        return False
+
+    if r_serv_db.hexists('user:tokens', token):
+        return True
+    else:
+        return False
+
+# ============ DECORATOR ============
+
+def token_required(funct):
+    @wraps(funct)
+    def api_token(*args, **kwargs):
+        data = authErrors()
+        if data:
+            return Response(json.dumps(data[0], indent=2, sort_keys=True), mimetype='application/json'), data[1]
+        else:
+            return funct(*args, **kwargs)
+    return api_token
+
+def get_auth_from_header():
+    token = request.headers.get('Authorization').replace(' ', '') # remove space
+    return token
+
+def authErrors():
+    # Check auth
+    if not request.headers.get('Authorization'):
+        return ({'status': 'error', 'reason': 'Authentication needed'}, 401)
+    token = get_auth_from_header()
+    data = None
+    # verify token format
+
+    try:
+        authenticated = False
+        if verify_token(token):
+            authenticated = True
+
+        if not authenticated:
+            data = ({'status': 'error', 'reason': 'Authentication failed'}, 401)
+    except Exception as e:
+        print(e)
+        data = ({'status': 'error', 'reason': 'Malformed Authentication String'}, 400)
+    if data:
+        return data
+    else:
+        return None
+
+# ============ FUNCTIONS ============
+
+def one():
+    return 1
+
+# ============= ROUTES ==============
+
+@restApi.route("/api", methods=['GET'])
+@login_required
+def api():
+    return 'api doc'
+
+@restApi.route("api/items", methods=['POST'])
+@token_required
+def items():
+    item = request.args.get('id')
+
+    return Response(json.dumps({'test': 2}), mimetype='application/json')
+
+# ========= REGISTRATION =========
+app.register_blueprint(restApi, url_prefix=baseUrl)

var/www/modules/restApi/templates/api_default.html

@@ -0,0 +1,222 @@
+<!DOCTYPE html>
+
+<html>
+<head>
+	<title>AIL-Framework</title>
+	<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png')}}">
+	<!-- Core CSS -->
+	<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
+
+	<!-- JS -->
+	<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
+
+
+</head>
+
+<body>
+
+	{% include 'nav_bar.html' %}
+
+	<div class="container-fluid">
+		<div class="row">
+
+			{% include 'crawler/menu_sidebar.html' %}
+
+			<div class="col-12 col-lg-10" id="core_content">
+
+				<div class="row">
+					<div class="col-xl-6">
+
+						<div class="card mt-1 mb-1">
+						  <div class="card-header text-white bg-dark">
+								<h5><a class="text-info" href="{{ url_for('hiddenServices.Crawler_Splash_last_by_type')}}?type=onion"><i class="fas fa-user-secret"></i> Onions Crawlers</a></h5>
+								<div class="row">
+									<div class="col-6">
+										<span class="badge badge-success" id="stat_onion_domain_up">{{ statDomains_onion['domains_up'] }}</span> UP
+										<span class="badge badge-danger ml-md-3" id="stat_onion_domain_down">{{ statDomains_onion['domains_down'] }}</span> DOWN
+									</div>
+									<div class="col-6">
+										<span class="badge badge-success" id="stat_onion_total">{{ statDomains_onion['total'] }}</span> Crawled
+										<span class="badge badge-warning ml-md-3" id="stat_onion_queue">{{ statDomains_onion['domains_queue'] }}</span> Queue
+									</div>
+								</div>
+							</div>
+						  <div class="card-body px-0 py-0 ">
+								<table class="table">
+									<tbody id="tbody_crawler_onion_info">
+										{% for crawler in crawler_metadata_onion %}
+											<tr>
+						      			<td>
+													<i class="fas fa-{%if crawler['status']%}check{%else%}times{%endif%}-circle" style="color:{%if crawler['status']%}Green{%else%}Red{%endif%};"></i> {{crawler['crawler_info']}}
+												</td>
+												<td>
+													{{crawler['crawling_domain']}}
+												</td>
+												<td style="color:{%if crawler['status']%}Green{%else%}Red{%endif%};">
+													{{crawler['status_info']}}
+												</td>
+											</tr>
+										{% endfor %}
+									</tbody>
+								</table>
+							</div>
+						</div>
+
+					</div>
+					<div class="col-xl-6">
+						<div class="card mt-1 mb-1">
+							<div class="card-header text-white bg-dark">
+								<h5><a class="text-info" href="{{ url_for('hiddenServices.Crawler_Splash_last_by_type')}}?type=regular"><i class="fab fa-html5"></i> Regular Crawlers</a></h5>
+								<div class="row">
+									<div class="col-6">
+										<span class="badge badge-success" id="stat_regular_domain_up">{{ statDomains_regular['domains_up'] }}</span> UP
+										<span class="badge badge-danger ml-md-3" id="stat_regular_domain_down">{{ statDomains_regular['domains_down'] }}</span> DOWN
+									</div>
+									<div class="col-6">
+										<span class="badge badge-success" id="stat_regular_total">{{ statDomains_regular['total'] }}</span> Crawled
+										<span class="badge badge-warning ml-md-3" id="stat_regular_queue">{{ statDomains_regular['domains_queue'] }}</span> Queue
+									</div>
+								</div>
+							</div>
+							<div class="card-body px-0 py-0 ">
+								<table class="table">
+									<tbody id="tbody_crawler_regular_info">
+										{% for crawler in crawler_metadata_regular %}
+											<tr>
+												<td>
+													<i class="fas fa-{%if crawler['status']%}check{%else%}times{%endif%}-circle" style="color:{%if crawler['status']%}Green{%else%}Red{%endif%};"></i> {{crawler['crawler_info']}}
+												</td>
+												<td>
+													{{crawler['crawling_domain']}}
+												</td>
+												<td style="color:{%if crawler['status']%}Green{%else%}Red{%endif%};">
+													{{crawler['status_info']}}
+												</td>
+											</tr>
+										{% endfor %}
+									</tbody>
+								</table>
+							</div>
+						</div>
+					</div>
+				</div>
+
+			</div>
+		</div>
+	</div>
+
+
+</body>
+
+<script>
+var to_refresh = false
+$(document).ready(function(){
+	$("#page-Crawler").addClass("active");
+	$("#nav_dashboard").addClass("active");
+	$( window ).focus(function() {
+			to_refresh = true
+			refresh_crawler_status();
+	});
+	$( window ).blur(function() {
+			to_refresh = false
+	});
+
+	to_refresh = true
+	refresh_crawler_status();
+});
+
+function toggle_sidebar(){
+	if($('#nav_menu').is(':visible')){
+		$('#nav_menu').hide();
+		$('#side_menu').removeClass('border-right')
+		$('#side_menu').removeClass('col-lg-2')
+		$('#core_content').removeClass('col-lg-10')
+	}else{
+		$('#nav_menu').show();
+		$('#side_menu').addClass('border-right')
+		$('#side_menu').addClass('col-lg-2')
+		$('#core_content').addClass('col-lg-10')
+	}
+}
+
+function refresh_crawler_status(){
+
+	$.getJSON("{{ url_for('hiddenServices.crawler_dashboard_json') }}",
+		function(data) {
+
+			$('#stat_onion_domain_up').text(data.statDomains_onion['domains_up']);
+			$('#stat_onion_domain_down').text(data.statDomains_onion['domains_down']);
+			$('#stat_onion_total').text(data.statDomains_onion['total']);
+			$('#stat_onion_queue').text(data.statDomains_onion['domains_queue']);
+
+			$('#stat_regular_domain_up').text(data.statDomains_regular['domains_up']);
+			$('#stat_regular_domain_down').text(data.statDomains_regular['domains_down']);
+			$('#stat_regular_total').text(data.statDomains_regular['total']);
+			$('#stat_regular_queue').text(data.statDomains_regular['domains_queue']);
+
+			if(data.crawler_metadata_onion.length!=0){
+				$("#tbody_crawler_onion_info").empty();
+				var tableRef = document.getElementById('tbody_crawler_onion_info');
+				for (var i = 0; i < data.crawler_metadata_onion.length; i++) {
+						var crawler = data.crawler_metadata_onion[i];
+						var newRow = tableRef.insertRow(tableRef.rows.length);
+						var text_color;
+						var icon;
+						if(crawler['status']){
+							text_color = 'Green';
+							icon = 'check';
+						} else {
+							text_color = 'Red';
+							icon = 'times';
+						}
+
+				    var newCell  = newRow.insertCell(0);
+						newCell.innerHTML = "<td><i class=\"fas fa-"+icon+"-circle\" style=\"color:"+text_color+";\"></i> "+crawler['crawler_info']+"</td>";
+
+						newCell  = newRow.insertCell(1);
+						newCell.innerHTML = "<td>"+crawler['crawling_domain']+"</td>";
+
+						newCell  = newRow.insertCell(2);
+						newCell.innerHTML = "<td><div style=\"color:"+text_color+";\">"+crawler['status_info']+"</div></td>";
+
+						//$("#panel_crawler").show();
+				}
+			}
+			if(data.crawler_metadata_regular.length!=0){
+				$("#tbody_crawler_regular_info").empty();
+				var tableRef = document.getElementById('tbody_crawler_regular_info');
+				for (var i = 0; i < data.crawler_metadata_regular.length; i++) {
+						var crawler = data.crawler_metadata_regular[i];
+						var newRow = tableRef.insertRow(tableRef.rows.length);
+						var text_color;
+						var icon;
+						if(crawler['status']){
+							text_color = 'Green';
+							icon = 'check';
+						} else {
+							text_color = 'Red';
+							icon = 'times';
+						}
+
+				    var newCell  = newRow.insertCell(0);
+						newCell.innerHTML = "<td><i class=\"fas fa-"+icon+"-circle\" style=\"color:"+text_color+";\"></i> "+crawler['crawler_info']+"</td>";
+
+						newCell  = newRow.insertCell(1);
+						newCell.innerHTML = "<td>"+crawler['crawling_domain']+"</td>";
+
+						newCell  = newRow.insertCell(2);
+						newCell.innerHTML = "<td><div style=\"color:"+text_color+";\">"+crawler['status_info']+"</div></td>";
+
+						//$("#panel_crawler").show();
+				}
+			}
+		}
+	);
+
+	if (to_refresh) {
+    setTimeout("refresh_crawler_status()", 10000);
+	}
+}
+</script>

var/www/modules/restApi/templates/header_api.html

@@ -0,0 +1 @@
+<li id='page-hiddenServices'><a href="{{ url_for('hiddenServices.dashboard') }}"><i class="fa fa-user-secret"></i> hidden Services </a></li>