mirror of https://github.com/CIRCL/AIL-framework
chg: [user_management 2.0] add update scripts + fix create_default_user
parent
821cf3cbea
commit
a9837c6e27
|
@ -20,6 +20,8 @@ indexdir/
|
|||
logs/
|
||||
old/
|
||||
|
||||
DEFAULT_PASSWORD
|
||||
|
||||
# Webstuff
|
||||
var/www/static/
|
||||
!var/www/static/css/dygraph_gallery.css
|
||||
|
|
|
@ -42,8 +42,6 @@ Redis and ARDB overview
|
|||
| | **background update name** |
|
||||
| | **...** |
|
||||
| | |
|
||||
| ail:update_date_v1.5 | **update date** |
|
||||
| | |
|
||||
| ail:update_error | **update message error** |
|
||||
| | |
|
||||
| ail:update_in_progress | **update version in progress** |
|
||||
|
@ -52,6 +50,10 @@ Redis and ARDB overview
|
|||
| ail:current_background_script | **name of the background script currently executed** |
|
||||
| ail:current_background_script_stat | **progress in % of the background script** |
|
||||
|
||||
| Hset Key | Field | Value |
|
||||
| ------ | ------ | ------ |
|
||||
| ail:update_date | **update tag** | **update date** |
|
||||
|
||||
##### User Management:
|
||||
| Hset Key | Field | Value |
|
||||
| ------ | ------ | ------ |
|
||||
|
|
|
@ -101,6 +101,7 @@ fi
|
|||
|
||||
pushd var/www/
|
||||
./update_thirdparty.sh
|
||||
python3 create_default_user.py
|
||||
popd
|
||||
|
||||
mkdir -p $AIL_HOME/PASTES
|
||||
|
|
|
@ -1,108 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
|
||||
echo "Currently unmaintained, continue at your own risk of not having a working AIL at the end :( Will be merged into main install deps later on."
|
||||
exit 1
|
||||
|
||||
set -e
|
||||
set -x
|
||||
|
||||
sudo pacman -Syu
|
||||
|
||||
sudo pacman -S python2-pip screen gcc unzip freetype2 python2 git --needed
|
||||
sudo yaourt -S snappy --needed
|
||||
sudo pip2 install virtualenv
|
||||
|
||||
#Needed for bloom filters
|
||||
sudo pacman -S openssl python2-numpy --needed
|
||||
|
||||
# DNS deps
|
||||
sudo pacman -S adns --needed
|
||||
|
||||
#Needed for redis-lvlDB
|
||||
sudo pacman -S libev gmp --needed
|
||||
|
||||
#needed for mathplotlib
|
||||
test ! -L /usr/include/ft2build.h && sudo ln -s freetype2/ft2build.h /usr/include/
|
||||
sudo easy_install-2.7 -U distribute
|
||||
|
||||
# REDIS #
|
||||
test ! -d redis/ && git clone https://github.com/antirez/redis.git
|
||||
pushd redis/
|
||||
git checkout 2.8
|
||||
make
|
||||
popd
|
||||
|
||||
# REDIS LEVEL DB #
|
||||
test ! -d redis-leveldb/ && git clone https://github.com/KDr2/redis-leveldb.git
|
||||
pushd redis-leveldb/
|
||||
git submodule init
|
||||
git submodule update
|
||||
make
|
||||
popd
|
||||
|
||||
# Faup
|
||||
test ! -d faup/ && git clone https://github.com/stricaud/faup.git
|
||||
pushd faup/
|
||||
test ! -d build && mkdir build
|
||||
cd build
|
||||
cmake .. && make
|
||||
sudo make install
|
||||
echo '/usr/local/lib' | sudo tee -a /etc/ld.so.conf.d/faup.conf
|
||||
sudo ldconfig
|
||||
popd
|
||||
|
||||
# tlsh
|
||||
test ! -d tlsh && git clone git://github.com/trendmicro/tlsh.git
|
||||
pushd tlsh/
|
||||
./make.sh
|
||||
pushd build/release/
|
||||
sudo make install
|
||||
sudo ldconfig
|
||||
popd
|
||||
popd
|
||||
|
||||
|
||||
|
||||
if [ ! -f bin/packages/config.cfg ]; then
|
||||
cp bin/packages/config.cfg.sample bin/packages/config.cfg
|
||||
fi
|
||||
|
||||
pushd var/www/
|
||||
./update_thirdparty.sh
|
||||
popd
|
||||
|
||||
virtualenv AILENV
|
||||
|
||||
echo export AIL_HOME=$(pwd) >> ./AILENV/bin/activate
|
||||
echo export AIL_BIN=$(pwd)/bin/ >> ./AILENV/bin/activate
|
||||
echo export AIL_FLASK=$(pwd)/var/www/ >> ./AILENV/bin/activate
|
||||
echo export AIL_REDIS=$(pwd)/redis/src/ >> ./AILENV/bin/activate
|
||||
echo export AIL_LEVELDB=$(pwd)/redis-leveldb/ >> ./AILENV/bin/activate
|
||||
|
||||
. ./AILENV/bin/activate
|
||||
|
||||
mkdir -p $AIL_HOME/{PASTES,Blooms,dumps}
|
||||
mkdir -p $AIL_HOME/LEVEL_DB_DATA/2017
|
||||
mkdir -p $AIL_HOME/LEVEL_DB_DATA/3017
|
||||
|
||||
pip install -U pip
|
||||
pip install -U -r pip_packages_requirement.txt
|
||||
|
||||
# Pyfaup
|
||||
pushd faup/src/lib/bindings/python/
|
||||
python setup.py install
|
||||
popd
|
||||
|
||||
# Py tlsh
|
||||
pushd tlsh/py_ext
|
||||
python setup.py build
|
||||
python setup.py install
|
||||
|
||||
# Download the necessary NLTK corpora and sentiment vader
|
||||
HOME=$(pwd) python -m textblob.download_corpora
|
||||
python -m nltk.downloader vader_lexicon
|
||||
python -m nltk.downloader punkt
|
||||
|
||||
#Create the file all_module and update the graph in doc
|
||||
$AIL_HOME/doc/generate_modules_data_flow_graph.sh
|
|
@ -0,0 +1,41 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import redis
|
||||
import datetime
|
||||
import configparser
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
start_deb = time.time()
|
||||
|
||||
configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg')
|
||||
if not os.path.exists(configfile):
|
||||
raise Exception('Unable to find the configuration file. \
|
||||
Did you set environment variables? \
|
||||
Or activate the virtualenv.')
|
||||
cfg = configparser.ConfigParser()
|
||||
cfg.read(configfile)
|
||||
|
||||
r_serv = redis.StrictRedis(
|
||||
host=cfg.get("ARDB_DB", "host"),
|
||||
port=cfg.getint("ARDB_DB", "port"),
|
||||
db=cfg.getint("ARDB_DB", "db"),
|
||||
decode_responses=True)
|
||||
|
||||
#Set current ail version
|
||||
r_serv.set('ail:version', 'v2.0')
|
||||
|
||||
# use new update_date format
|
||||
date_tag_to_replace = ['v1.5', 'v1.7']
|
||||
for tag in date_tag_to_replace:
|
||||
if r_serv.exists('ail:update_date_{}'.format(tag)):
|
||||
date_tag = r_serv.get('ail:update_date_{}'.format(tag))
|
||||
r_serv.hset('ail:update_date', tag, date_tag)
|
||||
r_serv.delete('ail:update_date_{}'.format(tag))
|
||||
|
||||
#Set current ail version
|
||||
r_serv.hset('ail:update_date', 'v2.0', datetime.datetime.now().strftime("%Y%m%d"))
|
|
@ -0,0 +1,62 @@
|
|||
#!/bin/bash
|
||||
|
||||
[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1;
|
||||
|
||||
export PATH=$AIL_HOME:$PATH
|
||||
export PATH=$AIL_REDIS:$PATH
|
||||
export PATH=$AIL_ARDB:$PATH
|
||||
export PATH=$AIL_BIN:$PATH
|
||||
export PATH=$AIL_FLASK:$PATH
|
||||
|
||||
GREEN="\\033[1;32m"
|
||||
DEFAULT="\\033[0;39m"
|
||||
|
||||
echo -e $GREEN"Shutting down AIL ..."$DEFAULT
|
||||
bash ${AIL_BIN}/LAUNCH.sh -k
|
||||
wait
|
||||
|
||||
echo ""
|
||||
echo -e $GREEN"Update requirement"$DEFAULT
|
||||
echo ""
|
||||
pip3 install flask-login
|
||||
wait
|
||||
echo ""
|
||||
pip3 install bcrypt
|
||||
wait
|
||||
echo ""
|
||||
echo ""
|
||||
|
||||
bash ${AIL_BIN}LAUNCH.sh -lav &
|
||||
wait
|
||||
echo ""
|
||||
|
||||
echo ""
|
||||
echo -e $GREEN"Updating AIL VERSION ..."$DEFAULT
|
||||
echo ""
|
||||
python ${AIL_HOME}/update/v2.0/Update.py
|
||||
wait
|
||||
echo ""
|
||||
echo ""
|
||||
|
||||
echo ""
|
||||
echo -e $GREEN"Update thirdparty ..."$DEFAULT
|
||||
bash ${AIL_BIN}/LAUNCH.sh -t
|
||||
wait
|
||||
echo ""
|
||||
|
||||
echo ""
|
||||
echo -e $GREEN"Create Default User"$DEFAULT
|
||||
echo ""
|
||||
python3 ${AIL_FLASK}create_default_user.py
|
||||
|
||||
|
||||
echo ""
|
||||
echo -e $GREEN"Shutting down ARDB ..."$DEFAULT
|
||||
bash ${AIL_BIN}/LAUNCH.sh -k
|
||||
wait
|
||||
|
||||
exit 0
|
|
@ -36,18 +36,6 @@ import Flask_config
|
|||
from Role_Manager import create_user_db, check_password_strength
|
||||
from Role_Manager import login_admin, login_analyst
|
||||
|
||||
def flask_init():
|
||||
# # TODO: move this to update
|
||||
# role init
|
||||
if not r_serv_db.exists('ail:all_role'):
|
||||
r_serv_db.zadd('ail:all_role', 1, 'admin')
|
||||
r_serv_db.zadd('ail:all_role', 2, 'analyst')
|
||||
|
||||
# check if an account exists
|
||||
if not r_serv_db.exists('user:all'):
|
||||
password = secrets.token_urlsafe()
|
||||
create_user_db('admin@admin.test', password, role='admin',default=True)
|
||||
|
||||
# CONFIG #
|
||||
cfg = Flask_config.cfg
|
||||
baseUrl = cfg.get("Flask", "baseurl")
|
||||
|
|
|
@ -3,20 +3,44 @@
|
|||
|
||||
import os
|
||||
import sys
|
||||
|
||||
import redis
|
||||
import secrets
|
||||
import configparser
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
|
||||
sys.path.append('./modules/')
|
||||
sys.path.append(os.path.join(os.environ['AIL_FLASK'], 'modules'))
|
||||
|
||||
from Role_Manager import create_user_db, get_default_admin_token
|
||||
from Role_Manager import create_user_db, edit_user_db, get_default_admin_token
|
||||
|
||||
|
||||
|
||||
configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg')
|
||||
if not os.path.exists(configfile):
|
||||
raise Exception('Unable to find the configuration file. \
|
||||
Did you set environment variables? \
|
||||
Or activate the virtualenv.')
|
||||
cfg = configparser.ConfigParser()
|
||||
cfg.read(configfile)
|
||||
|
||||
r_serv = redis.StrictRedis(
|
||||
host=cfg.get("ARDB_DB", "host"),
|
||||
port=cfg.getint("ARDB_DB", "port"),
|
||||
db=cfg.getint("ARDB_DB", "db"),
|
||||
decode_responses=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
# create role_list
|
||||
if not r_serv.exists('ail:all_role'):
|
||||
r_serv.zadd('ail:all_role', 1, 'admin')
|
||||
r_serv.zadd('ail:all_role', 2, 'analyst')
|
||||
|
||||
username = 'admin@admin.test'
|
||||
password = secrets.token_urlsafe()
|
||||
create_user_db(username, password, role='admin', default=True)
|
||||
if r_serv.exists('user_metadata:admin@admin.test'):
|
||||
edit_user_db(username, password=password, role='admin')
|
||||
else:
|
||||
create_user_db(username, password, role='admin', default=True)
|
||||
token = get_default_admin_token()
|
||||
|
||||
default_passwd_file = os.path.join(os.environ['AIL_HOME'], 'DEFAULT_PASSWORD')
|
||||
|
|
|
@ -93,10 +93,10 @@ def one():
|
|||
|
||||
# ============= ROUTES ==============
|
||||
|
||||
@restApi.route("/api", methods=['GET'])
|
||||
@login_required
|
||||
def api():
|
||||
return 'api doc'
|
||||
# @restApi.route("/api", methods=['GET'])
|
||||
# @login_required
|
||||
# def api():
|
||||
# return 'api doc'
|
||||
|
||||
@restApi.route("api/items", methods=['POST'])
|
||||
@token_required
|
||||
|
|
|
@ -26,82 +26,6 @@
|
|||
|
||||
<div class="col-12 col-lg-10" id="core_content">
|
||||
|
||||
<div class="row">
|
||||
<div class="col-xl-6">
|
||||
|
||||
<div class="card mt-1 mb-1">
|
||||
<div class="card-header text-white bg-dark">
|
||||
<h5><a class="text-info" href="{{ url_for('hiddenServices.Crawler_Splash_last_by_type')}}?type=onion"><i class="fas fa-user-secret"></i> Onions Crawlers</a></h5>
|
||||
<div class="row">
|
||||
<div class="col-6">
|
||||
<span class="badge badge-success" id="stat_onion_domain_up">{{ statDomains_onion['domains_up'] }}</span> UP
|
||||
<span class="badge badge-danger ml-md-3" id="stat_onion_domain_down">{{ statDomains_onion['domains_down'] }}</span> DOWN
|
||||
</div>
|
||||
<div class="col-6">
|
||||
<span class="badge badge-success" id="stat_onion_total">{{ statDomains_onion['total'] }}</span> Crawled
|
||||
<span class="badge badge-warning ml-md-3" id="stat_onion_queue">{{ statDomains_onion['domains_queue'] }}</span> Queue
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="card-body px-0 py-0 ">
|
||||
<table class="table">
|
||||
<tbody id="tbody_crawler_onion_info">
|
||||
{% for crawler in crawler_metadata_onion %}
|
||||
<tr>
|
||||
<td>
|
||||
<i class="fas fa-{%if crawler['status']%}check{%else%}times{%endif%}-circle" style="color:{%if crawler['status']%}Green{%else%}Red{%endif%};"></i> {{crawler['crawler_info']}}
|
||||
</td>
|
||||
<td>
|
||||
{{crawler['crawling_domain']}}
|
||||
</td>
|
||||
<td style="color:{%if crawler['status']%}Green{%else%}Red{%endif%};">
|
||||
{{crawler['status_info']}}
|
||||
</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
<div class="col-xl-6">
|
||||
<div class="card mt-1 mb-1">
|
||||
<div class="card-header text-white bg-dark">
|
||||
<h5><a class="text-info" href="{{ url_for('hiddenServices.Crawler_Splash_last_by_type')}}?type=regular"><i class="fab fa-html5"></i> Regular Crawlers</a></h5>
|
||||
<div class="row">
|
||||
<div class="col-6">
|
||||
<span class="badge badge-success" id="stat_regular_domain_up">{{ statDomains_regular['domains_up'] }}</span> UP
|
||||
<span class="badge badge-danger ml-md-3" id="stat_regular_domain_down">{{ statDomains_regular['domains_down'] }}</span> DOWN
|
||||
</div>
|
||||
<div class="col-6">
|
||||
<span class="badge badge-success" id="stat_regular_total">{{ statDomains_regular['total'] }}</span> Crawled
|
||||
<span class="badge badge-warning ml-md-3" id="stat_regular_queue">{{ statDomains_regular['domains_queue'] }}</span> Queue
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="card-body px-0 py-0 ">
|
||||
<table class="table">
|
||||
<tbody id="tbody_crawler_regular_info">
|
||||
{% for crawler in crawler_metadata_regular %}
|
||||
<tr>
|
||||
<td>
|
||||
<i class="fas fa-{%if crawler['status']%}check{%else%}times{%endif%}-circle" style="color:{%if crawler['status']%}Green{%else%}Red{%endif%};"></i> {{crawler['crawler_info']}}
|
||||
</td>
|
||||
<td>
|
||||
{{crawler['crawling_domain']}}
|
||||
</td>
|
||||
<td style="color:{%if crawler['status']%}Green{%else%}Red{%endif%};">
|
||||
{{crawler['status_info']}}
|
||||
</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
@ -109,114 +33,3 @@
|
|||
|
||||
|
||||
</body>
|
||||
|
||||
<script>
|
||||
var to_refresh = false
|
||||
$(document).ready(function(){
|
||||
$("#page-Crawler").addClass("active");
|
||||
$("#nav_dashboard").addClass("active");
|
||||
$( window ).focus(function() {
|
||||
to_refresh = true
|
||||
refresh_crawler_status();
|
||||
});
|
||||
$( window ).blur(function() {
|
||||
to_refresh = false
|
||||
});
|
||||
|
||||
to_refresh = true
|
||||
refresh_crawler_status();
|
||||
});
|
||||
|
||||
function toggle_sidebar(){
|
||||
if($('#nav_menu').is(':visible')){
|
||||
$('#nav_menu').hide();
|
||||
$('#side_menu').removeClass('border-right')
|
||||
$('#side_menu').removeClass('col-lg-2')
|
||||
$('#core_content').removeClass('col-lg-10')
|
||||
}else{
|
||||
$('#nav_menu').show();
|
||||
$('#side_menu').addClass('border-right')
|
||||
$('#side_menu').addClass('col-lg-2')
|
||||
$('#core_content').addClass('col-lg-10')
|
||||
}
|
||||
}
|
||||
|
||||
function refresh_crawler_status(){
|
||||
|
||||
$.getJSON("{{ url_for('hiddenServices.crawler_dashboard_json') }}",
|
||||
function(data) {
|
||||
|
||||
$('#stat_onion_domain_up').text(data.statDomains_onion['domains_up']);
|
||||
$('#stat_onion_domain_down').text(data.statDomains_onion['domains_down']);
|
||||
$('#stat_onion_total').text(data.statDomains_onion['total']);
|
||||
$('#stat_onion_queue').text(data.statDomains_onion['domains_queue']);
|
||||
|
||||
$('#stat_regular_domain_up').text(data.statDomains_regular['domains_up']);
|
||||
$('#stat_regular_domain_down').text(data.statDomains_regular['domains_down']);
|
||||
$('#stat_regular_total').text(data.statDomains_regular['total']);
|
||||
$('#stat_regular_queue').text(data.statDomains_regular['domains_queue']);
|
||||
|
||||
if(data.crawler_metadata_onion.length!=0){
|
||||
$("#tbody_crawler_onion_info").empty();
|
||||
var tableRef = document.getElementById('tbody_crawler_onion_info');
|
||||
for (var i = 0; i < data.crawler_metadata_onion.length; i++) {
|
||||
var crawler = data.crawler_metadata_onion[i];
|
||||
var newRow = tableRef.insertRow(tableRef.rows.length);
|
||||
var text_color;
|
||||
var icon;
|
||||
if(crawler['status']){
|
||||
text_color = 'Green';
|
||||
icon = 'check';
|
||||
} else {
|
||||
text_color = 'Red';
|
||||
icon = 'times';
|
||||
}
|
||||
|
||||
var newCell = newRow.insertCell(0);
|
||||
newCell.innerHTML = "<td><i class=\"fas fa-"+icon+"-circle\" style=\"color:"+text_color+";\"></i> "+crawler['crawler_info']+"</td>";
|
||||
|
||||
newCell = newRow.insertCell(1);
|
||||
newCell.innerHTML = "<td>"+crawler['crawling_domain']+"</td>";
|
||||
|
||||
newCell = newRow.insertCell(2);
|
||||
newCell.innerHTML = "<td><div style=\"color:"+text_color+";\">"+crawler['status_info']+"</div></td>";
|
||||
|
||||
//$("#panel_crawler").show();
|
||||
}
|
||||
}
|
||||
if(data.crawler_metadata_regular.length!=0){
|
||||
$("#tbody_crawler_regular_info").empty();
|
||||
var tableRef = document.getElementById('tbody_crawler_regular_info');
|
||||
for (var i = 0; i < data.crawler_metadata_regular.length; i++) {
|
||||
var crawler = data.crawler_metadata_regular[i];
|
||||
var newRow = tableRef.insertRow(tableRef.rows.length);
|
||||
var text_color;
|
||||
var icon;
|
||||
if(crawler['status']){
|
||||
text_color = 'Green';
|
||||
icon = 'check';
|
||||
} else {
|
||||
text_color = 'Red';
|
||||
icon = 'times';
|
||||
}
|
||||
|
||||
var newCell = newRow.insertCell(0);
|
||||
newCell.innerHTML = "<td><i class=\"fas fa-"+icon+"-circle\" style=\"color:"+text_color+";\"></i> "+crawler['crawler_info']+"</td>";
|
||||
|
||||
newCell = newRow.insertCell(1);
|
||||
newCell.innerHTML = "<td>"+crawler['crawling_domain']+"</td>";
|
||||
|
||||
newCell = newRow.insertCell(2);
|
||||
newCell.innerHTML = "<td><div style=\"color:"+text_color+";\">"+crawler['status_info']+"</div></td>";
|
||||
|
||||
//$("#panel_crawler").show();
|
||||
}
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
if (to_refresh) {
|
||||
setTimeout("refresh_crawler_status()", 10000);
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
|
|
@ -1 +0,0 @@
|
|||
<li id='page-hiddenServices'><a href="{{ url_for('hiddenServices.dashboard') }}"><i class="fa fa-user-secret"></i> hidden Services </a></li>
|
|
@ -7,6 +7,7 @@
|
|||
|
||||
<!-- Core CSS -->
|
||||
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
|
||||
</head>
|
||||
|
||||
<body>
|
||||
|
|
|
@ -7,6 +7,7 @@
|
|||
|
||||
<!-- Core CSS -->
|
||||
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
|
||||
</head>
|
||||
|
||||
<body>
|
||||
|
|
Loading…
Reference in New Issue