pull/350/head
Terrtia 2019-05-02 13:44:29 +02:00
commit 1bf752fbfc
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
7 changed files with 89 additions and 68 deletions

View File

@ -1,7 +1,7 @@
language: python
python:
- "3.5"
- "3.6"
sudo: required
@ -12,7 +12,7 @@ env:
AIL_FLASK=$TRAVIS_BUILD_DIR/var/www/ AIL_REDIS=$TRAVIS_BUILD_DIR/redis/src/ \
AIL_LEVELDB=$TRAVIS_BUILD_DIR/redis-leveldb/ PATH=$AIL_HOME:$AIL_REDIS:$AIL_LEVELDB:$PATH
dist: xenial
install:
- ./installing_deps.sh
@ -20,10 +20,7 @@ install:
script:
- pushd bin
- ./launch_redis.sh
- ./launch_lvldb.sh
- ./launch_logs.sh
- ./launch_queues.sh
- ./LAUNCH.sh -l
- popd
- cd tests
- nosetests --with-coverage --cover-package=../bin -d

View File

@ -1,18 +1,17 @@
AIL
===
![Logo](./doc/logo/logo-small.png?raw=true "AIL logo")
AIL framework - Framework for Analysis of Information Leaks
AIL is a modular framework to analyse potential information leaks from unstructured data sources like pastes from Pastebin or similar services or unstructured data streams. AIL framework is flexible and can be extended to support other functionalities to mine or process sensitive information (e.g. data leak prevention).
![Dashboard](./doc/screenshots/dashboard.png?raw=true "AIL framework dashboard")
<table>
<tr>
<td>Latest Release</td>
<td><a href="https://github.com/CIRCL/AIL-framework/releases/latest"><img src="https://img.shields.io/github/release/CIRCL/AIL-framework/all.svg"></a></td>
</tr>
<td>Travis</td>
<td><a href="https://travis-ci.org/CIRCL/AIL-framework"><img src="https://img.shields.io/travis/CIRCL/AIL-framework.svg" /></a></td>
</tr>
<tr>
<td>Gitter</td>
<td><a href="https://gitter.im/SteveClement/AIL-framework?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge"><img src="https://badges.gitter.im/SteveClement/AIL-framework.svg" /></a></td>
</tr>
<tr>
<td>Contributors</td>
@ -24,6 +23,14 @@ AIL is a modular framework to analyse potential information leaks from unstructu
</tr>
</table>
![Logo](./doc/logo/logo-small.png?raw=true "AIL logo")
AIL framework - Framework for Analysis of Information Leaks
AIL is a modular framework to analyse potential information leaks from unstructured data sources like pastes from Pastebin or similar services or unstructured data streams. AIL framework is flexible and can be extended to support other functionalities to mine or process sensitive information (e.g. data leak prevention).
![Dashboard](./doc/screenshots/dashboard.png?raw=true "AIL framework dashboard")
Features
--------

View File

@ -17,6 +17,11 @@ cd ${AIL_HOME}
if [ -e "${DIR}/AILENV/bin/python" ]; then
ENV_PY="${DIR}/AILENV/bin/python"
export AIL_VENV=${AIL_HOME}/AILENV/
elif [ ! -z "$TRAVIS" ]; then
echo "Travis detected"
ENV_PY="~/virtualenv/python3.6/bin/python"
export AIL_VENV="~/virtualenv/python3.6/"
else
echo "Please make sure you have a AIL-framework environment, au revoir"
exit 1
@ -29,7 +34,6 @@ export AIL_BIN=${AIL_HOME}/bin/
export AIL_FLASK=${AIL_HOME}/var/www/
export AIL_REDIS=${AIL_HOME}/redis/src/
export AIL_ARDB=${AIL_HOME}/ardb/src/
export AIL_VENV=${AIL_HOME}/AILENV/
export PATH=$AIL_VENV/bin:$PATH
export PATH=$AIL_HOME:$PATH

View File

@ -1,40 +1,43 @@
#!/bin/bash
# halt on errors
set -e
set -x
## bash debug mode togle below
#set -x
sudo apt-get update
sudo apt-get install python3-pip virtualenv python3-dev python3-tk libfreetype6-dev \
screen g++ python-tk unzip libsnappy-dev cmake -y
screen g++ python-tk unzip libsnappy-dev cmake -qq
#optional tor install
sudo apt-get install tor -y
sudo apt-get install tor -qq
#Needed for bloom filters
sudo apt-get install libssl-dev libfreetype6-dev python-numpy -y
sudo apt-get install libssl-dev libfreetype6-dev python-numpy -qq
#pyMISP
#sudo apt-get -y install python3-pip
# DNS deps
sudo apt-get install libadns1 libadns1-dev -y
sudo apt-get install libadns1 libadns1-dev -qq
#Needed for redis-lvlDB
sudo apt-get install libev-dev libgmp-dev -y
sudo apt-get install libev-dev libgmp-dev -qq
#Need for generate-data-flow graph
sudo apt-get install graphviz -y
sudo apt-get install graphviz -qq
# install nosetests
sudo pip3 install nose
sudo apt-get install python3-nose -qq
# ssdeep
sudo apt-get install libfuzzy-dev -y
sudo apt-get install build-essential libffi-dev automake autoconf libtool -y
sudo apt-get install libfuzzy-dev -qq
sudo apt-get install build-essential libffi-dev automake autoconf libtool -qq
# sflock, gz requirement
sudo apt-get install p7zip-full -y
sudo apt-get install p7zip-full -qq
# REDIS #
test ! -d redis/ && git clone https://github.com/antirez/redis.git

0
installing_deps_archlinux.sh Normal file → Executable file
View File

View File

@ -105,6 +105,7 @@ def all_hash_search():
show_decoded_files = request.form.get('show_decoded_files')
return redirect(url_for('hashDecoded.hashDecoded_page', date_from=date_from, date_to=date_to, type=type, encoding=encoding, show_decoded_files=show_decoded_files))
@hashDecoded.route("/hashDecoded/", methods=['GET'])
def hashDecoded_page():
date_from = request.args.get('date_from')
@ -221,17 +222,20 @@ def hashDecoded_page():
return render_template("hashDecoded.html", l_64=b64_metadata, vt_enabled=vt_enabled, l_type=l_type, type=type, daily_type_chart=daily_type_chart, daily_date=daily_date,
encoding=encoding, all_encoding=all_encoding, date_from=date_from, date_to=date_to, show_decoded_files=show_decoded_files)
@hashDecoded.route('/hashDecoded/hash_by_type')
def hash_by_type():
type = request.args.get('type')
type = 'text/plain'
return render_template('hash_type.html',type = type)
@hashDecoded.route('/hashDecoded/hash_hash')
def hash_hash():
hash = request.args.get('hash')
return render_template('hash_hash.html')
@hashDecoded.route('/hashDecoded/showHash')
def showHash():
hash = request.args.get('hash')
@ -284,6 +288,7 @@ def showHash():
first_seen=first_seen, list_hash_decoder=list_hash_decoder,
last_seen=last_seen, nb_seen_in_all_pastes=nb_seen_in_all_pastes, sparkline_values=sparkline_values)
@hashDecoded.route('/hashDecoded/downloadHash')
def downloadHash():
hash = request.args.get('hash')
@ -319,6 +324,7 @@ def downloadHash():
else:
return 'hash: ' + hash + " don't exist"
@hashDecoded.route('/hashDecoded/hash_by_type_json')
def hash_by_type_json():
type = request.args.get('type')
@ -351,6 +357,7 @@ def hash_by_type_json():
else:
return jsonify()
@hashDecoded.route('/hashDecoded/decoder_type_json')
def decoder_type_json():
date_from = request.args.get('date_from')
@ -405,6 +412,7 @@ def decoder_type_json():
to_json.append({'name': decoder, 'value': nb_decoded[decoder]})
return jsonify(to_json)
@hashDecoded.route('/hashDecoded/top5_type_json')
def top5_type_json():
date_from = request.args.get('date_from')
@ -481,6 +489,7 @@ def daily_type_json():
return jsonify(type_value)
@hashDecoded.route('/hashDecoded/range_type_json')
def range_type_json():
date_from = request.args.get('date_from')
@ -536,14 +545,13 @@ def range_type_json():
return jsonify(range_type)
@hashDecoded.route('/hashDecoded/hash_graph_line_json')
def hash_graph_line_json():
hash = request.args.get('hash')
date_from = request.args.get('date_from')
date_to = request.args.get('date_to')
#hash = '9c748d28d78a64aef99e7ba866a433eb635c6d7a'
if date_from is None or date_to is None:
nb_days_seen_in_pastes = 30
else:
@ -552,7 +560,7 @@ def hash_graph_line_json():
date_range_seen_in_pastes = get_date_range(nb_days_seen_in_pastes)
#verify input
# verify input
if r_serv_metadata.hget('metadata_hash:'+hash, 'estimated_type') is not None:
json_seen_in_paste = []
for date in date_range_seen_in_pastes:
@ -560,7 +568,7 @@ def hash_graph_line_json():
if nb_seen_this_day is None:
nb_seen_this_day = 0
date = date[0:4] + '-' + date[4:6] + '-' + date[6:8]
json_seen_in_paste.append({ 'date' : date, 'value' : int( nb_seen_this_day )})
json_seen_in_paste.append({'date': date, 'value': int(nb_seen_this_day)})
return jsonify(json_seen_in_paste)
else:
@ -633,12 +641,14 @@ def hash_graph_node_json():
else:
return jsonify({})
@hashDecoded.route('/hashDecoded/hash_types')
def hash_types():
date_from = 20180701
date_to = 20180706
return render_template('hash_types.html', date_from=date_from, date_to=date_to)
@hashDecoded.route('/hashDecoded/send_file_to_vt_js')
def send_file_to_vt_js():
hash = request.args.get('hash')
@ -667,7 +677,7 @@ def update_vt_result():
hash = request.args.get('hash')
params = {'apikey': vt_auth, 'resource': hash}
response = requests.get('https://www.virustotal.com/vtapi/v2/file/report',params=params)
response = requests.get('https://www.virustotal.com/vtapi/v2/file/report', params=params)
if response.status_code == 200:
json_response = response.json()
response_code = json_response['response_code']
@ -676,7 +686,7 @@ def update_vt_result():
total = json_response['total']
positive = json_response['positives']
b64_vt_report = 'Detection {}/{}'.format(positive,total)
b64_vt_report = 'Detection {}/{}'.format(positive, total)
# no report found
elif response_code == 0:
b64_vt_report = 'No report found'
@ -690,7 +700,7 @@ def update_vt_result():
return jsonify(hash=hash, report_vt=b64_vt_report)
elif response.status_code == 403:
Flask_config.vt_enabled = False
print('VT is disabled')
print('Virustotal key is incorrect (e.g. for public API not for virustotal intelligence), authentication failed or reaching limits.')
return jsonify()
else:
# TODO FIXME make json response

View File

@ -2,7 +2,7 @@
set -e
wget http://dygraphs.com/dygraph-combined.js -O ./static/js/dygraph-combined.js
wget -q http://dygraphs.com/dygraph-combined.js -O ./static/js/dygraph-combined.js
SBADMIN_VERSION='3.3.7'
BOOTSTRAP_VERSION='4.2.1'
@ -12,28 +12,28 @@ D3_JS_VERSION='5.5.0'
rm -rf temp
mkdir temp
wget https://github.com/twbs/bootstrap/releases/download/v${BOOTSTRAP_VERSION}/bootstrap-${BOOTSTRAP_VERSION}-dist.zip -O temp/bootstrap${BOOTSTRAP_VERSION}.zip
wget https://github.com/FezVrasta/popper.js/archive/v1.14.3.zip -O temp/popper.zip
wget https://github.com/BlackrockDigital/startbootstrap-sb-admin/archive/v${SBADMIN_VERSION}.zip -O temp/${SBADMIN_VERSION}.zip
wget https://github.com/BlackrockDigital/startbootstrap-sb-admin-2/archive/v${SBADMIN_VERSION}.zip -O temp/${SBADMIN_VERSION}-2.zip
wget https://github.com/FortAwesome/Font-Awesome/archive/v4.7.0.zip -O temp/FONT_AWESOME_4.7.0.zip
wget https://github.com/FortAwesome/Font-Awesome/archive/5.7.1.zip -O temp/FONT_AWESOME_${FONT_AWESOME_VERSION}.zip
wget https://github.com/d3/d3/releases/download/v${D3_JS_VERSION}/d3.zip -O temp/d3_${D3_JS_VERSION}.zip
wget -q https://github.com/twbs/bootstrap/releases/download/v${BOOTSTRAP_VERSION}/bootstrap-${BOOTSTRAP_VERSION}-dist.zip -O temp/bootstrap${BOOTSTRAP_VERSION}.zip
wget -q https://github.com/FezVrasta/popper.js/archive/v1.14.3.zip -O temp/popper.zip
wget -q https://github.com/BlackrockDigital/startbootstrap-sb-admin/archive/v${SBADMIN_VERSION}.zip -O temp/${SBADMIN_VERSION}.zip
wget -q https://github.com/BlackrockDigital/startbootstrap-sb-admin-2/archive/v${SBADMIN_VERSION}.zip -O temp/${SBADMIN_VERSION}-2.zip
wget -q https://github.com/FortAwesome/Font-Awesome/archive/v4.7.0.zip -O temp/FONT_AWESOME_4.7.0.zip
wget -q https://github.com/FortAwesome/Font-Awesome/archive/5.7.1.zip -O temp/FONT_AWESOME_${FONT_AWESOME_VERSION}.zip
wget -q https://github.com/d3/d3/releases/download/v${D3_JS_VERSION}/d3.zip -O temp/d3_${D3_JS_VERSION}.zip
# dateRangePicker
wget https://github.com/moment/moment/archive/2.22.2.zip -O temp/moment_2.22.2.zip
wget https://github.com/longbill/jquery-date-range-picker/archive/v0.18.0.zip -O temp/daterangepicker_v0.18.0.zip
wget -q https://github.com/moment/moment/archive/2.22.2.zip -O temp/moment_2.22.2.zip
wget -q https://github.com/longbill/jquery-date-range-picker/archive/v0.18.0.zip -O temp/daterangepicker_v0.18.0.zip
unzip temp/bootstrap${BOOTSTRAP_VERSION}.zip -d temp/
unzip temp/popper.zip -d temp/
unzip temp/${SBADMIN_VERSION}.zip -d temp/
unzip temp/${SBADMIN_VERSION}-2.zip -d temp/
unzip temp/FONT_AWESOME_4.7.0.zip -d temp/
unzip temp/FONT_AWESOME_${FONT_AWESOME_VERSION}.zip -d temp/
unzip temp/d3_${D3_JS_VERSION}.zip -d temp/
unzip -qq temp/bootstrap${BOOTSTRAP_VERSION}.zip -d temp/
unzip -qq temp/popper.zip -d temp/
unzip -qq temp/${SBADMIN_VERSION}.zip -d temp/
unzip -qq temp/${SBADMIN_VERSION}-2.zip -d temp/
unzip -qq temp/FONT_AWESOME_4.7.0.zip -d temp/
unzip -qq temp/FONT_AWESOME_${FONT_AWESOME_VERSION}.zip -d temp/
unzip -qq temp/d3_${D3_JS_VERSION}.zip -d temp/
unzip temp/moment_2.22.2.zip -d temp/
unzip temp/daterangepicker_v0.18.0.zip -d temp/
unzip -qq temp/moment_2.22.2.zip -d temp/
unzip -qq temp/daterangepicker_v0.18.0.zip -d temp/
mv temp/bootstrap-${BOOTSTRAP_VERSION}-dist/js/bootstrap.min.js ./static/js/bootstrap4.min.js
mv temp/bootstrap-${BOOTSTRAP_VERSION}-dist/js/bootstrap.min.js.map ./static/js/bootstrap.min.js.map
@ -70,38 +70,38 @@ mv temp/jquery-date-range-picker-0.18.0/dist/jquery.daterangepicker.min.js ./sta
rm -rf temp
JQVERSION="1.12.4"
wget http://code.jquery.com/jquery-${JQVERSION}.js -O ./static/js/jquery.js
wget -q http://code.jquery.com/jquery-${JQVERSION}.js -O ./static/js/jquery.js
#Ressources for dataTable
wget https://cdn.datatables.net/1.10.12/js/jquery.dataTables.min.js -O ./static/js/jquery.dataTables.min.js
wget https://cdn.datatables.net/plug-ins/1.10.7/integration/bootstrap/3/dataTables.bootstrap.css -O ./static/css/dataTables.bootstrap.css
wget https://cdn.datatables.net/plug-ins/1.10.7/integration/bootstrap/3/dataTables.bootstrap.js -O ./static/js/dataTables.bootstrap.js
wget -q https://cdn.datatables.net/1.10.12/js/jquery.dataTables.min.js -O ./static/js/jquery.dataTables.min.js
wget -q https://cdn.datatables.net/plug-ins/1.10.7/integration/bootstrap/3/dataTables.bootstrap.css -O ./static/css/dataTables.bootstrap.css
wget -q https://cdn.datatables.net/plug-ins/1.10.7/integration/bootstrap/3/dataTables.bootstrap.js -O ./static/js/dataTables.bootstrap.js
wget https://cdn.datatables.net/1.10.18/css/dataTables.bootstrap4.min.css -O ./static/css/dataTables.bootstrap.min.css
wget https://cdn.datatables.net/1.10.18/js/dataTables.bootstrap4.min.js -O ./static/js/dataTables.bootstrap.min.js
wget -q https://cdn.datatables.net/1.10.18/css/dataTables.bootstrap4.min.css -O ./static/css/dataTables.bootstrap.min.css
wget -q https://cdn.datatables.net/1.10.18/js/dataTables.bootstrap4.min.js -O ./static/js/dataTables.bootstrap.min.js
#Ressource for graph
wget https://raw.githubusercontent.com/flot/flot/958e5fd43c6dff4bab3e1fd5cb6109df5c1e8003/jquery.flot.js -O ./static/js/jquery.flot.js
wget https://raw.githubusercontent.com/flot/flot/958e5fd43c6dff4bab3e1fd5cb6109df5c1e8003/jquery.flot.pie.js -O ./static/js/jquery.flot.pie.js
wget https://raw.githubusercontent.com/flot/flot/958e5fd43c6dff4bab3e1fd5cb6109df5c1e8003/jquery.flot.time.js -O ./static/js/jquery.flot.time.js
wget https://raw.githubusercontent.com/flot/flot/958e5fd43c6dff4bab3e1fd5cb6109df5c1e8003/jquery.flot.stack.js -O ./static/js/jquery.flot.stack.js
wget -q https://raw.githubusercontent.com/flot/flot/958e5fd43c6dff4bab3e1fd5cb6109df5c1e8003/jquery.flot.js -O ./static/js/jquery.flot.js
wget -q https://raw.githubusercontent.com/flot/flot/958e5fd43c6dff4bab3e1fd5cb6109df5c1e8003/jquery.flot.pie.js -O ./static/js/jquery.flot.pie.js
wget -q https://raw.githubusercontent.com/flot/flot/958e5fd43c6dff4bab3e1fd5cb6109df5c1e8003/jquery.flot.time.js -O ./static/js/jquery.flot.time.js
wget -q https://raw.githubusercontent.com/flot/flot/958e5fd43c6dff4bab3e1fd5cb6109df5c1e8003/jquery.flot.stack.js -O ./static/js/jquery.flot.stack.js
#Ressources for sparkline and canvasJS and slider
wget http://omnipotent.net/jquery.sparkline/2.1.2/jquery.sparkline.min.js -O ./static/js/jquery.sparkline.min.js
wget -q http://omnipotent.net/jquery.sparkline/2.1.2/jquery.sparkline.min.js -O ./static/js/jquery.sparkline.min.js
mkdir temp
wget http://canvasjs.com/fdm/chart/ -O temp/canvasjs.zip
unzip temp/canvasjs.zip -d temp/
wget -q http://canvasjs.com/fdm/chart/ -O temp/canvasjs.zip
unzip -qq temp/canvasjs.zip -d temp/
mv temp/jquery.canvasjs.min.js ./static/js/jquery.canvasjs.min.js
wget https://jqueryui.com/resources/download/jquery-ui-1.12.0.zip -O temp/jquery-ui.zip
unzip temp/jquery-ui.zip -d temp/
wget -q https://jqueryui.com/resources/download/jquery-ui-1.12.0.zip -O temp/jquery-ui.zip
unzip -qq temp/jquery-ui.zip -d temp/
mv temp/jquery-ui-1.12.0/jquery-ui.min.js ./static/js/jquery-ui.min.js
mv temp/jquery-ui-1.12.0/jquery-ui.min.css ./static/css/jquery-ui.min.css
rm -rf temp
mkdir -p ./static/image
pushd static/image
wget https://www.circl.lu/assets/images/logos/AIL.png -O AIL.png
wget -q https://www.circl.lu/assets/images/logos/AIL.png -O AIL.png
popd
if ! [[ -n "$AIL_HOME" ]]