CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

fix duplicate export attribute + tag export whitelist

pull/208/head
Terrtia 2018-06-15 17:25:43 +02:00
parent 31fc33ab03
commit aee1b50ef1
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
10 changed files with 483 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
bin/MISP_The_Hive_feeder.py
View File

@ -38,7 +38,10 @@ except:
# import The Hive Keys
try:
from theHiveKEYS import the_hive_url, the_hive_key
flag_the_hive = True
if the_hive_url == '':
flag_the_hive = False
else:
flag_the_hive = True
except:
print('The HIVE keys not present')
flag_the_hive = False
@ -73,16 +76,18 @@ def create_the_hive_alert(source, path, content, tag):
# Create the Alert
id = None
response = HiveApi.create_alert(alert)
if response.status_code == 201:
#print(json.dumps(response.json(), indent=4, sort_keys=True))
print('Alert Created')
print('')
id = response.json()['id']
else:
print('ko: {}/{}'.format(response.status_code, response.text))
return 0
try:
response = HiveApi.create_alert(alert)
if response.status_code == 201:
#print(json.dumps(response.json(), indent=4, sort_keys=True))
print('Alert Created')
print('')
id = response.json()['id']
else:
print('ko: {}/{}'.format(response.status_code, response.text))
return 0
except:
print('hive connection error')
if __name__ == "__main__":
@ -121,14 +126,14 @@ if __name__ == "__main__":
p = Process(config_section)
# create MISP connection
if flag_misp:
#try:
pymisp = PyMISP(misp_url, misp_key, misp_verifycert)
misp_wrapper = ailleakObject.ObjectWrapper(pymisp)
r_serv_db.set('ail:misp', True)
print('Connected to MISP:', misp_url)
#except:
#flag_misp = False
#print('Not connected to MISP')
try:
pymisp = PyMISP(misp_url, misp_key, misp_verifycert)
misp_wrapper = ailleakObject.ObjectWrapper(pymisp)
r_serv_db.set('ail:misp', True)
print('Connected to MISP:', misp_url)
except:
flag_misp = False
print('Not connected to MISP')
# create The HIVE connection
if flag_the_hive:
@ -158,8 +163,18 @@ if __name__ == "__main__":
full_path = os.path.join(os.environ['AIL_HOME'],
p.config.get("Directories", "pastes"), path)
if HiveApi != False:
create_the_hive_alert(source, path, full_path, tag)
if HiveApi != False:
if int(r_serv_db.get('hive:auto-alerts')) == 1:
whitelist_hive = r_serv_db.scard('whitelist_hive')
if r_serv_db.scard('whitelist_hive') == 0 or r_serv_db.sismember('whitelist_hive', tag):
create_the_hive_alert(source, path, full_path, tag)
else:
print('hive, auto alerts creation disable')
if flag_misp:
misp_wrapper.pushToMISP(uuid_ail, path, tag)
if int(r_serv_db.get('misp:auto-events')) == 1:
if r_serv_db.scard('whitelist_misp') == 0 or r_serv_db.sismember('whitelist_misp', tag):
misp_wrapper.pushToMISP(uuid_ail, path, tag)
else:
print('misp, auto events creation disable')

22
bin/ailleakObject.py
View File

@ -51,29 +51,29 @@ class ObjectWrapper:
self.p_content = self.paste.get_p_content()
self.p_tag = tag
'''print(path)
temp = self.paste._get_p_duplicate()
#beautifier
if not temp:
temp = ''
temp = json.dumps(temp)
print(temp)
self.p_duplicate_number = len(temp) if len(temp) >= 0 else 0
p_duplicate_number = len(temp) if len(temp) >= 0 else 0
to_ret = ""
for dup in temp[:self.maxDuplicateToPushToMISP]:
print(dup)
for dup in temp[:10]:
dup = dup.replace('\'','\"').replace('(','[').replace(')',']')
dup = json.loads(dup)
algo = dup[0]
path = dup[1].split('/')[-6:]
path = '/'.join(path)[:-3] # -3 removes .gz
perc = dup[2]
if algo == 'tlsh':
perc = 100 - int(dup[2])
else:
perc = dup[2]
to_ret += "{}: {} [{}%]\n".format(path, algo, perc)
self.p_duplicate = to_ret'''
self.p_duplicate = ""
self.p_duplicate_number = 0
p_duplicate = to_ret
self.mispObject = AilLeakObject(self.uuid_ail, self.p_source, self.p_date, self.p_content, self.p_duplicate, self.p_duplicate_number)
self.mispObject = AilLeakObject(self.uuid_ail, self.p_source, self.p_date, self.p_content, p_duplicate, p_duplicate_number)
def date_to_str(self, date):
return "{0}-{1}-{2}".format(date.year, date.month, date.day)

13
var/www/Flask_server.py
View File

@ -135,6 +135,19 @@ for tag in taxonomies.get('gdpr').machinetags():
for tag in taxonomies.get('fpf').machinetags():
r_serv_tags.sadd('active_tag_fpf', tag)
# ========== INITIAL tags auto export ============
r_serv_db = redis.StrictRedis(
host=cfg.get("ARDB_DB", "host"),
port=cfg.getint("ARDB_DB", "port"),
db=cfg.getint("ARDB_DB", "db"),
decode_responses=True)
infoleak_tags = taxonomies.get('infoleak').machinetags()
infoleak_automatic_tags = []
for tag in taxonomies.get('infoleak').machinetags():
if tag.split('=')[0][:] == 'infoleak:automatic-detection':
r_serv_db.sadd('list_export_tags', tag)
r_serv_db.sadd('list_export_tags', 'submitted')
# ============ MAIN ============
if __name__ == "__main__":

15
var/www/modules/Flask_config.py
View File

@ -101,19 +101,24 @@ try:
print('Misp connected')
except:
print('Misp not connected')
pymisp = None
pymisp = False
misp_event_url = '#'
# The Hive #
from thehive4py.api import TheHiveApi
import thehive4py.exceptions
try:
from theHiveKEYS import the_hive_url, the_hive_key
HiveApi = TheHiveApi(the_hive_url, the_hive_key)
hive_case_url = the_hive_url+'/index.html#/case/id_here/details'
print('The Hive connected')
if the_hive_url == '':
HiveApi = False
hive_case_url = '#'
print('The HIVE not connected')
else:
HiveApi = TheHiveApi(the_hive_url, the_hive_key)
hive_case_url = the_hive_url+'/index.html#/case/id_here/details'
print('The Hive connected')
except:
print('The HIVE not connected')
HiveApi = None
HiveApi = False
hive_case_url = '#'
# VARIABLES #

139
var/www/modules/PasteSubmit/Flask_PasteSubmit.py
View File

@ -5,7 +5,7 @@
Flask functions and routes for the trending modules page
'''
import redis
from flask import Flask, render_template, jsonify, request, Blueprint, url_for
from flask import Flask, render_template, jsonify, request, Blueprint, url_for, redirect
import unicodedata
import string
@ -16,6 +16,7 @@ import datetime
import uuid
from io import BytesIO
from Date import Date
import json
import Paste
@ -140,6 +141,28 @@ def misp_create_event(distribution, threat_level_id, analysis, info, l_tags, pat
ail_uuid = r_serv_db.get('ail:uuid')
pseudofile = BytesIO(paste.get_p_content().encode())
temp = paste._get_p_duplicate()
#beautifier
if not temp:
temp = ''
p_duplicate_number = len(temp) if len(temp) >= 0 else 0
to_ret = ""
for dup in temp[:10]:
dup = dup.replace('\'','\"').replace('(','[').replace(')',']')
dup = json.loads(dup)
algo = dup[0]
path = dup[1].split('/')[-6:]
path = '/'.join(path)[:-3] # -3 removes .gz
if algo == 'tlsh':
perc = 100 - int(dup[2])
else:
perc = dup[2]
to_ret += "{}: {} [{}%]\n".format(path, algo, perc)
p_duplicate = to_ret
today = datetime.date.today()
# [0-3]
published = False
@ -167,6 +190,10 @@ def misp_create_event(distribution, threat_level_id, analysis, info, l_tags, pat
# FIXME TODO: delete this
leak_obj.add_attribute('type', value='Onion', type='text')
if p_duplicate_number > 0:
leak_obj.add_attribute('duplicate', value=p_duplicate, type='text')
leak_obj.add_attribute('duplicate_number', value=p_duplicate_number, type='counter')
try:
templateID = [x['ObjectTemplate']['id'] for x in pymisp.get_object_templates_list() if x['ObjectTemplate']['name'] == obj_name][0]
except IndexError:
@ -176,11 +203,8 @@ def misp_create_event(distribution, threat_level_id, analysis, info, l_tags, pat
if 'errors' in r:
return False
else:
#if self._p_duplicate_number > 0:
#event.add_attribute('duplicate', value=self._p_duplicate, type='text')
#event.add_attribute('duplicate_number', value=self._p_duplicate_number, type='counter')
event_url = misp_event_url + eventid
return eventid
return event_url
def hive_create_case(hive_tlp, threat_level, hive_description, hive_case_title, l_tags, path):
@ -403,8 +427,11 @@ def create_misp_event():
l_tags = list(r_serv_metadata.smembers('tag:'+path))
event = misp_create_event(distribution, threat_level_id, analysis, info, l_tags, path)
return event
if event != False:
return redirect(event)
else:
return 'error'
return 'error'
@PasteSubmit.route("/PasteSubmit/create_hive_case", methods=['POST'])
def create_hive_case():
@ -421,8 +448,104 @@ def create_hive_case():
l_tags = list(r_serv_metadata.smembers('tag:'+path))
case = hive_create_case(hive_tlp, threat_level, hive_description, hive_case_title, l_tags, path)
if case != False:
return redirect(case)
else:
return 'error'
return case
return 'error'
@PasteSubmit.route("/PasteSubmit/edit_tag_export")
def edit_tag_export():
misp_auto_events = r_serv_db.get('misp:auto-events')
hive_auto_alerts = r_serv_db.get('hive:auto-alerts')
whitelist_misp = r_serv_db.scard('whitelist_misp')
whitelist_hive = r_serv_db.scard('whitelist_hive')
list_export_tags = list(r_serv_db.smembers('list_export_tags'))
status_misp = []
status_hive = []
# empty whitelist
if whitelist_misp == 0:
for tag in list_export_tags:
status_misp.append(True)
else:
for tag in list_export_tags:
if r_serv_db.sismember('whitelist_misp', tag):
status_misp.append(True)
else:
status_misp.append(False)
# empty whitelist
if whitelist_hive == 0:
for tag in list_export_tags:
status_hive.append(True)
else:
for tag in list_export_tags:
if r_serv_db.sismember('whitelist_hive', tag):
status_hive.append(True)
else:
status_hive.append(False)
if int(misp_auto_events) == 1:
misp_active = True
else:
misp_active = False
if int(hive_auto_alerts) == 1:
hive_active = True
else:
hive_active = False
return render_template("edit_tag_export.html",
misp_active=misp_active,
hive_active=hive_active,
list_export_tags=list_export_tags,
status_misp=status_misp,
status_hive=status_hive)
@PasteSubmit.route("/PasteSubmit/tag_export_edited", methods=['POST'])
def tag_export_edited():
tag_enabled_misp = request.form.getlist('tag_enabled_misp')
tag_enabled_hive = request.form.getlist('tag_enabled_hive')
list_export_tags = list(r_serv_db.smembers('list_export_tags'))
r_serv_db.delete('whitelist_misp')
r_serv_db.delete('whitelist_hive')
for tag in tag_enabled_misp:
if r_serv_db.sismember('list_export_tags', tag):
r_serv_db.sadd('whitelist_misp', tag)
else:
return 'invalid input'
for tag in tag_enabled_hive:
if r_serv_db.sismember('list_export_tags', tag):
r_serv_db.sadd('whitelist_hive', tag)
else:
return 'invalid input'
return redirect(url_for('PasteSubmit.edit_tag_export'))
@PasteSubmit.route("/PasteSubmit/enable_misp_auto_event")
def enable_misp_auto_event():
r_serv_db.set('misp:auto-events', 1)
return edit_tag_export()
@PasteSubmit.route("/PasteSubmit/disable_misp_auto_event")
def disable_misp_auto_event():
r_serv_db.set('misp:auto-events', 0)
return edit_tag_export()
@PasteSubmit.route("/PasteSubmit/enable_hive_auto_alert")
def enable_hive_auto_alert():
r_serv_db.set('hive:auto-alerts', 1)
return edit_tag_export()
@PasteSubmit.route("/PasteSubmit/disable_hive_auto_alert")
def disable_hive_auto_alert():
r_serv_db.set('hive:auto-alerts', 0)
return edit_tag_export()
# ========= REGISTRATION =========
app.register_blueprint(PasteSubmit)

264
var/www/modules/PasteSubmit/templates/edit_tag_export.html Normal file
View File

@ -0,0 +1,264 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Analysis Information Leak framework</title>
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
<link href="/static//css/dataTables.bootstrap.css" rel="stylesheet" type="text/css" />
<!-- JS -->
<script type="text/javascript" src="{{ url_for('static', filename='js/dygraph-combined.js') }}"></script>
<script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="/static//js/bootstrap.min.js"></script>
<script src="/static//js/jquery.dataTables.min.js"></script>
<script src="/static//js/dataTables.bootstrap.js"></script>
<style>
.tooltip-inner {
text-align: left;
height: 200%;
width: 200%;
max-width: 500px;
max-height: 500px;
font-size: 13px;
}
xmp {
white-space:pre-wrap;
word-wrap:break-word;
}
.test thead{
background: #d91f2d;
color: #fff;
}
</style>
</head>
<body>
{% include 'navbar.html' %}
<div id="page-wrapper">
<div class="row">
<div class="col-lg-12">
</div>
<!-- /.col-lg-12 -->
</div>
<!-- /.row -->
<div class="container">
<div class="row">
<div class="col-md-6">
<div class="panel panel-primary">
<div class="panel-heading"> MISP Auto Event Creation
{% if misp_active %}
<span class="label label-success pull-right"> Enabled</span>
{% endif %}
{% if not misp_active %}
<span class="label label-danger pull-right"> Disabled</span>
{% endif %}
</div>
<div class="panel-body">
<div class="text-center">
<img id="misp-logo" src="{{url_for('static', filename='image/misp-logo.png') }}" class="text-center" >
</div>
<br>
{% if misp_active %}
<a href="{{ url_for('PasteSubmit.disable_misp_auto_event') }}" class="btn btn-danger pull-right">
<i class="fa fa-times fa"></i> Disable Event Creation
</a>
{% endif %}
{% if not misp_active %}
<a href="{{ url_for('PasteSubmit.enable_misp_auto_event') }}" class="btn btn-success pull-right">
<i class="fa fa-check-square-o fa"></i> Enable Event Creation
</a>
{% endif %}
</div>
</div>
</div>
<div class="col-md-6">
<div class="panel panel-primary" style="min-width: 500px;">
<div class="panel-heading">The hive auto export
{% if hive_active %}
<span class="label label-success pull-right"> Enabled</span>
<span class="pull-right">&nbsp;&nbsp;&nbsp;</span>
<span class="badge badge-light pull-right">{{ badge }}</span>
{% endif %}
{% if not hive_active %}
<span class="label label-danger pull-right"> Disabled</span>
{% endif %}
</div>
<div class="panel-body">
<div class="text-center">
<img id="misp-logo" src="{{url_for('static', filename='image/thehive-logo.png') }}" width="500">
</div>
{% if hive_active %}
<a href="{{ url_for('PasteSubmit.disable_hive_auto_alert') }}" class="btn btn-danger pull-right">
<i class="fa fa-times fa"></i> Disable Alert Creation
</a>
{% endif %}
{% if not hive_active %}
<a href="{{ url_for('PasteSubmit.enable_hive_auto_alert') }}" class="btn btn-success pull-right">
<i class="fa fa-check-square-o fa"></i> Enable Alert Creation
</a>
{% endif %}
</div>
</div>
</div>
</div>
</div>
<form action="/PasteSubmit/tag_export_edited" id="checkboxForm" method='post'>
<div class="container">
<div class="row">
<div class="col-md-6">
<div class="panel panel-info">
<div class="panel-heading">
Metadata :
<ul class="list-group"><li class="list-group-item">
<table class="test table table-striped table-bordered table-hover table-responsive " id="myTable_misp">
<thead>
<tr>
<th>Whitelist</th>
<th>Tag</th>
</tr>
</thead>
<tbody>
{% for tag in list_export_tags %}
<tr>
<td>
{% if status_misp[loop.index0] %}
<div style="display:none;">Enabled</div>
<input type="checkbox" value="{{ tag }}" name="tag_enabled_misp" checked>
{% endif %}
{% if not status_misp[loop.index0] %}
<div style="display:none;">Disabled</div>
<input type="checkbox" value="{{ tag }}" name="tag_enabled_misp" >
{% endif %}
</td>
<td>{{ tag }}</td>
</tr>
{% endfor %}
</tbody>
</table>
</li></ul>
</div></div>
</div>
<div class="col-md-6">
<div class="panel panel-info">
<div class="panel-heading">
Metadata :
<ul class="list-group"><li class="list-group-item">
<table class="test table table-striped table-bordered table-hover table-responsive " id="myTable_hive">
<thead>
<tr>
<th>Whitelist</th>
<th>Tag</th>
</tr>
</thead>
<tbody>
{% for tag in list_export_tags %}
<tr>
<td>
{% if status_hive[loop.index0] %}
<div style="display:none;">Enabled</div>
<input type="checkbox" value="{{ tag }}" name="tag_enabled_hive" checked>
{% endif %}
{% if not status_hive[loop.index0] %}
<div style="display:none;">Disabled</div>
<input type="checkbox" value="{{ tag }}" name="tag_enabled_hive" >
{% endif %}
</td>
<td>{{ tag }}</td>
</tr>
{% endfor %}
</tbody>
</table>
</li></ul>
</div></div>
</div>
</div>
</div>
<div class="input-group-btn text-center">
<button class="btn btn-primary btn-lg" onclick="submitActiveTags()">
<i class="fa fa-check-square-o fa"></i>
Update Tags
</button>
</div>
</form>
</div>
<!-- /#page-wrapper -->
</body>
<script>
var table_misp
var table_hive
$(document).ready(function(){
table_misp = $('#myTable_misp').DataTable(
{
"aLengthMenu": [[5, 10, 15, -1], [5, 10, 15, "All"]],
"iDisplayLength": 10,
"order": [[ 1, "asc" ]]
}
);
table_hive = $('#myTable_hive').DataTable(
{
"aLengthMenu": [[5, 10, 15, -1], [5, 10, 15, "All"]],
"iDisplayLength": 10,
"order": [[ 1, "asc" ]]
}
);
});
</script>
<script>
function submitActiveTags(){
table_misp.destroy()
table_misp = $('#myTable_misp').DataTable(
{
"iDisplayLength": -1,
}
);
table_hive.destroy()
table_hive = $('#myTable_hive').DataTable(
{
"iDisplayLength": -1,
}
);
}
</script>
</html>

1
var/www/modules/search/Flask_search.py
View File

@ -122,7 +122,6 @@ def search():
# Search full line
schema = Schema(title=TEXT(stored=True), path=ID(stored=True), content=TEXT)
print(selected_index)
ix = index.open_dir(selected_index)
with ix.searcher() as searcher:
query = QueryParser("content", ix.schema).parse(" ".join(q))

2
var/www/modules/search/templates/search.html
View File

@ -66,7 +66,7 @@
</br>
<div class="panel panel-default">
<div class="panel-heading">
<i class="glyphicon glyphicon-search"></i> <b id="numberOfRes">{{ r|length }}</b> Results for "<strong>{{ query }}</strong>
<i class="glyphicon glyphicon-search"></i> <b id="numberOfRes">{{ r|length }}</b> Results for "<strong>{{ query }}</strong>"
<div class="pull-right">
</div>

13
var/www/modules/showpaste/Flask_showpaste.py
View File

@ -114,7 +114,18 @@ def showpaste(content_range):
else:
list_tags.append( (tag, False) )
return render_template("show_saved_paste.html", date=p_date, bootstrap_label=bootstrap_label, active_taxonomies=active_taxonomies, active_galaxies=active_galaxies, list_tags=list_tags, source=p_source, encoding=p_encoding, language=p_language, size=p_size, mime=p_mime, lineinfo=p_lineinfo, content=p_content, initsize=len(p_content), duplicate_list = p_duplicate_list, simil_list = p_simil_list, hashtype_list = p_hashtype_list, date_list=p_date_list)
if Flask_config.pymisp is False:
misp = False
else:
misp = True
if Flask_config.HiveApi is False:
hive = False
else:
hive = True
return render_template("show_saved_paste.html", date=p_date, bootstrap_label=bootstrap_label, active_taxonomies=active_taxonomies, active_galaxies=active_galaxies, list_tags=list_tags, source=p_source, encoding=p_encoding, language=p_language, size=p_size, mime=p_mime, lineinfo=p_lineinfo, content=p_content, initsize=len(p_content), duplicate_list = p_duplicate_list, simil_list = p_simil_list, hashtype_list = p_hashtype_list, date_list=p_date_list,
misp=misp, hive=hive)
# ============ ROUTES ============

4
var/www/modules/showpaste/templates/show_saved_paste.html
View File

@ -171,6 +171,7 @@
</table>
<div>
{% if misp %}
<button class="btn btn-default btn-lg" data-toggle="modal" data-target="#myModal_MISP">
Create
<img id="misp-logo" src="{{url_for('static', filename='image/misp-logo.png') }}" height="32">
@ -234,7 +235,9 @@
</div>
</div>
{% endif %}
{% if hive %}
<button class="btn btn-primary btn-lg" data-toggle="modal" data-target="#myModal_hive">
Create
<img id="thehive-icon" src="{{url_for('static', filename='image/thehive_icon.png') }}">
@ -294,6 +297,7 @@
</div>
</div>
{% endif %}
</div>
</div>