mirror of https://github.com/CIRCL/AIL-framework
chg: [API + import] add API format + item_import refractor
Terrtia
parent
44d6eb8570
commit
3a8531cafa
15
OVERVIEW.md
15
OVERVIEW.md
|
|
@ -38,6 +38,21 @@ Redis and ARDB overview
|
|||
| failed_login_ip:**ip** | **nb login failed** | TTL
|
||||
| failed_login_user_id:**user_id** | **nb login failed** | TTL
|
||||
|
||||
##### Item Import:
|
||||
|
||||
| Key | Value |
|
||||
| ------ | ------ |
|
||||
| **uuid**:nb_total | **nb total** | TTL *(if imported)*
|
||||
| **uuid**:nb_end | **nb** | TTL *(if imported)*
|
||||
| **uuid**:nb_sucess | **nb success** | TTL *(if imported)*
|
||||
| **uuid**:end | **0 (in progress) or (item imported)** | TTL *(if imported)*
|
||||
| **uuid**:processing | **process status: 0 or 1** | TTL *(if imported)*
|
||||
| **uuid**:error | **error message** | TTL *(if imported)*
|
||||
|
||||
| Set Key | Value |
|
||||
| ------ | ------ |
|
||||
| **uuid**:paste_submit_link | **item_path** | TTL *(if imported)*
|
||||
|
||||
## DB0 - Core:
|
||||
|
||||
##### Update keys:
|
||||
|
|
|
|||
|
|
@ -0,0 +1,34 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import redis
|
||||
|
||||
import Flask_config
|
||||
|
||||
r_serv_db = Flask_config.r_serv_db
|
||||
r_serv_log = Flask_config.r_serv_log
|
||||
|
||||
def create_import_queue(ltags, ltagsgalaxies, paste_content, UUID, password, isfile = False):
|
||||
|
||||
# save temp value on disk
|
||||
r_serv_db.set(UUID + ':ltags', ltags)
|
||||
r_serv_db.set(UUID + ':ltagsgalaxies', ltagsgalaxies)
|
||||
r_serv_db.set(UUID + ':paste_content', paste_content)
|
||||
r_serv_db.set(UUID + ':password', password)
|
||||
r_serv_db.set(UUID + ':isfile', isfile)
|
||||
|
||||
r_serv_log.set(UUID + ':end', 0)
|
||||
r_serv_log.set(UUID + ':processing', 0)
|
||||
r_serv_log.set(UUID + ':nb_total', -1)
|
||||
r_serv_log.set(UUID + ':nb_end', 0)
|
||||
r_serv_log.set(UUID + ':nb_sucess', 0)
|
||||
|
||||
# save UUID on disk
|
||||
r_serv_db.sadd('submitted:uuid', UUID)
|
||||
return UUID
|
||||
|
||||
def import_text_item():
|
||||
res = r_serv_db.smembers('submitted:uuid')
|
||||
print(res)
|
||||
return res
|
||||
|
|
@ -0,0 +1,61 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import redis
|
||||
|
||||
import Flask_config
|
||||
|
||||
from pytaxonomies import Taxonomies
|
||||
from pymispgalaxies import Galaxies, Clusters
|
||||
|
||||
r_serv_tags = Flask_config.r_serv_tags
|
||||
|
||||
def get_taxonomie_from_tag(tag):
|
||||
return tag.split(':')[0]
|
||||
|
||||
def get_galaxy_from_tag(tag):
|
||||
galaxy = tag.split(':')[1]
|
||||
galaxy = galaxy.split('=')[0]
|
||||
return galaxy
|
||||
|
||||
def get_active_taxonomies():
|
||||
return r_serv_tags.smembers('active_taxonomies')
|
||||
|
||||
def get_active_galaxies():
|
||||
return r_serv_tags.smembers('active_galaxies')
|
||||
|
||||
def is_taxonomie_tag_enabled(taxonomie, tag):
|
||||
if tag in r_serv_tags.smembers('active_tag_' + taxonomie):
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
def is_galaxy_tag_enabled(taxonomie, galaxy):
|
||||
if tag in r_serv_tags.smembers('active_tag_galaxies_' + galaxy):
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
# Check if tags are enabled in AIL
|
||||
def is_valid_tags_taxonomies_galaxy(list_tags, list_tags_galaxy):
|
||||
if list_tags:
|
||||
active_taxonomies = Tags.get_active_taxonomies()
|
||||
|
||||
for tag in list_tags:
|
||||
taxonomie = get_taxonomie_from_tag(tag)
|
||||
if taxonomie not in active_taxonomies:
|
||||
return False
|
||||
if not is_taxonomie_tag_enabled(taxonomie, tag):
|
||||
return False
|
||||
|
||||
if list_tags_galaxy:
|
||||
active_galaxies = Tags.get_active_galaxies()
|
||||
|
||||
for tag in list_tags_galaxy:
|
||||
galaxy = get_galaxy_from_tag(tag)
|
||||
if galaxy not in active_galaxies:
|
||||
return False
|
||||
if not is_galaxy_tag_enabled(galaxy, tag):
|
||||
return False
|
||||
return True
|
||||
|
|
@ -92,7 +92,6 @@ def remove_submit_uuid(uuid):
|
|||
r_serv_log_submit.expire(uuid + ':nb_sucess', expire_time)
|
||||
r_serv_log_submit.expire(uuid + ':nb_end', expire_time)
|
||||
r_serv_log_submit.expire(uuid + ':error', expire_time)
|
||||
r_serv_log_submit.srem(uuid + ':paste_submit_link', '')
|
||||
r_serv_log_submit.expire(uuid + ':paste_submit_link', expire_time)
|
||||
|
||||
# delete uuid
|
||||
|
|
@ -230,8 +229,6 @@ if __name__ == "__main__":
|
|||
r_serv_log_submit.set(uuid + ':nb_total', -1)
|
||||
r_serv_log_submit.set(uuid + ':nb_end', 0)
|
||||
r_serv_log_submit.set(uuid + ':nb_sucess', 0)
|
||||
r_serv_log_submit.set(uuid + ':error', 'error:')
|
||||
r_serv_log_submit.sadd(uuid + ':paste_submit_link', '')
|
||||
|
||||
|
||||
r_serv_log_submit.set(uuid + ':processing', 1)
|
||||
|
|
|
|||
|
|
@ -67,15 +67,15 @@ log_dir = os.path.join(os.environ['AIL_HOME'], 'logs')
|
|||
if not os.path.isdir(log_dir):
|
||||
os.makedirs(logs_dir)
|
||||
|
||||
log_filename = os.path.join(log_dir, 'flask_server.logs')
|
||||
logger = logging.getLogger()
|
||||
formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
|
||||
handler_log = logging.handlers.TimedRotatingFileHandler(log_filename, when="midnight", interval=1)
|
||||
handler_log.suffix = '%Y-%m-%d.log'
|
||||
handler_log.setFormatter(formatter)
|
||||
handler_log.setLevel(30)
|
||||
logger.addHandler(handler_log)
|
||||
logger.setLevel(30)
|
||||
# log_filename = os.path.join(log_dir, 'flask_server.logs')
|
||||
# logger = logging.getLogger()
|
||||
# formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
|
||||
# handler_log = logging.handlers.TimedRotatingFileHandler(log_filename, when="midnight", interval=1)
|
||||
# handler_log.suffix = '%Y-%m-%d.log'
|
||||
# handler_log.setFormatter(formatter)
|
||||
# handler_log.setLevel(30)
|
||||
# logger.addHandler(handler_log)
|
||||
# logger.setLevel(30)
|
||||
|
||||
# ========= =========#
|
||||
|
||||
|
|
@ -226,7 +226,7 @@ def login():
|
|||
# login failed
|
||||
else:
|
||||
# set brute force protection
|
||||
logger.warning("Login failed, ip={}, username={}".format(current_ip, username))
|
||||
#logger.warning("Login failed, ip={}, username={}".format(current_ip, username))
|
||||
r_cache.incr('failed_login_ip:{}'.format(current_ip))
|
||||
r_cache.expire('failed_login_ip:{}'.format(current_ip), 300)
|
||||
r_cache.incr('failed_login_user_id:{}'.format(username))
|
||||
|
|
|
|||
|
|
@ -12,7 +12,6 @@ import sys
|
|||
|
||||
# FLASK #
|
||||
app = None
|
||||
#secret_key = 'ail-super-secret_key01C'
|
||||
|
||||
# CONFIG #
|
||||
configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg')
|
||||
|
|
@ -146,7 +145,7 @@ if HiveApi != False:
|
|||
HiveApi = False
|
||||
print('The Hive not connected')
|
||||
|
||||
# VARIABLES #
|
||||
#### VARIABLES ####
|
||||
baseUrl = cfg.get("Flask", "baseurl")
|
||||
baseUrl = baseUrl.replace('/', '')
|
||||
if baseUrl != '':
|
||||
|
|
|
|||
|
|
@ -23,6 +23,9 @@ import json
|
|||
|
||||
import Paste
|
||||
|
||||
import Import_helper
|
||||
import Tags
|
||||
|
||||
from pytaxonomies import Taxonomies
|
||||
from pymispgalaxies import Galaxies, Clusters
|
||||
|
||||
|
|
@ -108,44 +111,6 @@ def launch_submit(ltags, ltagsgalaxies, paste_content, UUID, password, isfile =
|
|||
# save UUID on disk
|
||||
r_serv_db.sadd('submitted:uuid', UUID)
|
||||
|
||||
|
||||
def addTagsVerification(tags, tagsgalaxies):
|
||||
|
||||
list_tag = tags.split(',')
|
||||
list_tag_galaxies = tagsgalaxies.split(',')
|
||||
|
||||
taxonomies = Taxonomies()
|
||||
active_taxonomies = r_serv_tags.smembers('active_taxonomies')
|
||||
|
||||
active_galaxies = r_serv_tags.smembers('active_galaxies')
|
||||
|
||||
if list_tag != ['']:
|
||||
for tag in list_tag:
|
||||
# verify input
|
||||
tax = tag.split(':')[0]
|
||||
if tax in active_taxonomies:
|
||||
if tag in r_serv_tags.smembers('active_tag_' + tax):
|
||||
pass
|
||||
else:
|
||||
return False
|
||||
else:
|
||||
return False
|
||||
|
||||
if list_tag_galaxies != ['']:
|
||||
for tag in list_tag_galaxies:
|
||||
# verify input
|
||||
gal = tag.split(':')[1]
|
||||
gal = gal.split('=')[0]
|
||||
|
||||
if gal in active_galaxies:
|
||||
if tag in r_serv_tags.smembers('active_tag_galaxies_' + gal):
|
||||
pass
|
||||
else:
|
||||
return False
|
||||
else:
|
||||
return False
|
||||
return True
|
||||
|
||||
def date_to_str(date):
|
||||
return "{0}-{1}-{2}".format(date.year, date.month, date.day)
|
||||
|
||||
|
|
@ -279,11 +244,9 @@ def hive_create_case(hive_tlp, threat_level, hive_description, hive_case_title,
|
|||
@login_required
|
||||
@login_analyst
|
||||
def PasteSubmit_page():
|
||||
#active taxonomies
|
||||
active_taxonomies = r_serv_tags.smembers('active_taxonomies')
|
||||
|
||||
#active galaxies
|
||||
active_galaxies = r_serv_tags.smembers('active_galaxies')
|
||||
# Get all active tags/galaxy
|
||||
active_taxonomies = Tags.get_active_taxonomies()
|
||||
active_galaxies = Tags.get_active_galaxies()
|
||||
|
||||
return render_template("submit_items.html",
|
||||
active_taxonomies = active_taxonomies,
|
||||
|
|
@ -301,6 +264,9 @@ def submit():
|
|||
ltagsgalaxies = request.form['tags_galaxies']
|
||||
paste_content = request.form['paste_content']
|
||||
|
||||
print(ltags)
|
||||
print(ltagsgalaxies)
|
||||
|
||||
is_file = False
|
||||
if 'file' in request.files:
|
||||
file = request.files['file']
|
||||
|
|
@ -311,12 +277,16 @@ def submit():
|
|||
submitted_tag = 'infoleak:submission="manual"'
|
||||
|
||||
#active taxonomies
|
||||
active_taxonomies = r_serv_tags.smembers('active_taxonomies')
|
||||
active_taxonomies = Tags.get_active_taxonomies()
|
||||
#active galaxies
|
||||
active_galaxies = r_serv_tags.smembers('active_galaxies')
|
||||
active_galaxies = Tags.get_active_galaxies()
|
||||
|
||||
if ltags or ltagsgalaxies:
|
||||
if not addTagsVerification(ltags, ltagsgalaxies):
|
||||
|
||||
list_tag = tags.split(',')
|
||||
list_tag_galaxies = tagsgalaxies.split(',')
|
||||
|
||||
if not Tags.is_valid_tags_taxonomies_galaxy(ltags, ltagsgalaxies):
|
||||
content = 'INVALID TAGS'
|
||||
print(content)
|
||||
return content, 400
|
||||
|
|
@ -358,7 +328,7 @@ def submit():
|
|||
|
||||
paste_content = full_path
|
||||
|
||||
launch_submit(ltags, ltagsgalaxies, paste_content, UUID, password ,True)
|
||||
Import_helper.create_import_queue(ltags, ltagsgalaxies, paste_content, UUID, password ,True)
|
||||
|
||||
return render_template("submit_items.html",
|
||||
active_taxonomies = active_taxonomies,
|
||||
|
|
@ -381,7 +351,7 @@ def submit():
|
|||
# clean file name
|
||||
#id = clean_filename(paste_name)
|
||||
|
||||
launch_submit(ltags, ltagsgalaxies, paste_content, UUID, password)
|
||||
Import_helper.create_import_queue(ltags, ltagsgalaxies, paste_content, UUID, password)
|
||||
|
||||
return render_template("submit_items.html",
|
||||
active_taxonomies = active_taxonomies,
|
||||
|
|
@ -433,10 +403,10 @@ def submit_status():
|
|||
else:
|
||||
prog = 0
|
||||
|
||||
if error == 'error:':
|
||||
isError = False
|
||||
else:
|
||||
if error:
|
||||
isError = True
|
||||
else:
|
||||
isError = False
|
||||
|
||||
if end == '0':
|
||||
end = False
|
||||
|
|
|
|||
|
|
@ -8,10 +8,13 @@
|
|||
import os
|
||||
import re
|
||||
import sys
|
||||
import uuid
|
||||
import json
|
||||
import redis
|
||||
import datetime
|
||||
|
||||
import Import_helper
|
||||
|
||||
from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response
|
||||
from flask_login import login_required
|
||||
|
||||
|
|
@ -20,6 +23,7 @@ from functools import wraps
|
|||
# ============ VARIABLES ============
|
||||
import Flask_config
|
||||
|
||||
|
||||
app = Flask_config.app
|
||||
cfg = Flask_config.cfg
|
||||
baseUrl = Flask_config.baseUrl
|
||||
|
|
@ -108,8 +112,20 @@ def authErrors(user_role):
|
|||
else:
|
||||
return None
|
||||
|
||||
# ============ API CORE =============
|
||||
|
||||
|
||||
|
||||
# ============ FUNCTIONS ============
|
||||
|
||||
def is_valid_uuid_v4(header_uuid):
|
||||
try:
|
||||
header_uuid=header_uuid.replace('-', '')
|
||||
uuid_test = uuid.UUID(hex=header_uuid, version=4)
|
||||
return uuid_test.hex == header_uuid
|
||||
except:
|
||||
return False
|
||||
|
||||
def one():
|
||||
return 1
|
||||
|
||||
|
|
@ -127,5 +143,57 @@ def items():
|
|||
|
||||
return Response(json.dumps({'test': 2}), mimetype='application/json')
|
||||
|
||||
|
||||
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
||||
#
|
||||
# POST JSON FORMAT
|
||||
#
|
||||
# {
|
||||
# "type": "text", (default value)
|
||||
# "tags": [], (default value)
|
||||
# "default_ags": True, (default value)
|
||||
# "galaxy" [], (default value)
|
||||
# "text": "", mandatory if type = text
|
||||
# }
|
||||
#
|
||||
# response: {"uuid": "uuid"}
|
||||
#
|
||||
# # # #
|
||||
# GET
|
||||
#
|
||||
# {
|
||||
# "uuid": "uuid", mandatory
|
||||
# }
|
||||
#
|
||||
# response: {"uuid": "uuid"}
|
||||
#
|
||||
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
||||
|
||||
@restApi.route("api/import/item", methods=['POST'])
|
||||
@token_required('admin')
|
||||
def import_item():
|
||||
data = request.get_json()
|
||||
if not data:
|
||||
return Response(json.dumps({'status': 'error', 'reason': 'Malformed JSON'}, indent=2, sort_keys=True), mimetype='application/json'), 400
|
||||
|
||||
# TODO: add submitted tag
|
||||
|
||||
UUID = 'uuuuuuu'
|
||||
|
||||
return Response(json.dumps({'uuid': UUID}, indent=2, sort_keys=True), mimetype='application/json')
|
||||
|
||||
@restApi.route("api/import/item/<UUID>", methods=['GET'])
|
||||
@token_required('admin')
|
||||
def import_item_uuid(UUID):
|
||||
|
||||
# Verify uuid
|
||||
if not is_valid_uuid_v4(UUID):
|
||||
Response(json.dumps({'status': 'error', 'reason': 'Invalid uuid'}), mimetype='application/json'), 400
|
||||
|
||||
|
||||
|
||||
|
||||
return Response(json.dumps({'item_id': 4}), mimetype='application/json')
|
||||
|
||||
# ========= REGISTRATION =========
|
||||
app.register_blueprint(restApi, url_prefix=baseUrl)
|
||||
|
|
|
|||
Loading…
Reference in New Issue