From 3a8531cafad319a1076cac47574bb6d68d44700c Mon Sep 17 00:00:00 2001 From: Terrtia Date: Thu, 25 Jul 2019 17:26:32 +0200 Subject: [PATCH] chg: [API + import] add API format + item_import refractor --- OVERVIEW.md | 15 ++++ bin/packages/Import_helper.py | 34 +++++++++ bin/packages/Tags.py | 61 ++++++++++++++++ bin/submit_paste.py | 3 - var/www/Flask_server.py | 20 +++--- var/www/modules/Flask_config.py | 3 +- .../modules/PasteSubmit/Flask_PasteSubmit.py | 72 ++++++------------- var/www/modules/restApi/Flask_restApi.py | 68 ++++++++++++++++++ 8 files changed, 210 insertions(+), 66 deletions(-) create mode 100755 bin/packages/Import_helper.py create mode 100755 bin/packages/Tags.py diff --git a/OVERVIEW.md b/OVERVIEW.md index 38ac7e7f..ee553848 100644 --- a/OVERVIEW.md +++ b/OVERVIEW.md @@ -38,6 +38,21 @@ Redis and ARDB overview | failed_login_ip:**ip** | **nb login failed** | TTL | failed_login_user_id:**user_id** | **nb login failed** | TTL +##### Item Import: + +| Key | Value | +| ------ | ------ | +| **uuid**:nb_total | **nb total** | TTL *(if imported)* +| **uuid**:nb_end | **nb** | TTL *(if imported)* +| **uuid**:nb_sucess | **nb success** | TTL *(if imported)* +| **uuid**:end | **0 (in progress) or (item imported)** | TTL *(if imported)* +| **uuid**:processing | **process status: 0 or 1** | TTL *(if imported)* +| **uuid**:error | **error message** | TTL *(if imported)* + +| Set Key | Value | +| ------ | ------ | +| **uuid**:paste_submit_link | **item_path** | TTL *(if imported)* + ## DB0 - Core: ##### Update keys: diff --git a/bin/packages/Import_helper.py b/bin/packages/Import_helper.py new file mode 100755 index 00000000..85a8b0d5 --- /dev/null +++ b/bin/packages/Import_helper.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +import os +import redis + +import Flask_config + +r_serv_db = Flask_config.r_serv_db +r_serv_log = Flask_config.r_serv_log + +def create_import_queue(ltags, ltagsgalaxies, paste_content, UUID, password, isfile = False): + + # save temp value on disk + r_serv_db.set(UUID + ':ltags', ltags) + r_serv_db.set(UUID + ':ltagsgalaxies', ltagsgalaxies) + r_serv_db.set(UUID + ':paste_content', paste_content) + r_serv_db.set(UUID + ':password', password) + r_serv_db.set(UUID + ':isfile', isfile) + + r_serv_log.set(UUID + ':end', 0) + r_serv_log.set(UUID + ':processing', 0) + r_serv_log.set(UUID + ':nb_total', -1) + r_serv_log.set(UUID + ':nb_end', 0) + r_serv_log.set(UUID + ':nb_sucess', 0) + + # save UUID on disk + r_serv_db.sadd('submitted:uuid', UUID) + return UUID + +def import_text_item(): + res = r_serv_db.smembers('submitted:uuid') + print(res) + return res diff --git a/bin/packages/Tags.py b/bin/packages/Tags.py new file mode 100755 index 00000000..d916a29d --- /dev/null +++ b/bin/packages/Tags.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +import os +import redis + +import Flask_config + +from pytaxonomies import Taxonomies +from pymispgalaxies import Galaxies, Clusters + +r_serv_tags = Flask_config.r_serv_tags + +def get_taxonomie_from_tag(tag): + return tag.split(':')[0] + +def get_galaxy_from_tag(tag): + galaxy = tag.split(':')[1] + galaxy = galaxy.split('=')[0] + return galaxy + +def get_active_taxonomies(): + return r_serv_tags.smembers('active_taxonomies') + +def get_active_galaxies(): + return r_serv_tags.smembers('active_galaxies') + +def is_taxonomie_tag_enabled(taxonomie, tag): + if tag in r_serv_tags.smembers('active_tag_' + taxonomie): + return True + else: + return False + +def is_galaxy_tag_enabled(taxonomie, galaxy): + if tag in r_serv_tags.smembers('active_tag_galaxies_' + galaxy): + return True + else: + return False + +# Check if tags are enabled in AIL +def is_valid_tags_taxonomies_galaxy(list_tags, list_tags_galaxy): + if list_tags: + active_taxonomies = Tags.get_active_taxonomies() + + for tag in list_tags: + taxonomie = get_taxonomie_from_tag(tag) + if taxonomie not in active_taxonomies: + return False + if not is_taxonomie_tag_enabled(taxonomie, tag): + return False + + if list_tags_galaxy: + active_galaxies = Tags.get_active_galaxies() + + for tag in list_tags_galaxy: + galaxy = get_galaxy_from_tag(tag) + if galaxy not in active_galaxies: + return False + if not is_galaxy_tag_enabled(galaxy, tag): + return False + return True diff --git a/bin/submit_paste.py b/bin/submit_paste.py index 34009774..e6875b3b 100755 --- a/bin/submit_paste.py +++ b/bin/submit_paste.py @@ -92,7 +92,6 @@ def remove_submit_uuid(uuid): r_serv_log_submit.expire(uuid + ':nb_sucess', expire_time) r_serv_log_submit.expire(uuid + ':nb_end', expire_time) r_serv_log_submit.expire(uuid + ':error', expire_time) - r_serv_log_submit.srem(uuid + ':paste_submit_link', '') r_serv_log_submit.expire(uuid + ':paste_submit_link', expire_time) # delete uuid @@ -230,8 +229,6 @@ if __name__ == "__main__": r_serv_log_submit.set(uuid + ':nb_total', -1) r_serv_log_submit.set(uuid + ':nb_end', 0) r_serv_log_submit.set(uuid + ':nb_sucess', 0) - r_serv_log_submit.set(uuid + ':error', 'error:') - r_serv_log_submit.sadd(uuid + ':paste_submit_link', '') r_serv_log_submit.set(uuid + ':processing', 1) diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py index 8ba4526e..ab22ffd1 100755 --- a/var/www/Flask_server.py +++ b/var/www/Flask_server.py @@ -67,15 +67,15 @@ log_dir = os.path.join(os.environ['AIL_HOME'], 'logs') if not os.path.isdir(log_dir): os.makedirs(logs_dir) -log_filename = os.path.join(log_dir, 'flask_server.logs') -logger = logging.getLogger() -formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') -handler_log = logging.handlers.TimedRotatingFileHandler(log_filename, when="midnight", interval=1) -handler_log.suffix = '%Y-%m-%d.log' -handler_log.setFormatter(formatter) -handler_log.setLevel(30) -logger.addHandler(handler_log) -logger.setLevel(30) +# log_filename = os.path.join(log_dir, 'flask_server.logs') +# logger = logging.getLogger() +# formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') +# handler_log = logging.handlers.TimedRotatingFileHandler(log_filename, when="midnight", interval=1) +# handler_log.suffix = '%Y-%m-%d.log' +# handler_log.setFormatter(formatter) +# handler_log.setLevel(30) +# logger.addHandler(handler_log) +# logger.setLevel(30) # ========= =========# @@ -226,7 +226,7 @@ def login(): # login failed else: # set brute force protection - logger.warning("Login failed, ip={}, username={}".format(current_ip, username)) + #logger.warning("Login failed, ip={}, username={}".format(current_ip, username)) r_cache.incr('failed_login_ip:{}'.format(current_ip)) r_cache.expire('failed_login_ip:{}'.format(current_ip), 300) r_cache.incr('failed_login_user_id:{}'.format(username)) diff --git a/var/www/modules/Flask_config.py b/var/www/modules/Flask_config.py index ff5ba02a..0e0d0e8b 100644 --- a/var/www/modules/Flask_config.py +++ b/var/www/modules/Flask_config.py @@ -12,7 +12,6 @@ import sys # FLASK # app = None -#secret_key = 'ail-super-secret_key01C' # CONFIG # configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg') @@ -146,7 +145,7 @@ if HiveApi != False: HiveApi = False print('The Hive not connected') -# VARIABLES # +#### VARIABLES #### baseUrl = cfg.get("Flask", "baseurl") baseUrl = baseUrl.replace('/', '') if baseUrl != '': diff --git a/var/www/modules/PasteSubmit/Flask_PasteSubmit.py b/var/www/modules/PasteSubmit/Flask_PasteSubmit.py index efd0650e..11e405a7 100644 --- a/var/www/modules/PasteSubmit/Flask_PasteSubmit.py +++ b/var/www/modules/PasteSubmit/Flask_PasteSubmit.py @@ -23,6 +23,9 @@ import json import Paste +import Import_helper +import Tags + from pytaxonomies import Taxonomies from pymispgalaxies import Galaxies, Clusters @@ -108,44 +111,6 @@ def launch_submit(ltags, ltagsgalaxies, paste_content, UUID, password, isfile = # save UUID on disk r_serv_db.sadd('submitted:uuid', UUID) - -def addTagsVerification(tags, tagsgalaxies): - - list_tag = tags.split(',') - list_tag_galaxies = tagsgalaxies.split(',') - - taxonomies = Taxonomies() - active_taxonomies = r_serv_tags.smembers('active_taxonomies') - - active_galaxies = r_serv_tags.smembers('active_galaxies') - - if list_tag != ['']: - for tag in list_tag: - # verify input - tax = tag.split(':')[0] - if tax in active_taxonomies: - if tag in r_serv_tags.smembers('active_tag_' + tax): - pass - else: - return False - else: - return False - - if list_tag_galaxies != ['']: - for tag in list_tag_galaxies: - # verify input - gal = tag.split(':')[1] - gal = gal.split('=')[0] - - if gal in active_galaxies: - if tag in r_serv_tags.smembers('active_tag_galaxies_' + gal): - pass - else: - return False - else: - return False - return True - def date_to_str(date): return "{0}-{1}-{2}".format(date.year, date.month, date.day) @@ -279,11 +244,9 @@ def hive_create_case(hive_tlp, threat_level, hive_description, hive_case_title, @login_required @login_analyst def PasteSubmit_page(): - #active taxonomies - active_taxonomies = r_serv_tags.smembers('active_taxonomies') - - #active galaxies - active_galaxies = r_serv_tags.smembers('active_galaxies') + # Get all active tags/galaxy + active_taxonomies = Tags.get_active_taxonomies() + active_galaxies = Tags.get_active_galaxies() return render_template("submit_items.html", active_taxonomies = active_taxonomies, @@ -301,6 +264,9 @@ def submit(): ltagsgalaxies = request.form['tags_galaxies'] paste_content = request.form['paste_content'] + print(ltags) + print(ltagsgalaxies) + is_file = False if 'file' in request.files: file = request.files['file'] @@ -311,12 +277,16 @@ def submit(): submitted_tag = 'infoleak:submission="manual"' #active taxonomies - active_taxonomies = r_serv_tags.smembers('active_taxonomies') + active_taxonomies = Tags.get_active_taxonomies() #active galaxies - active_galaxies = r_serv_tags.smembers('active_galaxies') + active_galaxies = Tags.get_active_galaxies() if ltags or ltagsgalaxies: - if not addTagsVerification(ltags, ltagsgalaxies): + + list_tag = tags.split(',') + list_tag_galaxies = tagsgalaxies.split(',') + + if not Tags.is_valid_tags_taxonomies_galaxy(ltags, ltagsgalaxies): content = 'INVALID TAGS' print(content) return content, 400 @@ -358,7 +328,7 @@ def submit(): paste_content = full_path - launch_submit(ltags, ltagsgalaxies, paste_content, UUID, password ,True) + Import_helper.create_import_queue(ltags, ltagsgalaxies, paste_content, UUID, password ,True) return render_template("submit_items.html", active_taxonomies = active_taxonomies, @@ -381,7 +351,7 @@ def submit(): # clean file name #id = clean_filename(paste_name) - launch_submit(ltags, ltagsgalaxies, paste_content, UUID, password) + Import_helper.create_import_queue(ltags, ltagsgalaxies, paste_content, UUID, password) return render_template("submit_items.html", active_taxonomies = active_taxonomies, @@ -433,10 +403,10 @@ def submit_status(): else: prog = 0 - if error == 'error:': - isError = False - else: + if error: isError = True + else: + isError = False if end == '0': end = False diff --git a/var/www/modules/restApi/Flask_restApi.py b/var/www/modules/restApi/Flask_restApi.py index f73434de..07e3240f 100644 --- a/var/www/modules/restApi/Flask_restApi.py +++ b/var/www/modules/restApi/Flask_restApi.py @@ -8,10 +8,13 @@ import os import re import sys +import uuid import json import redis import datetime +import Import_helper + from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response from flask_login import login_required @@ -20,6 +23,7 @@ from functools import wraps # ============ VARIABLES ============ import Flask_config + app = Flask_config.app cfg = Flask_config.cfg baseUrl = Flask_config.baseUrl @@ -108,8 +112,20 @@ def authErrors(user_role): else: return None +# ============ API CORE ============= + + + # ============ FUNCTIONS ============ +def is_valid_uuid_v4(header_uuid): + try: + header_uuid=header_uuid.replace('-', '') + uuid_test = uuid.UUID(hex=header_uuid, version=4) + return uuid_test.hex == header_uuid + except: + return False + def one(): return 1 @@ -127,5 +143,57 @@ def items(): return Response(json.dumps({'test': 2}), mimetype='application/json') + +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +# +# POST JSON FORMAT +# +# { +# "type": "text", (default value) +# "tags": [], (default value) +# "default_ags": True, (default value) +# "galaxy" [], (default value) +# "text": "", mandatory if type = text +# } +# +# response: {"uuid": "uuid"} +# +# # # # +# GET +# +# { +# "uuid": "uuid", mandatory +# } +# +# response: {"uuid": "uuid"} +# +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # + +@restApi.route("api/import/item", methods=['POST']) +@token_required('admin') +def import_item(): + data = request.get_json() + if not data: + return Response(json.dumps({'status': 'error', 'reason': 'Malformed JSON'}, indent=2, sort_keys=True), mimetype='application/json'), 400 + + # TODO: add submitted tag + + UUID = 'uuuuuuu' + + return Response(json.dumps({'uuid': UUID}, indent=2, sort_keys=True), mimetype='application/json') + +@restApi.route("api/import/item/", methods=['GET']) +@token_required('admin') +def import_item_uuid(UUID): + + # Verify uuid + if not is_valid_uuid_v4(UUID): + Response(json.dumps({'status': 'error', 'reason': 'Invalid uuid'}), mimetype='application/json'), 400 + + + + + return Response(json.dumps({'item_id': 4}), mimetype='application/json') + # ========= REGISTRATION ========= app.register_blueprint(restApi, url_prefix=baseUrl)