CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

fix: [api] fix token length

pull/594/head
Terrtia 2023-04-21 16:16:10 +02:00
parent e0899e6e0d
commit 3f8656a835
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
3 changed files with 105 additions and 134 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
bin/lib/objects/Items.py
View File

@ -406,9 +406,43 @@ def _manual_set_items_date_first_last():
if last != 0: if last != 0:
update_obj_date(last, 'item') update_obj_date(last, 'item')
#### API ####
def api_get_item(data):
item_id = data.get('id', None)
if not item_id:
return {'status': 'error', 'reason': 'Mandatory parameter(s) not provided'}, 400
item = Item(item_id)
if not item.exists():
return {'status': 'error', 'reason': 'Item not found'}, 404
options = set()
if data.get('content'):
options.add('content')
if data.get('crawler'):
options.add('crawler')
if data.get('duplicates'):
options.add('duplicates')
if data.get('lines'):
options.add('lines')
if data.get('mimetype'):
options.add('mimetype')
if data.get('parent'):
options.add('parent')
if data.get('size'):
options.add('size')
# TODO correlation
return item.get_meta(options=options), 200
# -- API -- #
################################################################################ ################################################################################
################################################################################ ################################################################################
################################################################################ ################################################################################
# TODO # TODO
def exist_item(item_id): def exist_item(item_id):

66
doc/README.md
View File

@ -1239,64 +1239,36 @@ curl -k https://127.0.0.1:7000/api/v1/get/import/item --header "Authorization: i
# FUTURE endpoints # FUTURE endpoints
### Text search by daterange <details>
##### ``api/search/textIndexer/item`` POST <summary>Endpoints</summary>
### Get tagged items by daterange ### Submit a domain to crawl TODO
##### ``api/search/tag/item`` POST ##### ``api/add/crawler/task`` POST
### Submit a domain to crawl ### Create a term/set/regex/yara tracker
##### ``api/add/crawler/domain`` POST ##### ``api/add/tracker/`` POST
### Create a term/set/regex tracker ### Get tracker
##### ``api/add/termTracker/`` POST ##### ``api/get/tracker`` POST
### Get tracker items list
##### ``api/get/termTracker/item`` POST
----- -----
### Check if a tor/regular domain have been crawled
##### ``api/get/crawler/domain/`` POST
### Check if a tor/regular domain have been crawled
##### ``api/get/crawler/domain/metadata/ <domain><port>`` POST
### Get domain tags ### Get domain tags
##### ``api/get/crawler/domain/tag/ <domain><port>`` POST ##### ``api/get/domain/tags/<domain>`` POST
### Get domain history ### Get domain history
##### ``api/get/crawler/domain/history/ <domain><port>`` POST ##### ``api/get/domain/history/<domain>`` POST
### Get domain list of items
##### ``api/get/crawler/domain/item/ <domain><port>`` POST
----- -----
### Create auto-crawlers
##### ``api/add/crawler/autoCrawler/`` POST
-----
### get item by mime type/ decoded type
##### ``api/get/decoded`` POST
### Check if a decoded item exists (via sha1)
##### ``api/get/decoded/exist/<sha1>`` POST
### Get decoded item metadata ### Get decoded item metadata
### Check if a decoded item exists (via sha1) ### Check if a decoded item exists (via sha1)
##### ``api/get/decoded/metadata/<sha1>`` POST ##### ``api/get/decoded/metadata/<sha1>`` POST
### Get decoded item correlation (1 depth)
##### ``api/get/decoded/metadata/<sha1>`` POST
----- -----
@ -1304,18 +1276,20 @@ curl -k https://127.0.0.1:7000/api/v1/get/import/item --header "Authorization: i
##### ``api/get/cryptocurrency`` POST ##### ``api/get/cryptocurrency`` POST
### Check if a cryptocurrency address (bitcoin, ..) exists ### Check if a cryptocurrency address (bitcoin, ..) exists
##### ``api/get/cryptocurrency/exist/<bitcoin_address>`` POST ##### ``api/get/cryptocurrency/<bitcoin_address>`` POST
### Get cryptocurrency address metadata ### Get cryptocurrency address metadata
##### ``api/get/cryptocurrency/metadata/<bitcoin_address>`` POST ##### ``api/get/cryptocurrency/metadata/<coin_address>`` POST
----- -----
### Item correlation (1 depth) ### Object correlation (1 depth)
##### ``api/get/item/correlation/`` POST ##### ``api/get/correlation/`` POST
### Create MISP event from item ### Create MISP event from object
##### ``api/export/item/misp`` POST ##### ``api/export/misp`` POST
</details>
-----
### Create TheHive case from item
##### ``api/export/item/thehive`` POST

135
var/www/modules/restApi/Flask_restApi.py
View File

@ -10,38 +10,42 @@ import re
import sys import sys
import uuid import uuid
import json import json
import datetime
sys.path.append(os.environ['AIL_BIN']) sys.path.append(os.environ['AIL_BIN'])
################################## ##################################
# Import Project packages # Import Project packages
################################## ##################################
from lib.ConfigLoader import ConfigLoader
from lib import crawlers from lib import crawlers
from lib import Users from lib import Users
from lib.objects.Items import Item from lib.objects import Items
from lib import Tag from lib import Tag
from lib import Tracker from lib import Tracker
from packages import Term from packages import Term # TODO REMOVE ME
from packages import Import_helper from packages import Import_helper
from importer.FeederImporter import api_add_json_feeder_to_queue from importer.FeederImporter import api_add_json_feeder_to_queue
from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response, escape from flask import jsonify, request, Blueprint, redirect, url_for, Response, escape
from flask_login import login_required
from functools import wraps from functools import wraps
# ============ VARIABLES ============ # ============ VARIABLES ============
config_loader = ConfigLoader()
baseUrl = config_loader.get_config_str("Flask", "baseurl")
baseUrl = baseUrl.replace('/', '')
if baseUrl != '':
baseUrl = '/' + baseUrl
r_cache = config_loader.get_redis_conn("Redis_Cache")
config_loader = None
import Flask_config import Flask_config
app = Flask_config.app app = Flask_config.app
baseUrl = Flask_config.baseUrl
r_cache = Flask_config.r_cache
restApi = Blueprint('restApi', __name__, template_folder='templates') restApi = Blueprint('restApi', __name__, template_folder='templates')
@ -51,7 +55,7 @@ def check_token_format(token, search=re.compile(r'[^a-zA-Z0-9_-]').search):
return not bool(search(token)) return not bool(search(token))
def verify_token(token): def verify_token(token):
if len(token) != 41: if len(token) != 55:
return False return False
if not check_token_format(token): if not check_token_format(token):
@ -76,7 +80,7 @@ def token_required(user_role):
def actual_decorator(funct): def actual_decorator(funct):
@wraps(funct) @wraps(funct)
def api_token(*args, **kwargs): def api_token(*args, **kwargs):
data = authErrors(user_role) data = auth_errors(user_role)
if data: if data:
return Response(json.dumps(data[0], indent=2, sort_keys=True), mimetype='application/json'), data[1] return Response(json.dumps(data[0], indent=2, sort_keys=True), mimetype='application/json'), data[1]
else: else:
@ -88,7 +92,7 @@ def get_auth_from_header():
token = request.headers.get('Authorization').replace(' ', '') # remove space token = request.headers.get('Authorization').replace(' ', '') # remove space
return token return token
def authErrors(user_role): def auth_errors(user_role):
# Check auth # Check auth
if not request.headers.get('Authorization'): if not request.headers.get('Authorization'):
return {'status': 'error', 'reason': 'Authentication needed'}, 401 return {'status': 'error', 'reason': 'Authentication needed'}, 401
@ -98,16 +102,18 @@ def authErrors(user_role):
# brute force protection # brute force protection
current_ip = request.remote_addr current_ip = request.remote_addr
login_failed_ip = r_cache.get('failed_login_ip_api:{}'.format(current_ip)) login_failed_ip = r_cache.get(f'failed_login_ip_api:{current_ip}')
# brute force by ip # brute force by ip
if login_failed_ip: if login_failed_ip:
login_failed_ip = int(login_failed_ip) login_failed_ip = int(login_failed_ip)
if login_failed_ip >= 5: if login_failed_ip >= 5:
return {'status': 'error', 'reason': 'Max Connection Attempts reached, Please wait {}s'.format(r_cache.ttl('failed_login_ip_api:{}'.format(current_ip)))}, 401 return {'status': 'error',
'reason': 'Max Connection Attempts reached, Please wait {}s'.format(r_cache.ttl('failed_login_ip_api:{}'.format(current_ip)))
}, 401
try: try:
authenticated = False authenticated = False
if verify_token(token): if verify_token(token): # TODO Improve Returned error
authenticated = True authenticated = True
# check user role # check user role
@ -115,8 +121,8 @@ def authErrors(user_role):
data = ({'status': 'error', 'reason': 'Access Forbidden'}, 403) data = ({'status': 'error', 'reason': 'Access Forbidden'}, 403)
if not authenticated: if not authenticated:
r_cache.incr('failed_login_ip_api:{}'.format(current_ip)) r_cache.incr(f'failed_login_ip_api:{current_ip}')
r_cache.expire('failed_login_ip_api:{}'.format(current_ip), 300) r_cache.expire(f'failed_login_ip_api:{current_ip}', 300)
data = ({'status': 'error', 'reason': 'Authentication failed'}, 401) data = ({'status': 'error', 'reason': 'Authentication failed'}, 401)
except Exception as e: except Exception as e:
print(e) print(e)
@ -160,7 +166,6 @@ def is_valid_uuid_v4(header_uuid):
# { # {
# "id": item_id, mandatory # "id": item_id, mandatory
# "content": true, # "content": true,
# "tags": true,
# #
# #
# } # }
@ -175,17 +180,7 @@ def is_valid_uuid_v4(header_uuid):
@token_required('read_only') @token_required('read_only')
def get_item_id(): def get_item_id():
data = request.get_json() data = request.get_json()
res = Item.get_item(data) res = Items.api_get_item(data)
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
@restApi.route("api/v1/get/item/default", methods=['POST'])
@token_required('read_only')
def get_item_id_basic():
data = request.get_json()
item_id = data.get('id', None)
req_data = {'id': item_id, 'date': True, 'content': True, 'tags': True}
res = Item.get_item(req_data)
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
@ -197,6 +192,7 @@ def get_item_id_basic():
# #
# response: { # response: {
# "id": "item_id", # "id": "item_id",
# "date": "date",
# "tags": [], # "tags": [],
# } # }
# #
@ -204,7 +200,6 @@ def get_item_id_basic():
@restApi.route("api/v1/get/item/tag", methods=['POST']) @restApi.route("api/v1/get/item/tag", methods=['POST'])
@token_required('read_only') @token_required('read_only')
def get_item_tag(): def get_item_tag():
data = request.get_json() data = request.get_json()
item_id = data.get('id', None) item_id = data.get('id', None)
req_data = {'id': item_id, 'date': False, 'tags': True} req_data = {'id': item_id, 'date': False, 'tags': True}
@ -309,17 +304,6 @@ def get_item_sources():
res = Item.api_get_items_sources() res = Item.api_get_items_sources()
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
# @restApi.route("api/v1/get/item/source/check", methods=['POST'])
# @token_required('read_only')
# def get_check_item_source():
# data = request.get_json()
# source = data.get('source', None)
# req_data = {'source': source}
# res = Item.check_item_source(req_data)
# return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# # # # # # # # # # # # # # TAGS # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # TAGS # # # # # # # # # # # # # # # # #
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
@ -390,6 +374,8 @@ def get_tracker_metadata_api():
res = Tracker.get_tracker_metadata_api(req_data) res = Tracker.get_tracker_metadata_api(req_data)
return Response(json.dumps(res[0], indent=2, sort_keys=False), mimetype='application/json'), res[1] return Response(json.dumps(res[0], indent=2, sort_keys=False), mimetype='application/json'), res[1]
'''
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# # # # # # # # # # # # CRYPTOCURRENCY # # # # # # # # # # # # # # # # # # # # # # # # # # CRYPTOCURRENCY # # # # # # # # # # # # # #
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
@ -476,8 +462,6 @@ def get_pgp_name_item():
res = 0 res = 0
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
'''
@restApi.route("api/v1/get/item/cryptocurrency/key", methods=['POST']) @restApi.route("api/v1/get/item/cryptocurrency/key", methods=['POST'])
@ -511,26 +495,20 @@ def get_item_cryptocurrency_bitcoin():
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# # # # # # # # # # # # # # CRAWLER # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # CRAWLER # # # # # # # # # # # # # # # #
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
@restApi.route("api/v1/crawl/domain", methods=['POST']) # # TODO: ADD RESULT JSON Response
@restApi.route("api/v1/add/crawler/task", methods=['POST'])
@token_required('analyst') @token_required('analyst')
def crawl_domain(): def add_crawler_task():
data = request.get_json() data = request.get_json()
user_token = get_auth_from_header()
url = data.get('url', None) user_id = Users.get_token_user(user_token)
screenshot = data.get('screenshot', None) res = crawlers.api_add_crawler_task(data, user_id=user_id)
har = data.get('har', None) if res:
depth_limit = data.get('depth_limit', None)
max_pages = data.get('max_pages', None)
auto_crawler = data.get('auto_crawler', None)
crawler_delta = data.get('crawler_delta', None)
crawler_type = data.get('url', None)
cookiejar_uuid = data.get('url', None)
user_agent = data.get('url', None)
res = crawlers.api_create_crawler_task(json_dict)
res[0]['domain'] = domain
return create_json_response(res[0], res[1]) return create_json_response(res[0], res[1])
dict_res = {'url': data['url']}
return create_json_response(dict_res, 200)
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# # # # # # # # # # # # # # DOMAIN # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # DOMAIN # # # # # # # # # # # # # # # #
@ -574,27 +552,11 @@ def get_crawled_domain_list():
dict_res['domain_type'] = domain_type dict_res['domain_type'] = domain_type
return create_json_response(dict_res, res[1]) return create_json_response(dict_res, res[1])
# # TODO: ADD RESULT JSON Response
@restApi.route("api/v1/add/crawler/task", methods=['POST'])
@token_required('analyst')
def add_crawler_task():
data = request.get_json()
user_token = get_auth_from_header()
user_id = get_user_from_token(user_token)
res = crawlers.api_add_crawler_task(data, user_id=user_id)
if res:
return create_json_response(res[0], res[1])
dict_res = {'url': data['url']}
return create_json_response(dict_res, 200)
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# # # # # # # # # # # # # IMPORT # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # IMPORT # # # # # # # # # # # # # # # # # #
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# #
# POST JSON FORMAT # POST JSON FORMAT
@ -616,12 +578,12 @@ def import_item():
data = request.get_json() data = request.get_json()
if not data: if not data:
return Response(json.dumps({'status': 'error', 'reason': 'Malformed JSON'}, indent=2, sort_keys=True), mimetype='application/json'), 400 return create_json_response({'status': 'error', 'reason': 'Malformed JSON'}, 400)
# unpack json # unpack json
text_to_import = data.get('text', None) text_to_import = data.get('text', None)
if not text_to_import: if not text_to_import:
return Response(json.dumps({'status': 'error', 'reason': 'No text supplied'}, indent=2, sort_keys=True), mimetype='application/json'), 400 return create_json_response({'status': 'error', 'reason': 'No text supplied'}, 400)
tags = data.get('tags', []) tags = data.get('tags', [])
if not type(tags) is list: if not type(tags) is list:
@ -631,19 +593,19 @@ def import_item():
galaxy = [] galaxy = []
if not Tag.is_valid_tags_taxonomies_galaxy(tags, galaxy): if not Tag.is_valid_tags_taxonomies_galaxy(tags, galaxy):
return Response(json.dumps({'status': 'error', 'reason': 'Tags or Galaxy not enabled'}, indent=2, sort_keys=True), mimetype='application/json'), 400 return create_json_response({'status': 'error', 'reason': 'Tags or Galaxy not enabled'}, 400)
default_tags = data.get('default_tags', True) default_tags = data.get('default_tags', True)
if default_tags: if default_tags:
tags.append('infoleak:submission="manual"') tags.append('infoleak:submission="manual"')
if sys.getsizeof(text_to_import) > 900000: if sys.getsizeof(text_to_import) > 900000:
return Response(json.dumps({'status': 'error', 'reason': 'Size exceeds default'}, indent=2, sort_keys=True), mimetype='application/json'), 413 return create_json_response({'status': 'error', 'reason': 'Size exceeds default'}, 413)
UUID = str(uuid.uuid4()) import_uuid = str(uuid.uuid4())
Import_helper.create_import_queue(tags, galaxy, text_to_import, UUID) Import_helper.create_import_queue(tags, galaxy, text_to_import, import_uuid)
return Response(json.dumps({'uuid': UUID}, indent=2, sort_keys=True), mimetype='application/json') return Response(json.dumps({'uuid': import_uuid}, indent=2, sort_keys=True), mimetype='application/json')
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# GET # GET
@ -662,13 +624,13 @@ def import_item():
@token_required('analyst') @token_required('analyst')
def import_item_uuid(): def import_item_uuid():
data = request.get_json() data = request.get_json()
UUID = data.get('uuid', None) import_uuid = data.get('uuid', None)
# Verify uuid # Verify uuid
if not is_valid_uuid_v4(UUID): if not is_valid_uuid_v4(import_uuid):
return Response(json.dumps({'status': 'error', 'reason': 'Invalid uuid'}), mimetype='application/json'), 400 return Response(json.dumps({'status': 'error', 'reason': 'Invalid uuid'}), mimetype='application/json'), 400
data = Import_helper.check_import_status(UUID) data = Import_helper.check_import_status(import_uuid)
if data: if data:
return Response(json.dumps(data[0]), mimetype='application/json'), data[1] return Response(json.dumps(data[0]), mimetype='application/json'), data[1]
@ -700,5 +662,6 @@ def import_json_item():
def v1_ping(): def v1_ping():
return Response(json.dumps({'status': 'pong'}), mimetype='application/json'), 200 return Response(json.dumps({'status': 'pong'}), mimetype='application/json'), 200
# ========= REGISTRATION ========= # ========= REGISTRATION =========
app.register_blueprint(restApi, url_prefix=baseUrl) app.register_blueprint(restApi, url_prefix=baseUrl)