diff --git a/bin/packages/Date.py b/bin/packages/Date.py index 72b960b1..85edb0be 100644 --- a/bin/packages/Date.py +++ b/bin/packages/Date.py @@ -40,3 +40,13 @@ class Date(object): comp_month = str(computed_date.month).zfill(2) comp_day = str(computed_date.day).zfill(2) return comp_year + comp_month + comp_day + +def date_add_day(date, num_day=1): + new_date = datetime.date(int(date[0:4]), int(date[4:6]), int(date[6:8])) + datetime.timedelta(num_day) + new_date = str(new_date).replace('-', '') + return new_date + +def date_substract_day(date, num_day=1): + new_date = datetime.date(int(date[0:4]), int(date[4:6]), int(date[6:8])) - datetime.timedelta(num_day) + new_date = str(new_date).replace('-', '') + return new_date diff --git a/bin/packages/Item.py b/bin/packages/Item.py index 1bc77b79..a2276fbd 100755 --- a/bin/packages/Item.py +++ b/bin/packages/Item.py @@ -14,3 +14,7 @@ def exist_item(item_id): return True else: return False + +def get_item_date(item_id): + l_directory = item_id.split('/') + return '{}{}{}'.format(l_directory[-4], l_directory[-3], l_directory[-2]) diff --git a/bin/packages/Tag.py b/bin/packages/Tag.py new file mode 100755 index 00000000..eb5a5bb3 --- /dev/null +++ b/bin/packages/Tag.py @@ -0,0 +1,210 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +import os +import redis + +import Flask_config +import Date +import Item + +from pytaxonomies import Taxonomies +from pymispgalaxies import Galaxies, Clusters + +r_serv_tags = Flask_config.r_serv_tags +r_serv_metadata = Flask_config.r_serv_metadata + +def get_taxonomie_from_tag(tag): + return tag.split(':')[0] + +def get_galaxy_from_tag(tag): + galaxy = tag.split(':')[1] + galaxy = galaxy.split('=')[0] + return galaxy + +def get_active_taxonomies(): + return r_serv_tags.smembers('active_taxonomies') + +def get_active_galaxies(): + return r_serv_tags.smembers('active_galaxies') + +def is_taxonomie_tag_enabled(taxonomie, tag): + if tag in r_serv_tags.smembers('active_tag_' + taxonomie): + return True + else: + return False + +def is_galaxy_tag_enabled(galaxy, tag): + if tag in r_serv_tags.smembers('active_tag_galaxies_' + galaxy): + return True + else: + return False + +# Check if tags are enabled in AIL +def is_valid_tags_taxonomies_galaxy(list_tags, list_tags_galaxy): + print(list_tags) + print(list_tags_galaxy) + if list_tags: + active_taxonomies = get_active_taxonomies() + + for tag in list_tags: + taxonomie = get_taxonomie_from_tag(tag) + if taxonomie not in active_taxonomies: + return False + if not is_taxonomie_tag_enabled(taxonomie, tag): + return False + + if list_tags_galaxy: + active_galaxies = get_active_galaxies() + + for tag in list_tags_galaxy: + galaxy = get_galaxy_from_tag(tag) + if galaxy not in active_galaxies: + return False + if not is_galaxy_tag_enabled(galaxy, tag): + return False + return True + +def get_item_tags(item_id): + tags = r_serv_metadata.smembers('tag:'+item_id) + if tags: + return list(tags) + else: + return '[]' + +# TEMPLATE + API QUERY +def add_items_tag(tags=[], galaxy_tags=[], item_id=None): + res_dict = {} + if item_id == None: + return ({'status': 'error', 'reason': 'Item id not found'}, 400) + if not tags and not galaxy_tags: + return ({'status': 'error', 'reason': 'Tags or Galaxy not specified'}, 400) + + res_dict['tags'] = [] + for tag in tags: + taxonomie = get_taxonomie_from_tag(tag) + if is_taxonomie_tag_enabled(taxonomie, tag): + add_item_tag(tag, item_id) + res_dict['tags'].append(tag) + else: + return ({'status': 'error', 'reason': 'Tags or Galaxy not enabled'}, 400) + + for tag in galaxy_tags: + galaxy = get_galaxy_from_tag(tag) + if is_galaxy_tag_enabled(galaxy, tag): + add_item_tag(tag, item_id) + res_dict['tags'].append(tag) + else: + return ({'status': 'error', 'reason': 'Tags or Galaxy not enabled'}, 400) + + res_dict['id'] = item_id + return (res_dict, 200) + + +def add_item_tag(tag, item_path): + + item_date = int(Item.get_item_date(item_path)) + + #add tag + r_serv_metadata.sadd('tag:{}'.format(item_path), tag) + r_serv_tags.sadd('{}:{}'.format(tag, item_date), item_path) + + r_serv_tags.hincrby('daily_tags:{}'.format(item_date), tag, 1) + + tag_first_seen = r_serv_tags.hget('tag_metadata:{}'.format(tag), 'last_seen') + if tag_first_seen is None: + tag_first_seen = 99999999 + else: + tag_first_seen = int(tag_first_seen) + tag_last_seen = r_serv_tags.hget('tag_metadata:{}'.format(tag), 'last_seen') + if tag_last_seen is None: + tag_last_seen = 0 + else: + tag_last_seen = int(tag_last_seen) + + #add new tag in list of all used tags + r_serv_tags.sadd('list_tags', tag) + + # update fisrt_seen/last_seen + if item_date < tag_first_seen: + r_serv_tags.hset('tag_metadata:{}'.format(tag), 'first_seen', item_date) + + # update metadata last_seen + if item_date > tag_last_seen: + r_serv_tags.hset('tag_metadata:{}'.format(tag), 'last_seen', item_date) + +# API QUERY +def remove_item_tags(tags=[], item_id=None): + if item_id == None: + return ({'status': 'error', 'reason': 'Item id not found'}, 400) + if not tags: + return ({'status': 'error', 'reason': 'No Tag(s) specified'}, 400) + + dict_res = {} + dict_res[tags] = [] + for tag in tags: + res = remove_item_tag(tag, item_id) + if res[1] != 200: + return res + else: + dict_res[tags].append(tag) + dict_res[id] = item_id + return (dict_res, 200) + +# TEMPLATE + API QUERY +def remove_item_tag(tag, item_id): + item_date = int(Item.get_item_date(item_id)) + + #remove tag + r_serv_metadata.srem('tag:{}'.format(item_id), tag) + res = r_serv_tags.srem('{}:{}'.format(tag, item_date), item_id) + + if res ==1: + # no tag for this day + if int(r_serv_tags.hget('daily_tags:{}'.format(item_date), tag)) == 1: + r_serv_tags.hdel('daily_tags:{}'.format(item_date), tag) + else: + r_serv_tags.hincrby('daily_tags:{}'.format(item_date), tag, -1) + + tag_first_seen = int(r_serv_tags.hget('tag_metadata:{}'.format(tag), 'last_seen')) + tag_last_seen = int(r_serv_tags.hget('tag_metadata:{}'.format(tag), 'last_seen')) + # update fisrt_seen/last_seen + if item_date == tag_first_seen: + update_tag_first_seen(tag, tag_first_seen, tag_last_seen) + if item_date == tag_last_seen: + update_tag_last_seen(tag, tag_first_seen, tag_last_seen) + return ({'status': 'success'}, 200) + else: + return ({'status': 'error', 'reason': 'Item id or tag not found'}, 400) + +def update_tag_first_seen(tag, tag_first_seen, tag_last_seen): + if tag_first_seen == tag_last_seen: + if r_serv_tags.scard('{}:{}'.format(tag, tag_first_seen)) > 0: + r_serv_tags.hset('tag_metadata:{}'.format(tag), 'first_seen', tag_first_seen) + # no tag in db + else: + r_serv_tags.srem('list_tags', tag) + r_serv_tags.hdel('tag_metadata:{}'.format(tag), 'first_seen') + r_serv_tags.hdel('tag_metadata:{}'.format(tag), 'last_seen') + else: + if r_serv_tags.scard('{}:{}'.format(tag, tag_first_seen)) > 0: + r_serv_tags.hset('tag_metadata:{}'.format(tag), 'first_seen', tag_first_seen) + else: + tag_first_seen = Date.date_add_day(tag_first_seen) + update_tag_first_seen(tag, tag_first_seen, tag_last_seen) + +def update_tag_last_seen(tag, tag_first_seen, tag_last_seen): + if tag_first_seen == tag_last_seen: + if r_serv_tags.scard('{}:{}'.format(tag, tag_last_seen)) > 0: + r_serv_tags.hset('tag_metadata:{}'.format(tag), 'last_seen', tag_last_seen) + # no tag in db + else: + r_serv_tags.srem('list_tags', tag) + r_serv_tags.hdel('tag_metadata:{}'.format(tag), 'first_seen') + r_serv_tags.hdel('tag_metadata:{}'.format(tag), 'last_seen') + else: + if r_serv_tags.scard('{}:{}'.format(tag, tag_last_seen)) > 0: + r_serv_tags.hset('tag_metadata:{}'.format(tag), 'last_seen', tag_last_seen) + else: + tag_last_seen = Date.date_substract_day(tag_last_seen) + update_tag_last_seen(tag, tag_first_seen, tag_last_seen) diff --git a/bin/packages/Tags.py b/bin/packages/Tags.py deleted file mode 100755 index d6636ca7..00000000 --- a/bin/packages/Tags.py +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/env python3 -# -*-coding:UTF-8 -* - -import os -import redis - -import Flask_config - -from pytaxonomies import Taxonomies -from pymispgalaxies import Galaxies, Clusters - -r_serv_tags = Flask_config.r_serv_tags -r_serv_metadata = Flask_config.r_serv_metadata - -def get_taxonomie_from_tag(tag): - return tag.split(':')[0] - -def get_galaxy_from_tag(tag): - galaxy = tag.split(':')[1] - galaxy = galaxy.split('=')[0] - return galaxy - -def get_active_taxonomies(): - return r_serv_tags.smembers('active_taxonomies') - -def get_active_galaxies(): - return r_serv_tags.smembers('active_galaxies') - -def is_taxonomie_tag_enabled(taxonomie, tag): - if tag in r_serv_tags.smembers('active_tag_' + taxonomie): - return True - else: - return False - -def is_galaxy_tag_enabled(galaxy, tag): - if tag in r_serv_tags.smembers('active_tag_galaxies_' + galaxy): - return True - else: - return False - -# Check if tags are enabled in AIL -def is_valid_tags_taxonomies_galaxy(list_tags, list_tags_galaxy): - print(list_tags) - print(list_tags_galaxy) - if list_tags: - active_taxonomies = get_active_taxonomies() - - for tag in list_tags: - taxonomie = get_taxonomie_from_tag(tag) - if taxonomie not in active_taxonomies: - return False - if not is_taxonomie_tag_enabled(taxonomie, tag): - return False - - if list_tags_galaxy: - active_galaxies = get_active_galaxies() - - for tag in list_tags_galaxy: - galaxy = get_galaxy_from_tag(tag) - if galaxy not in active_galaxies: - return False - if not is_galaxy_tag_enabled(galaxy, tag): - return False - return True - -def get_item_tags(item_id): - tags = r_serv_metadata.smembers('tag:'+item_id) - if tags: - return list(tags) - else: - return '[]' diff --git a/doc/README.md b/doc/README.md new file mode 100644 index 00000000..44613c26 --- /dev/null +++ b/doc/README.md @@ -0,0 +1,464 @@ +# API DOCUMENTATION + +## General + +### Automation key + +The authentication of the automation is performed via a secure key available in the AIL UI interface. Make sure you keep that key secret as it gives access to the entire database! The API key is available in the ``Server Management`` menu under ``My Profile``. + +The authorization is performed by using the following header: + +~~~~ +Authorization: YOUR API KEY +~~~~ +### Accept and Content-Type headers + +When submitting data in a POST, PUT or DELETE operation you need to specify in what content-type you encoded the payload. This is done by setting the below Content-Type headers: + +~~~~ +Content-Type: application/json +~~~~ + +Example: + +~~~~ +curl --header "Authorization: YOUR API KEY" --header "Content-Type: application/json" https:/// +~~~~ + +## Item management + +### Get item: `api/get/item/info/` + +#### Description +Get a specific item information. + +**Method** : `GET` + +#### Parameters +- `id` + - item id + - *str - relative item path* + - mandatory + +#### JSON response +- `content` + - item content + - *str* +- `id` + - item id + - *str* +- `date` + - item date + - *str - YYMMDD* +- `tags` + - item tags list + - *list* + +#### Example +``` +curl https://127.0.0.1:7000/api/get/item/info/submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json" +``` + +#### Expected Success Response +**HTTP Status Code** : `200` + +```json + { + "content": "item content test", + "date": "20190726", + "id": "submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz", + "tags": + [ + "misp-galaxy:backdoor=\"Rosenbridge\"", + "infoleak:automatic-detection=\"pgp-message\"", + "infoleak:automatic-detection=\"encrypted-private-key\"", + "infoleak:submission=\"manual\"", + "misp-galaxy:backdoor=\"SLUB\"" + ] + } +``` + +#### Expected Fail Response + +**HTTP Status Code** : `400` + +``` + {'status': 'error', 'reason': 'Item not found'} +``` + + + + +### Get item content: `api/get/item/content/` + +#### Description +Get a specific item content. + +**Method** : `GET` + +#### Parameters +- `id` + - item id + - *str - relative item path* + - mandatory + +#### JSON response +- `content` + - item content + - *str* +- `id` + - item id + - *str* + +#### Example +``` +curl https://127.0.0.1:7000/api/get/item/content/submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json" +``` + +#### Expected Success Response +**HTTP Status Code** : `200` + +```json + { + "content": "item content test", + "id": "submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz" + } +``` + +#### Expected Fail Response + +**HTTP Status Code** : `400` + +``` + {'status': 'error', 'reason': 'Item not found'} +``` + + + +### Get item content: `api/get/item/tag/` + +#### Description +Get all tags from an item. + +**Method** : `GET` + +#### Parameters +- `id` + - item id + - *str - relative item path* + - mandatory + +#### JSON response +- `content` + - item content + - *str* +- `tags` + - item tags list + - *list* + +#### Example +``` +curl https://127.0.0.1:7000/api/get/item/tag/submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json" +``` + +#### Expected Success Response +**HTTP Status Code** : `200` + +```json + { + "id": "submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz", + "tags": + [ + "misp-galaxy:backdoor=\"Rosenbridge\"", + "infoleak:automatic-detection=\"pgp-message\"", + "infoleak:automatic-detection=\"encrypted-private-key\"", + "infoleak:submission=\"manual\"", + "misp-galaxy:backdoor=\"SLUB\"" + ] + } +``` + +#### Expected Fail Response + +**HTTP Status Code** : `400` + +``` + {'status': 'error', 'reason': 'Item not found'} +``` + + + +### add item tags: `api/add/item/tag` + +#### Description +Add tags to an item. + +**Method** : `POST` + +#### Parameters +- `id` + - item id + - *str - relative item path* + - mandatory +- `tags` + - list of tags + - *list* + - default: `[]` +- `galaxy` + - list of galaxy + - *list* + - default: `[]` + +#### JSON response +- `id` + - item id + - *str - relative item path* +- `tags` + - list of item tags added + - *list* + +#### Example +``` +curl https://127.0.0.1:7000/api/import/item --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json" --data @input.json -X POST +``` + +#### input.json Example +```json + { + "id": "submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz", + "tags": [ + "infoleak:analyst-detection=\"private-key\"", + "infoleak:analyst-detection=\"api-key\"" + ], + "galaxy": [ + "misp-galaxy:stealer=\"Vidar\"" + ] + } +``` + +#### Expected Success Response +**HTTP Status Code** : `200` + +```json + { + "id": "submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz", + "tags": [ + "infoleak:analyst-detection=\"private-key\"", + "infoleak:analyst-detection=\"api-key\"", + "misp-galaxy:stealer=\"Vidar\"" + ] + } +``` + +#### Expected Fail Response +**HTTP Status Code** : `400` + +``` + {'status': 'error', 'reason': 'Item id not found'} + {'status': 'error', 'reason': 'Tags or Galaxy not specified'} + {'status': 'error', 'reason': 'Tags or Galaxy not enabled'} +``` + + + + +### Delete item tags: `api/delete/item/tag` + +#### Description +Delete tags from an item. + +**Method** : `DELETE` + +#### Parameters +- `id` + - item id + - *str - relative item path* + - mandatory +- `tags` + - list of tags + - *list* + - default: `[]` + +#### JSON response +- `id` + - item id + - *str - relative item path* +- `tags` + - list of item tags deleted + - *list* + +#### Example +``` +curl https://127.0.0.1:7000/api/delete/item/tag --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json" --data @input.json -X DELETE +``` + +#### input.json Example +```json + { + "id": "submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz", + "tags": [ + "infoleak:analyst-detection=\"private-key\"", + "infoleak:analyst-detection=\"api-key\"", + "misp-galaxy:stealer=\"Vidar\"" + ] + } +``` + +#### Expected Success Response +**HTTP Status Code** : `200` + +```json + { + "id": "submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz", + "tags": [ + "infoleak:analyst-detection=\"private-key\"", + "infoleak:analyst-detection=\"api-key\"", + "misp-galaxy:stealer=\"Vidar\"" + ] + } +``` + +#### Expected Fail Response +**HTTP Status Code** : `400` + +``` + {'status': 'error', 'reason': 'Item id not found'} + {'status': 'error', 'reason': 'No Tag(s) specified'} +``` + + + + + + + +## Import management + + + +### Import item (currently: text only): `api/import/item` + +#### Description +Allows users to import new items. asynchronous function. + +**Method** : `POST` + +#### Parameters +- `type` + - import type + - *str* + - default: `text` +- `text` + - text to import + - *str* + - mandatory if type = text +- `default_tags` + - add default import tag + - *boolean* + - default: True +- `tags` + - list of tags + - *list* + - default: `[]` +- `galaxy` + - list of galaxy + - *list* + - default: `[]` + +#### JSON response +- `uuid` + - import uuid + - *uuid4* + +#### Example +``` +curl https://127.0.0.1:7000/api/import/item --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json" --data @input.json -X POST +``` + +#### input.json Example +```json + { + "type": "text", + "tags": [ + "infoleak:analyst-detection=\"private-key\"" + ], + "text": "text to import" + } +``` + +#### Expected Success Response +**HTTP Status Code** : `200` + +```json + { + "uuid": "0c3d7b34-936e-4f01-9cdf-2070184b6016" + } +``` + +#### Expected Fail Response +**HTTP Status Code** : `400` + +``` + {'status': 'error', 'reason': 'Malformed JSON'} + {'status': 'error', 'reason': 'No text supplied'} + {'status': 'error', 'reason': 'Tags or Galaxy not enabled'} + {'status': 'error', 'reason': 'Size exceeds default'} +``` + + + + + +### GET Import item info: `api/import/item/` + +#### Description + +Get import status and all items imported by uuid + +**Method** : `GET` + +#### Parameters + +- `uuid` + - import uuid + - *uuid4* + - mandatory + +#### JSON response + +- `status` + - import status + - *str* + - values: `in queue`, `in progress`, `imported` +- `items` + - list of imported items id + - *list* + - The full list of imported items is not complete until `status` = `"imported"` + +#### Example + +``` +curl -k https://127.0.0.1:7000/api/import/item/b20a69f1-99ad-4cb3-b212-7ce24b763b50 --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json" +``` + +#### Expected Success Response + +**HTTP Status Code** : `200` + +```json + { + "items": [ + "submitted/2019/07/26/b20a69f1-99ad-4cb3-b212-7ce24b763b50.gz" + ], + "status": "imported" + } +``` + +#### Expected Fail Response + +**HTTP Status Code** : `400` + +``` + {'status': 'error', 'reason': 'Invalid uuid'} + {'status': 'error', 'reason': 'Unknow uuid'} +``` diff --git a/doc/api/submit_paste.py b/doc/api/submit_paste.py deleted file mode 100755 index 3e1e2299..00000000 --- a/doc/api/submit_paste.py +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env python3 -# -*-coding:UTF-8 -* - -''' -submit your own pastes in AIL - -empty values must be initialized -''' - -import requests - -if __name__ == '__main__': - - #AIL url - url = 'http://localhost:7000' - - ail_url = url + '/PasteSubmit/submit' - - # MIPS TAXONOMIE, need to be initialized (tags_taxonomies = '') - tags_taxonomies = 'CERT-XLM:malicious-code=\"ransomware\",CERT-XLM:conformity=\"standard\"' - - # MISP GALAXY, need to be initialized (tags_galaxies = '') - tags_galaxies = 'misp-galaxy:cert-seu-gocsector=\"Constituency\",misp-galaxy:cert-seu-gocsector=\"EU-Centric\"' - - # user paste input, need to be initialized (paste_content = '') - paste_content = 'paste content test' - - #file full or relative path - file_to_submit = 'test_file.zip' - - #compress file password, need to be initialized (password = '') - password = '' - - ''' - submit user text - ''' - r = requests.post(ail_url, data={ 'password': password, - 'paste_content': paste_content, - 'tags_taxonomies': tags_taxonomies, - 'tags_galaxies': tags_galaxies}) - print(r.status_code, r.reason) - - - ''' - submit a file - ''' - with open(file_submit,'rb') as f: - - r = requests.post(ail_url, data={ 'password': password, - 'paste_content': paste_content, - 'tags_taxonomies': tags_taxonomies, - 'tags_galaxies': tags_galaxies}, files={'file': (file_to_submit, f.read() )}) - print(r.status_code, r.reason) diff --git a/doc/dia/ZMQ_Queuing_Tree.jpg b/doc/dia/ZMQ_Queuing_Tree.jpg deleted file mode 100644 index b4297340..00000000 Binary files a/doc/dia/ZMQ_Queuing_Tree.jpg and /dev/null differ diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py index ab22ffd1..5f8d52b2 100755 --- a/var/www/Flask_server.py +++ b/var/www/Flask_server.py @@ -37,6 +37,8 @@ import Flask_config from Role_Manager import create_user_db, check_password_strength, check_user_role_integrity from Role_Manager import login_admin, login_analyst +Flask_dir = os.environ['AIL_FLASK'] + # CONFIG # cfg = Flask_config.cfg baseUrl = cfg.get("Flask", "baseurl") @@ -81,7 +83,7 @@ if not os.path.isdir(log_dir): # ========= TLS =========# ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2) -ssl_context.load_cert_chain(certfile='server.crt', keyfile='server.key') +ssl_context.load_cert_chain(certfile=os.path.join(Flask_dir, 'server.crt'), keyfile=os.path.join(Flask_dir, 'server.key')) #print(ssl_context.get_ciphers()) # ========= =========# @@ -112,13 +114,13 @@ try: toIgnoreModule.add(line) except IOError: - f = open('templates/ignored_modules.txt', 'w') + f = open(os.path.join(Flask_dir, 'templates', 'ignored_modules.txt'), 'w') f.close() # Dynamically import routes and functions from modules # Also, prepare header.html to_add_to_header_dico = {} -for root, dirs, files in os.walk('modules/'): +for root, dirs, files in os.walk(os.path.join(Flask_dir, 'modules')): sys.path.append(join(root)) # Ignore the module @@ -140,7 +142,7 @@ for root, dirs, files in os.walk('modules/'): #create header.html complete_header = "" -with open('templates/header_base.html', 'r') as f: +with open(os.path.join(Flask_dir, 'templates', 'header_base.html'), 'r') as f: complete_header = f.read() modified_header = complete_header @@ -159,7 +161,7 @@ for module_name, txt in to_add_to_header_dico.items(): modified_header = modified_header.replace('', '\n'.join(to_add_to_header)) #Write the header.html file -with open('templates/header.html', 'w') as f: +with open(os.path.join(Flask_dir, 'templates', 'header.html'), 'w') as f: f.write(modified_header) # ========= JINJA2 FUNCTIONS ======== diff --git a/var/www/modules/Tags/Flask_Tags.py b/var/www/modules/Tags/Flask_Tags.py index 8ab81297..d15b78a8 100644 --- a/var/www/modules/Tags/Flask_Tags.py +++ b/var/www/modules/Tags/Flask_Tags.py @@ -20,6 +20,7 @@ from pymispgalaxies import Galaxies, Clusters # ============ VARIABLES ============ import Flask_config +import Tag app = Flask_config.app cfg = Flask_config.cfg @@ -59,16 +60,6 @@ for name, tags in clusters.items(): #galaxie name + tags def one(): return 1 -def date_substract_day(date, num_day=1): - new_date = datetime.date(int(date[0:4]), int(date[4:6]), int(date[6:8])) - datetime.timedelta(num_day) - new_date = str(new_date).replace('-', '') - return new_date - -def date_add_day(date, num_day=1): - new_date = datetime.date(int(date[0:4]), int(date[4:6]), int(date[6:8])) + datetime.timedelta(num_day) - new_date = str(new_date).replace('-', '') - return new_date - def get_tags_with_synonyms(tag): str_synonyms = ' - synonyms: ' synonyms = r_serv_tags.smembers('synonym_tag_' + tag) @@ -131,93 +122,6 @@ def get_last_seen_from_tags_list(list_tags): min_last_seen = tag_last_seen return str(min_last_seen) -def add_item_tag(tag, item_path): - item_date = int(get_item_date(item_path)) - - #add tag - r_serv_metadata.sadd('tag:{}'.format(item_path), tag) - r_serv_tags.sadd('{}:{}'.format(tag, item_date), item_path) - - r_serv_tags.hincrby('daily_tags:{}'.format(item_date), tag, 1) - - tag_first_seen = r_serv_tags.hget('tag_metadata:{}'.format(tag), 'last_seen') - if tag_first_seen is None: - tag_first_seen = 99999999 - else: - tag_first_seen = int(tag_first_seen) - tag_last_seen = r_serv_tags.hget('tag_metadata:{}'.format(tag), 'last_seen') - if tag_last_seen is None: - tag_last_seen = 0 - else: - tag_last_seen = int(tag_last_seen) - - #add new tag in list of all used tags - r_serv_tags.sadd('list_tags', tag) - - # update fisrt_seen/last_seen - if item_date < tag_first_seen: - r_serv_tags.hset('tag_metadata:{}'.format(tag), 'first_seen', item_date) - - # update metadata last_seen - if item_date > tag_last_seen: - r_serv_tags.hset('tag_metadata:{}'.format(tag), 'last_seen', item_date) - -def remove_item_tag(tag, item_path): - item_date = int(get_item_date(item_path)) - - #remove tag - r_serv_metadata.srem('tag:{}'.format(item_path), tag) - res = r_serv_tags.srem('{}:{}'.format(tag, item_date), item_path) - - if res ==1: - # no tag for this day - if int(r_serv_tags.hget('daily_tags:{}'.format(item_date), tag)) == 1: - r_serv_tags.hdel('daily_tags:{}'.format(item_date), tag) - else: - r_serv_tags.hincrby('daily_tags:{}'.format(item_date), tag, -1) - - tag_first_seen = int(r_serv_tags.hget('tag_metadata:{}'.format(tag), 'last_seen')) - tag_last_seen = int(r_serv_tags.hget('tag_metadata:{}'.format(tag), 'last_seen')) - # update fisrt_seen/last_seen - if item_date == tag_first_seen: - update_tag_first_seen(tag, tag_first_seen, tag_last_seen) - if item_date == tag_last_seen: - update_tag_last_seen(tag, tag_first_seen, tag_last_seen) - else: - return 'Error incorrect tag' - -def update_tag_first_seen(tag, tag_first_seen, tag_last_seen): - if tag_first_seen == tag_last_seen: - if r_serv_tags.scard('{}:{}'.format(tag, tag_first_seen)) > 0: - r_serv_tags.hset('tag_metadata:{}'.format(tag), 'first_seen', tag_first_seen) - # no tag in db - else: - r_serv_tags.srem('list_tags', tag) - r_serv_tags.hdel('tag_metadata:{}'.format(tag), 'first_seen') - r_serv_tags.hdel('tag_metadata:{}'.format(tag), 'last_seen') - else: - if r_serv_tags.scard('{}:{}'.format(tag, tag_first_seen)) > 0: - r_serv_tags.hset('tag_metadata:{}'.format(tag), 'first_seen', tag_first_seen) - else: - tag_first_seen = date_add_day(tag_first_seen) - update_tag_first_seen(tag, tag_first_seen, tag_last_seen) - -def update_tag_last_seen(tag, tag_first_seen, tag_last_seen): - if tag_first_seen == tag_last_seen: - if r_serv_tags.scard('{}:{}'.format(tag, tag_last_seen)) > 0: - r_serv_tags.hset('tag_metadata:{}'.format(tag), 'last_seen', tag_last_seen) - # no tag in db - else: - r_serv_tags.srem('list_tags', tag) - r_serv_tags.hdel('tag_metadata:{}'.format(tag), 'first_seen') - r_serv_tags.hdel('tag_metadata:{}'.format(tag), 'last_seen') - else: - if r_serv_tags.scard('{}:{}'.format(tag, tag_last_seen)) > 0: - r_serv_tags.hset('tag_metadata:{}'.format(tag), 'last_seen', tag_last_seen) - else: - tag_last_seen = date_substract_day(tag_last_seen) - update_tag_last_seen(tag, tag_first_seen, tag_last_seen) - # ============= ROUTES ============== @Tags.route("/tags/", methods=['GET']) @@ -472,8 +376,9 @@ def remove_tag(): path = request.args.get('paste') tag = request.args.get('tag') - remove_item_tag(tag, path) - + res = Tag.remove_item_tag(tag, path) + if res[1] != 200: + str(res[0]) return redirect(url_for('showsavedpastes.showsavedpaste', paste=path)) @Tags.route("/Tags/confirm_tag") @@ -486,11 +391,11 @@ def confirm_tag(): tag = request.args.get('tag') if(tag[9:28] == 'automatic-detection'): - remove_item_tag(tag, path) + Tag.remove_item_tag(tag, path) tag = tag.replace('automatic-detection','analyst-detection', 1) #add analyst tag - add_item_tag(tag, path) + Tag.add_item_tag(tag, path) return redirect(url_for('showsavedpastes.showsavedpaste', paste=path)) @@ -530,42 +435,12 @@ def addTags(): list_tag = tags.split(',') list_tag_galaxies = tagsgalaxies.split(',') - taxonomies = Taxonomies() - active_taxonomies = r_serv_tags.smembers('active_taxonomies') - - active_galaxies = r_serv_tags.smembers('active_galaxies') - - if not path: - return 'INCORRECT INPUT0' - - if list_tag != ['']: - for tag in list_tag: - # verify input - tax = tag.split(':')[0] - if tax in active_taxonomies: - if tag in r_serv_tags.smembers('active_tag_' + tax): - add_item_tag(tag, path) - - else: - return 'INCORRECT INPUT1' - else: - return 'INCORRECT INPUT2' - - if list_tag_galaxies != ['']: - for tag in list_tag_galaxies: - # verify input - gal = tag.split(':')[1] - gal = gal.split('=')[0] - - if gal in active_galaxies: - if tag in r_serv_tags.smembers('active_tag_galaxies_' + gal): - add_item_tag(tag, path) - - else: - return 'INCORRECT INPUT3' - else: - return 'INCORRECT INPUT4' - + res = Tag.add_items_tag(list_tag, list_tag_galaxies, path) + print(res) + # error + if res[1] != 200: + return str(res[0]) + # success return redirect(url_for('showsavedpastes.showsavedpaste', paste=path)) diff --git a/var/www/modules/restApi/Flask_restApi.py b/var/www/modules/restApi/Flask_restApi.py index 2c10ad62..727cd524 100644 --- a/var/www/modules/restApi/Flask_restApi.py +++ b/var/www/modules/restApi/Flask_restApi.py @@ -16,7 +16,7 @@ import datetime import Import_helper import Item import Paste -import Tags +import Tag from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response from flask_login import login_required @@ -146,24 +146,49 @@ def items(): return Response(json.dumps({'test': 2}), mimetype='application/json') -# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # -# GET -# -# { -# "id": item_id, mandatory -# } -# -# response: { -# "id": "item_id", -# "date": "date", -# "tags": [], -# "content": "item content" -# } -# -# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # @restApi.route("api/get/item/info/", methods=['GET']) @token_required('admin') def get_item_id(item_id): + """ + **GET api/get/item/info/** + + **Get item** + + This function allows user to get a specific item information through their item_id. + + :param id: id of the item + :type id: item id + :return: item's information in json and http status code + + - Example:: + + curl -k https://127.0.0.1:7000/api/get/item/info/submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json" + + - Expected Success Response:: + + HTTP Status Code: 200 + + { + "content": "item content test", + "date": "20190726", + "id": "submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz", + "tags": + [ + "misp-galaxy:backdoor=\"Rosenbridge\"", + "infoleak:automatic-detection=\"pgp-message\"", + "infoleak:automatic-detection=\"encrypted-private-key\"", + "infoleak:submission=\"manual\"", + "misp-galaxy:backdoor=\"SLUB\"" + ] + } + + - Expected Fail Response:: + + HTTP Status Code: 400 + + {'status': 'error', 'reason': 'Item not found'} + + """ try: item_object = Paste.Paste(item_id) except FileNotFoundError: @@ -188,14 +213,214 @@ def get_item_id(item_id): @restApi.route("api/get/item/tag/", methods=['GET']) @token_required('admin') def get_item_tag(item_id): + """ + **GET api/get/item/tag/** + + **Get item tags** + + This function allows user to get all items tags form a specified item id. + + :param id: id of the item + :type id: item id + :return: item's tags list in json and http status code + + - Example:: + + curl -k https://127.0.0.1:7000/api/get/item/tag/submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json" + + - Expected Success Response:: + + HTTP Status Code: 200 + + { + "id": "submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz", + "tags": + [ + "misp-galaxy:backdoor=\"Rosenbridge\"", + "infoleak:automatic-detection=\"pgp-message\"", + "infoleak:automatic-detection=\"encrypted-private-key\"", + "infoleak:submission=\"manual\"", + "misp-galaxy:backdoor=\"SLUB\"" + ] + } + + - Expected Fail Response:: + + HTTP Status Code: 400 + + {'status': 'error', 'reason': 'Item not found'} + + """ if not Item.exist_item(item_id): return Response(json.dumps({'status': 'error', 'reason': 'Item not found'}, indent=2, sort_keys=True), mimetype='application/json'), 400 - tags = Tags.get_item_tags(item_id) + tags = Tag.get_item_tags(item_id) dict_tags = {} dict_tags['id'] = item_id dict_tags['tags'] = tags return Response(json.dumps(dict_tags, indent=2, sort_keys=True), mimetype='application/json') +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +# POST +# +# { +# "id": item_id, mandatory +# "tags": [tags to add], +# "galaxy": [galaxy to add], +# } +# +# response: { +# "id": "item_id", +# "tags": [tags added], +# } +# +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +@restApi.route("api/add/item/tag", methods=['POST']) +@token_required('admin') +def add_item_tags(): + """ + **POST api/add/item/tag** + + **add tags to an item** + + This function allows user to add tags and galaxy to an item. + + :param id: id of the item + :type id: item id + :param tags: list of tags (default=[]) + :type tags: list + :param galaxy: list of galaxy (default=[]) + :type galaxy: list + + :return: item id and tags added in json and http status code + + - Example:: + + curl -k https://127.0.0.1:7000/api/add/item/tag --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json" --data @input.json -X POST + + - input.json Example:: + + { + "id": "submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz", + "tags": [ + "infoleak:analyst-detection=\"private-key\"", + "infoleak:analyst-detection=\"api-key\"" + ], + "galaxy": [ + "misp-galaxy:stealer=\"Vidar\"" + ] + } + + - Expected Success Response:: + + HTTP Status Code: 200 + + { + "id": "submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz", + "tags": [ + "infoleak:analyst-detection=\"private-key\"", + "infoleak:analyst-detection=\"api-key\"", + "misp-galaxy:stealer=\"Vidar\"" + ] + } + + - Expected Fail Response:: + + HTTP Status Code: 400 + + {'status': 'error', 'reason': 'Item id not found'} + {'status': 'error', 'reason': 'Tags or Galaxy not specified'} + {'status': 'error', 'reason': 'Tags or Galaxy not enabled'} + + """ + data = request.get_json() + if not data: + return Response(json.dumps({'status': 'error', 'reason': 'Malformed JSON'}, indent=2, sort_keys=True), mimetype='application/json'), 400 + + item_id = data.get('id', None) + tags = data.get('tags', []) + galaxy = data.get('galaxy', []) + + res = Tag.add_items_tag(tags, galaxy, item_id) + return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] + +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +# DELETE +# +# { +# "id": item_id, mandatory +# "tags": [tags to delete], +# } +# +# response: { +# "id": "item_id", +# "tags": [tags deleted], +# } +# +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +@restApi.route("api/delete/item/tag", methods=['DELETE']) +@token_required('admin') +def delete_item_tags(): + """ + **DELET E api/delete/item/tag** + + **delete tags from an item** + + This function allows user to delete tags and galaxy from an item. + + :param id: id of the item + :type id: item id + :param tags: list of tags (default=[]) + :type tags: list + + :return: item id and tags deleted in json and http status code + + - Example:: + + curl -k https://127.0.0.1:7000/api/delete/item/tag --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json" --data @input.json -X DELET E + + - input.json Example:: + + { + "id": "submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz", + "tags": [ + "infoleak:analyst-detection=\"private-key\"", + "infoleak:analyst-detection=\"api-key\"", + "misp-galaxy:stealer=\"Vidar\"" + ] + } + + - Expected Success Response:: + + HTTP Status Code: 200 + + { + "id": "submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz", + "tags": [ + "infoleak:analyst-detection=\"private-key\"", + "infoleak:analyst-detection=\"api-key\"", + "misp-galaxy:stealer=\"Vidar\"" + ] + } + + - Expected Fail Response:: + + HTTP Status Code: 400 + + {'status': 'error', 'reason': 'Item id not found'} + {'status': 'error', 'reason': 'No Tag(s) specified} + {'status': 'error', 'reason': 'Malformed JSON'} + + """ + data = request.get_json() + if not data: + return Response(json.dumps({'status': 'error', 'reason': 'Malformed JSON'}, indent=2, sort_keys=True), mimetype='application/json'), 400 + + item_id = data.get('id', None) + tags = data.get('tags', []) + + res = Tag.remove_item_tags(tags, item_id) + return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] + # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # GET # @@ -212,6 +437,37 @@ def get_item_tag(item_id): @restApi.route("api/get/item/content/", methods=['GET']) @token_required('admin') def get_item_content(item_id): + """ + **GET api/get/item/content/** + + **Get item content** + + This function allows user to get a specific item content. + + :param id: id of the item + :type id: item id + :return: item's content in json and http status code + + - Example:: + + curl -k https://127.0.0.1:7000/api/get/item/content/submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json" + + - Expected Success Response:: + + HTTP Status Code: 200 + + { + "content": "item content test", + "id": "submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz" + } + + - Expected Fail Response:: + + HTTP Status Code: 400 + + {'status': 'error', 'reason': 'Item not found'} + + """ try: item_object = Paste.Paste(item_id) except FileNotFoundError: @@ -240,6 +496,58 @@ def get_item_content(item_id): @restApi.route("api/import/item", methods=['POST']) @token_required('admin') def import_item(): + """ + **POST api/import/item** + + **Import new item** + + This function allows user to import new items. asynchronous function. + + :param text: text to import + :type text: str + :param type: import type (default='text') + :type type: "text" + :param tags: list of tags (default=[]) + :type tags: list + :param galaxy: list of galaxy (default=[]) + :type galaxy: list + :param default_tags: add default tag (default=True) + :type default_tags: boolean + + :return: imported uuid in json and http status code + + - Example:: + + curl -k https://127.0.0.1:7000/api/import/item --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json" --data @input.json -X POST + + - input.json Example:: + + { + "type": "text", + "tags": [ + "infoleak:analyst-detection=\"private-key\"" + ], + "text": "text to import" + } + + - Expected Success Response:: + + HTTP Status Code: 200 + + { + "uuid": "0c3d7b34-936e-4f01-9cdf-2070184b6016" + } + + - Expected Fail Response:: + + HTTP Status Code: 400 + + {'status': 'error', 'reason': 'Malformed JSON'} + {'status': 'error', 'reason': 'No text supplied'} + {'status': 'error', 'reason': 'Tags or Galaxy not enabled'} + {'status': 'error', 'reason': 'Size exceeds default'} + + """ data = request.get_json() if not data: return Response(json.dumps({'status': 'error', 'reason': 'Malformed JSON'}, indent=2, sort_keys=True), mimetype='application/json'), 400 @@ -256,7 +564,7 @@ def import_item(): if not type(galaxy) is list: galaxy = [] - if not Tags.is_valid_tags_taxonomies_galaxy(tags, galaxy): + if not Tag.is_valid_tags_taxonomies_galaxy(tags, galaxy): return Response(json.dumps({'status': 'error', 'reason': 'Tags or Galaxy not enabled'}, indent=2, sort_keys=True), mimetype='application/json'), 400 default_tags = data.get('default_tags', True) @@ -287,6 +595,41 @@ def import_item(): @restApi.route("api/import/item/", methods=['GET']) @token_required('admin') def import_item_uuid(UUID): + """ + **GET api/import/item/** + + **Get import status and all items imported by uuid** + + This return the import status and a list of imported items. + The full list of imported items is not complete until 'status'='imported'. + + :param uuid: import uuid + :type uuid: uuid4 + :return: json: import status + imported items list + + - Example:: + + curl -k https://127.0.0.1:7000/api/import/item/b20a69f1-99ad-4cb3-b212-7ce24b763b50 --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json" + + - Expected Success Response:: + + HTTP Status Code: 200 + + { + "items": [ + "submitted/2019/07/26/b20a69f1-99ad-4cb3-b212-7ce24b763b50.gz" + ], + "status": "in queue"/"in progress"/"imported" + } + + - Expected Fail Response:: + + HTTP Status Code: 400 + + {'status': 'error', 'reason': 'Invalid uuid'} + {'status': 'error', 'reason': 'Unknow uuid'} + + """ # Verify uuid if not is_valid_uuid_v4(UUID):