CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

chg: [api] use POST with parameters + add API unittest

pull/370/head
Terrtia 2019-08-05 16:00:23 +02:00
parent fea7b07134
commit e28d563523
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
4 changed files with 80 additions and 335 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
bin/packages/Item.py
View File

@ -46,13 +46,11 @@ def get_item_content(item_id):
except UnicodeDecodeError:
item_content = None
except Exception as e:
print("ERROR in: " + item_id)
print(e)
item_content = None
if item_content is None:
try:
with gzip.open(item_full_path, 'r') as f:
item_content = f.read()
item_content = f.read().decode()
r_cache.set(item_full_path, item_content)
r_cache.expire(item_full_path, 300)
except:

4
bin/packages/Tag.py
View File

@ -42,8 +42,6 @@ def is_galaxy_tag_enabled(galaxy, tag):
# Check if tags are enabled in AIL
def is_valid_tags_taxonomies_galaxy(list_tags, list_tags_galaxy):
print(list_tags)
print(list_tags_galaxy)
if list_tags:
active_taxonomies = get_active_taxonomies()
@ -161,7 +159,7 @@ def remove_item_tags(tags=[], item_id=None):
if res[1] != 200:
return res
else:
dict_res[tags].append(tag)
dict_res['tags'].append(tag)
dict_res['id'] = item_id
return (dict_res, 200)

63
doc/README.md
View File

@ -27,12 +27,12 @@ curl --header "Authorization: YOUR_API_KEY" --header "Content-Type: application/
## Item management
### Get item: `api/v1/get/item/default/<path:item_id>`
### Get item: `api/v1/get/item/default`
#### Description
Get item default info.
**Method** : `GET`
**Method** : `POST`
#### Parameters
- `id`
@ -56,7 +56,14 @@ Get item default info.
#### Example
```
curl https://127.0.0.1:7000/api/get/item/default/submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json"
curl https://127.0.0.1:7000/api/v1/get/item/default --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json" --data @input.json -X POST
```
#### input.json Example
```json
{
"id": "submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz"
}
```
#### Expected Success Response
@ -92,12 +99,12 @@ curl https://127.0.0.1:7000/api/get/item/default/submitted/2019/07/26/3efb8a79-0
### Get item content: `api/v1/get/item/content/<path:item_id>`
### Get item content: `api/v1/get/item/content`
#### Description
Get a specific item content.
**Method** : `GET`
**Method** : `POST`
#### Parameters
- `id`
@ -115,7 +122,14 @@ Get a specific item content.
#### Example
```
curl https://127.0.0.1:7000/api/get/item/content/submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json"
curl https://127.0.0.1:7000/api/v1/get/item/content --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json" --data @input.json -X POST
```
#### input.json Example
```json
{
"id": "submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz"
}
```
#### Expected Success Response
@ -141,12 +155,12 @@ curl https://127.0.0.1:7000/api/get/item/content/submitted/2019/07/26/3efb8a79-0
### Get item content: `api/v1/get/item/tag/<path:item_id>`
### Get item content: `api/v1/get/item/tag`
#### Description
Get all tags from an item.
**Method** : `GET`
**Method** : `POST`
#### Parameters
- `id`
@ -164,7 +178,14 @@ Get all tags from an item.
#### Example
```
curl https://127.0.0.1:7000/api/v1/get/item/tag/submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json"
curl https://127.0.0.1:7000/api/v1/get/item/tag --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json" --data @input.json -X POST
```
#### input.json Example
```json
{
"id": "submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz"
}
```
#### Expected Success Response
@ -506,12 +527,12 @@ curl https://127.0.0.1:7000/api/v1/get/tag/all --header "Authorization: iHc1_ChZ
### Get tag metadata: `api/v1/get/tag/metadata/<tag>`
### Get tag metadata: `api/v1/get/tag/metadata`
#### Description
Get tag metadata.
**Method** : `GET`
**Method** : `POST`
#### Parameters
- `tag`
@ -531,7 +552,14 @@ Get tag metadata.
- *str - YYMMDD*
#### Example
```
curl https://127.0.0.1:7000/api/v1/get/tag/metadata/infoleak:submission=\"manual\" --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json"
curl https://127.0.0.1:7000/api/v1/get/tag/metadata --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json" --data @input.json -X POST
```
#### input.json Example
```json
{
"tag": "infoleak:submission=\"manual\""
}
```
#### Expected Success Response
@ -638,7 +666,7 @@ curl https://127.0.0.1:7000/api/v1/import/item --header "Authorization: iHc1_ChZ
Get import status and all items imported by uuid
**Method** : `GET`
**Method** : `POST`
#### Parameters
@ -661,7 +689,14 @@ Get import status and all items imported by uuid
#### Example
```
curl -k https://127.0.0.1:7000/api/v1/import/item/b20a69f1-99ad-4cb3-b212-7ce24b763b50 --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json"
curl -k https://127.0.0.1:7000/api/v1/get/import/item --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json" --data @input.json -X POST
```
#### input.json Example
```json
{
"uuid": "0c3d7b34-936e-4f01-9cdf-2070184b6016"
}
```
#### Expected Success Response

344
var/www/modules/restApi/Flask_restApi.py
View File

@ -166,53 +166,16 @@ def get_item_id():
else:
return 'description API endpoint'
@restApi.route("api/v1/get/item/default/<path:item_id>", methods=['GET'])
@restApi.route("api/v1/get/item/default", methods=['POST'])
@token_required('admin')
def get_item_id_basic(item_id):
"""
**POST api/get/item/default/<item_id>**
def get_item_id_basic():
**Get item**
This function allows user to get a specific item information through their item_id.
:param id: id of the item
:type id: item id
:return: item's information in json and http status code
- Example::
curl -k https://127.0.0.1:7000/api/get/item/default/<item_id> --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json --data @input.json -X POST"
- Expected Success Response::
HTTP Status Code: 200
{
"content": "item content test",
"date": "20190726",
"id": "submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz",
"tags":
[
"misp-galaxy:backdoor=\"Rosenbridge\"",
"infoleak:automatic-detection=\"pgp-message\"",
"infoleak:automatic-detection=\"encrypted-private-key\"",
"infoleak:submission=\"manual\"",
"misp-galaxy:backdoor=\"SLUB\""
]
}
- Expected Fail Response::
HTTP Status Code: 400
{'status': 'error', 'reason': 'Item not found'}
"""
data = {'id': item_id, 'date': True, 'content': True, 'tags': True}
res = Item.get_item(data)
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
if request.method == 'POST':
data = request.get_json()
item_id = data.get('id', None)
req_data = {'id': item_id, 'date': True, 'content': True, 'tags': True}
res = Item.get_item(req_data)
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# GET
@ -227,49 +190,14 @@ def get_item_id_basic(item_id):
# }
#
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
@restApi.route("api/v1/get/item/tag/<path:item_id>", methods=['GET'])
@restApi.route("api/v1/get/item/tag", methods=['POST'])
@token_required('admin')
def get_item_tag(item_id):
"""
**GET api/get/item/tag/<item id>**
def get_item_tag():
**Get item tags**
This function allows user to get all items tags form a specified item id.
:param id: id of the item
:type id: item id
:return: item's tags list in json and http status code
- Example::
curl -k https://127.0.0.1:7000/api/get/item/tag/submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json"
- Expected Success Response::
HTTP Status Code: 200
{
"id": "submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz",
"tags":
[
"misp-galaxy:backdoor=\"Rosenbridge\"",
"infoleak:automatic-detection=\"pgp-message\"",
"infoleak:automatic-detection=\"encrypted-private-key\"",
"infoleak:submission=\"manual\"",
"misp-galaxy:backdoor=\"SLUB\""
]
}
- Expected Fail Response::
HTTP Status Code: 400
{'status': 'error', 'reason': 'Item not found'}
"""
data = {'id': item_id, 'date': False, 'tags': True}
res = Item.get_item(data)
data = request.get_json()
item_id = data.get('id', None)
req_data = {'id': item_id, 'date': False, 'tags': True}
res = Item.get_item(req_data)
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
@ -290,61 +218,7 @@ def get_item_tag(item_id):
@restApi.route("api/v1/add/item/tag", methods=['POST'])
@token_required('admin')
def add_item_tags():
"""
**POST api/add/item/tag**
**add tags to an item**
This function allows user to add tags and galaxy to an item.
:param id: id of the item
:type id: item id
:param tags: list of tags (default=[])
:type tags: list
:param galaxy: list of galaxy (default=[])
:type galaxy: list
:return: item id and tags added in json and http status code
- Example::
curl -k https://127.0.0.1:7000/api/add/item/tag --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json" --data @input.json -X POST
- input.json Example::
{
"id": "submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz",
"tags": [
"infoleak:analyst-detection=\"private-key\"",
"infoleak:analyst-detection=\"api-key\""
],
"galaxy": [
"misp-galaxy:stealer=\"Vidar\""
]
}
- Expected Success Response::
HTTP Status Code: 200
{
"id": "submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz",
"tags": [
"infoleak:analyst-detection=\"private-key\"",
"infoleak:analyst-detection=\"api-key\"",
"misp-galaxy:stealer=\"Vidar\""
]
}
- Expected Fail Response::
HTTP Status Code: 400
{'status': 'error', 'reason': 'Item id not found'}
{'status': 'error', 'reason': 'Tags or Galaxy not specified'}
{'status': 'error', 'reason': 'Tags or Galaxy not enabled'}
"""
data = request.get_json()
if not data:
return Response(json.dumps({'status': 'error', 'reason': 'Malformed JSON'}, indent=2, sort_keys=True), mimetype='application/json'), 400
@ -373,57 +247,7 @@ def add_item_tags():
@restApi.route("api/v1/delete/item/tag", methods=['DELETE'])
@token_required('admin')
def delete_item_tags():
"""
**DELET E api/delete/item/tag**
**delete tags from an item**
This function allows user to delete tags and galaxy from an item.
:param id: id of the item
:type id: item id
:param tags: list of tags (default=[])
:type tags: list
:return: item id and tags deleted in json and http status code
- Example::
curl -k https://127.0.0.1:7000/api/delete/item/tag --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json" --data @input.json -X DELET E
- input.json Example::
{
"id": "submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz",
"tags": [
"infoleak:analyst-detection=\"private-key\"",
"infoleak:analyst-detection=\"api-key\"",
"misp-galaxy:stealer=\"Vidar\""
]
}
- Expected Success Response::
HTTP Status Code: 200
{
"id": "submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz",
"tags": [
"infoleak:analyst-detection=\"private-key\"",
"infoleak:analyst-detection=\"api-key\"",
"misp-galaxy:stealer=\"Vidar\""
]
}
- Expected Fail Response::
HTTP Status Code: 400
{'status': 'error', 'reason': 'Item id not found'}
{'status': 'error', 'reason': 'No Tag(s) specified}
{'status': 'error', 'reason': 'Malformed JSON'}
"""
data = request.get_json()
if not data:
return Response(json.dumps({'status': 'error', 'reason': 'Malformed JSON'}, indent=2, sort_keys=True), mimetype='application/json'), 400
@ -447,57 +271,31 @@ def delete_item_tags():
# }
#
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
@restApi.route("api/v1/get/item/content/<path:item_id>", methods=['GET'])
@restApi.route("api/v1/get/item/content", methods=['POST'])
@token_required('admin')
def get_item_content(item_id):
"""
**GET api/get/item/content/<item id>**
def get_item_content():
**Get item content**
This function allows user to get a specific item content.
:param id: id of the item
:type id: item id
:return: item's content in json and http status code
- Example::
curl -k https://127.0.0.1:7000/api/get/item/content/submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json"
- Expected Success Response::
HTTP Status Code: 200
{
"content": "item content test",
"id": "submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz"
}
- Expected Fail Response::
HTTP Status Code: 400
{'status': 'error', 'reason': 'Item not found'}
"""
data = {'id': item_id, 'date': False, 'content': True, 'tags': False}
res = Item.get_item(data)
data = request.get_json()
item_id = data.get('id', None)
req_data = {'id': item_id, 'date': False, 'content': True, 'tags': False}
res = Item.get_item(req_data)
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# # # # # # # # # # # # # # TAGS # # # # # # # # # # # # # # # # #
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
@restApi.route("api/v1/get/tag/metadata/<tag>", methods=['GET'])
@restApi.route("api/v1/get/tag/metadata", methods=['POST'])
@token_required('admin')
def get_tag_metadata(tag):
def get_tag_metadata():
data = request.get_json()
tag = data.get('tag', None)
if not Tag.is_tag_in_all_tag(tag):
return Response(json.dumps({'status': 'error', 'reason':'Tag not found'}, indent=2, sort_keys=True), mimetype='application/json'), 404
metadata = Tag.get_tag_metadata(tag)
return Response(json.dumps(metadata, indent=2, sort_keys=True), mimetype='application/json'), 200
@restApi.route("api/get/tag/all", methods=['GET'])
@restApi.route("api/v1/get/tag/all", methods=['GET'])
@token_required('admin')
def get_all_tags():
res = {'tags': Tag.get_all_tags()}
@ -528,58 +326,7 @@ def get_all_tags():
@restApi.route("api/v1/import/item", methods=['POST'])
@token_required('admin')
def import_item():
"""
**POST api/import/item**
**Import new item**
This function allows user to import new items. asynchronous function.
:param text: text to import
:type text: str
:param type: import type (default='text')
:type type: "text"
:param tags: list of tags (default=[])
:type tags: list
:param galaxy: list of galaxy (default=[])
:type galaxy: list
:param default_tags: add default tag (default=True)
:type default_tags: boolean
:return: imported uuid in json and http status code
- Example::
curl -k https://127.0.0.1:7000/api/import/item --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json" --data @input.json -X POST
- input.json Example::
{
"type": "text",
"tags": [
"infoleak:analyst-detection=\"private-key\""
],
"text": "text to import"
}
- Expected Success Response::
HTTP Status Code: 200
{
"uuid": "0c3d7b34-936e-4f01-9cdf-2070184b6016"
}
- Expected Fail Response::
HTTP Status Code: 400
{'status': 'error', 'reason': 'Malformed JSON'}
{'status': 'error', 'reason': 'No text supplied'}
{'status': 'error', 'reason': 'Tags or Galaxy not enabled'}
{'status': 'error', 'reason': 'Size exceeds default'}
"""
data = request.get_json()
if not data:
return Response(json.dumps({'status': 'error', 'reason': 'Malformed JSON'}, indent=2, sort_keys=True), mimetype='application/json'), 400
@ -624,44 +371,11 @@ def import_item():
# }
#
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
@restApi.route("api/v1/import/item/<UUID>", methods=['GET'])
@restApi.route("api/v1/get/import/item", methods=['POST'])
@token_required('admin')
def import_item_uuid(UUID):
"""
**GET api/import/item/<uuid4>**
**Get import status and all items imported by uuid**
This return the import status and a list of imported items.
The full list of imported items is not complete until 'status'='imported'.
:param uuid: import uuid
:type uuid: uuid4
:return: json: import status + imported items list
- Example::
curl -k https://127.0.0.1:7000/api/import/item/b20a69f1-99ad-4cb3-b212-7ce24b763b50 --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json"
- Expected Success Response::
HTTP Status Code: 200
{
"items": [
"submitted/2019/07/26/b20a69f1-99ad-4cb3-b212-7ce24b763b50.gz"
],
"status": "in queue"/"in progress"/"imported"
}
- Expected Fail Response::
HTTP Status Code: 400
{'status': 'error', 'reason': 'Invalid uuid'}
{'status': 'error', 'reason': 'Unknown uuid'}
"""
def import_item_uuid():
data = request.get_json()
UUID = data.get('uuid', None)
# Verify uuid
if not is_valid_uuid_v4(UUID):