CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

chg: [api] add advanced get item via POST + use same query for each get item

pull/370/head
Terrtia 2019-08-01 13:16:57 +02:00
parent 8c02c1b00b
commit 4c20f58a52
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
4 changed files with 243 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

74
bin/packages/Item.py
View File

@ -2,12 +2,15 @@
# -*-coding:UTF-8 -*
import os
import gzip
import redis
import Flask_config
import Date
import Tag
PASTES_FOLDER = Flask_config.PASTES_FOLDER
r_cache = Flask_config.r_cache
def exist_item(item_id):
if os.path.isfile(os.path.join(PASTES_FOLDER, item_id)):
@ -18,3 +21,74 @@ def exist_item(item_id):
def get_item_date(item_id):
l_directory = item_id.split('/')
return '{}{}{}'.format(l_directory[-4], l_directory[-3], l_directory[-2])
def get_item_size(item_id):
return round(os.path.getsize(os.path.join(PASTES_FOLDER, item_id))/1024.0, 2)
def get_lines_info(item_id, item_content=None):
if not item_content:
item_content = get_item_content(item_id)
max_length = 0
line_id = 0
nb_line = 0
for line in item_content.splitlines():
length = len(line)
if length > max_length:
max_length = length
nb_line += 1
return {'nb': nb_line, 'max_length': max_length}
def get_item_content(item_id):
item_full_path = os.path.join(PASTES_FOLDER, item_id)
try:
item_content = r_cache.get(item_full_path)
except UnicodeDecodeError:
item_content = None
except Exception as e:
print("ERROR in: " + item_id)
print(e)
item_content = None
if item_content is None:
try:
with gzip.open(item_full_path, 'r') as f:
item_content = f.read()
r_cache.set(item_full_path, item_content)
r_cache.expire(item_full_path, 300)
except:
item_content = ''
return str(item_content)
# API
def get_item(request_dict):
if not request_dict:
return Response({'status': 'error', 'reason': 'Malformed JSON'}, 400)
item_id = request_dict.get('id', None)
if not item_id:
return ( {'status': 'error', 'reason': 'Mandatory parameter(s) not provided'}, 400 )
if not exist_item(item_id):
return ( {'status': 'error', 'reason': 'Item not found'}, 404 )
dict_item = {}
dict_item['id'] = item_id
date = request_dict.get('date', True)
if date:
dict_item['date'] = get_item_date(item_id)
tags = request_dict.get('tags', True)
if tags:
dict_item['tags'] = Tag.get_item_tags(item_id)
size = request_dict.get('size', False)
if size:
dict_item['size'] = get_item_size(item_id)
content = request_dict.get('content', False)
if content:
dict_item['content'] = get_item_content(item_id)
lines_info = request_dict.get('lines', False)
if lines_info:
dict_item['lines'] = get_lines_info(item_id, dict_item.get('content', 'None'))
return (dict_item, 200)

130
doc/README.md
View File

@ -27,10 +27,10 @@ curl --header "Authorization: YOUR_API_KEY" --header "Content-Type: application/
## Item management
### Get item: `api/get/item/basic/<path:item_id>`
### Get item: `api/get/item/default/<path:item_id>`
#### Description
Get anitem basic information.
Get item default info.
**Method** : `GET`
@ -56,7 +56,7 @@ Get anitem basic information.
#### Example
```
curl https://127.0.0.1:7000/api/get/item/info/submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json"
curl https://127.0.0.1:7000/api/get/item/default/submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json"
```
#### Expected Success Response
@ -81,7 +81,10 @@ curl https://127.0.0.1:7000/api/get/item/info/submitted/2019/07/26/3efb8a79-08e9
#### Expected Fail Response
**HTTP Status Code** : `400`
```json
{"status": "error", "reason": "Mandatory parameter(s) not provided"}
```
**HTTP Status Code** : `404`
```json
{"status": "error", "reason": "Item not found"}
```
@ -128,7 +131,10 @@ curl https://127.0.0.1:7000/api/get/item/content/submitted/2019/07/26/3efb8a79-0
#### Expected Fail Response
**HTTP Status Code** : `400`
```json
{"status": "error", "reason": "Mandatory parameter(s) not provided"}
```
**HTTP Status Code** : `404`
```json
{"status": "error", "reason": "Item not found"}
```
@ -181,13 +187,125 @@ curl https://127.0.0.1:7000/api/get/item/tag/submitted/2019/07/26/3efb8a79-08e9-
#### Expected Fail Response
**HTTP Status Code** : `400`
```json
{"status": "error", "reason": "Mandatory parameter(s) not provided"}
```
**HTTP Status Code** : `404`
```json
{"status": "error", "reason": "Item not found"}
```
### Advanced Get item: `api/get/item`
#### Description
Get item. Filter requested field.
**Method** : `POST`
#### Parameters
- `id`
- item id
- *str - relative item path*
- mandatory
- `date`
- get item date
- *boolean*
- default: `true`
- `tags`
- get item tags
- *boolean*
- default: `true`
- `content`
- get item content
- *boolean*
- default: `false`
- `size`
- get item size
- *boolean*
- default: `false`
- `lines`
- get item lines info
- *boolean*
- default: `false`
#### JSON response
- `content`
- item content
- *str*
- `id`
- item id
- *str*
- `date`
- item date
- *str - YYMMDD*
- `tags`
- item tags list
- *list*
- `size`
- item size (Kb)
- *int*
- `lines`
- item lines info
- *{}*
- `max_length`
- line max length line
- *int*
- `nb`
- nb lines item
- *int*
#### Example
```
curl https://127.0.0.1:7000/api/get/item --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json" --data @input.json -X POST
```
#### input.json Example
```json
{
"id": "submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz",
"content": true,
"lines_info": true,
"tags": true,
"size": true
}
```
#### Expected Success Response
**HTTP Status Code** : `200`
```json
{
"content": "b'dsvcdsvcdsc vvvv'",
"date": "20190726",
"id": "submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz",
"lines": {
"max_length": 19,
"nb": 1
},
"size": 0.03,
"tags": [
"misp-galaxy:stealer=\"Vidar\"",
"infoleak:submission=\"manual\""
]
}
```
#### Expected Fail Response
**HTTP Status Code** : `400`
```json
{"status": "error", "reason": "Mandatory parameter(s) not provided"}
```
**HTTP Status Code** : `404`
```json
{"status": "error", "reason": "Item not found"}
```
### add item tags: `api/add/item/tag`
#### Description

10
var/www/modules/PasteSubmit/Flask_PasteSubmit.py
View File

@ -24,7 +24,7 @@ import json
import Paste
import Import_helper
import Tags
import Tag
from pytaxonomies import Taxonomies
from pymispgalaxies import Galaxies, Clusters
@ -224,8 +224,8 @@ def hive_create_case(hive_tlp, threat_level, hive_description, hive_case_title,
@login_analyst
def PasteSubmit_page():
# Get all active tags/galaxy
active_taxonomies = Tags.get_active_taxonomies()
active_galaxies = Tags.get_active_galaxies()
active_taxonomies = Tag.get_active_taxonomies()
active_galaxies = Tag.get_active_galaxies()
return render_template("submit_items.html",
active_taxonomies = active_taxonomies,
@ -253,9 +253,9 @@ def submit():
submitted_tag = 'infoleak:submission="manual"'
#active taxonomies
active_taxonomies = Tags.get_active_taxonomies()
active_taxonomies = Tag.get_active_taxonomies()
#active galaxies
active_galaxies = Tags.get_active_galaxies()
active_galaxies = Tag.get_active_galaxies()
if ltags or ltagsgalaxies:

66
var/www/modules/restApi/Flask_restApi.py
View File

@ -139,11 +139,38 @@ def one():
# def api():
# return 'api doc'
@restApi.route("api/get/item/basic/<path:item_id>", methods=['GET'])
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# POST
#
# {
# "id": item_id, mandatory
# "content": true,
# "tags": true,
#
#
# }
#
# response: {
# "id": "item_id",
# "tags": [],
# }
#
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
@restApi.route("api/get/item", methods=['GET', 'POST'])
@token_required('admin')
def get_item_id(item_id):
def get_item_id():
if request.method == 'POST':
data = request.get_json()
res = Item.get_item(data)
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
else:
return 'description API endpoint'
@restApi.route("api/get/item/default/<path:item_id>", methods=['GET'])
@token_required('admin')
def get_item_id_basic(item_id):
"""
**GET api/get/item/info/<item id>**
**POST api/get/item/default/<item_id>**
**Get item**
@ -155,7 +182,7 @@ def get_item_id(item_id):
- Example::
curl -k https://127.0.0.1:7000/api/get/item/info/submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json"
curl -k https://127.0.0.1:7000/api/get/item/default --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json --data @input.json -X POST"
- Expected Success Response::
@ -182,13 +209,10 @@ def get_item_id(item_id):
{'status': 'error', 'reason': 'Item not found'}
"""
try:
item_object = Paste.Paste(item_id)
except FileNotFoundError:
return Response(json.dumps({'status': 'error', 'reason': 'Item not found'}, indent=2, sort_keys=True), mimetype='application/json'), 404
data = item_object.get_item_dict()
return Response(json.dumps(data, indent=2, sort_keys=True), mimetype='application/json')
data = {'id': item_id, 'date': True, 'content': True, 'tags': True}
res = Item.get_item(data)
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# GET
@ -244,13 +268,9 @@ def get_item_tag(item_id):
{'status': 'error', 'reason': 'Item not found'}
"""
if not Item.exist_item(item_id):
return Response(json.dumps({'status': 'error', 'reason': 'Item not found'}, indent=2, sort_keys=True), mimetype='application/json'), 404
tags = Tag.get_item_tags(item_id)
dict_tags = {}
dict_tags['id'] = item_id
dict_tags['tags'] = tags
return Response(json.dumps(dict_tags, indent=2, sort_keys=True), mimetype='application/json')
data = {'id': item_id, 'date': False, 'tags': True}
res = Item.get_item(data)
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# POST
@ -461,15 +481,9 @@ def get_item_content(item_id):
{'status': 'error', 'reason': 'Item not found'}
"""
try:
item_object = Paste.Paste(item_id)
except FileNotFoundError:
return Response(json.dumps({'status': 'error', 'reason': 'Item not found'}, indent=2, sort_keys=True), mimetype='application/json'), 404
item_object = Paste.Paste(item_id)
dict_content = {}
dict_content['id'] = item_id
dict_content['content'] = item_object.get_p_content()
return Response(json.dumps(dict_content, indent=2, sort_keys=True), mimetype='application/json')
data = {'id': item_id, 'date': False, 'content': True, 'tags': False}
res = Item.get_item(data)
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
#