CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

chg: [API v1] add API documentation + update/delete items tags + Flask_tags refractor

pull/370/head
Terrtia 2019-07-30 13:49:21 +02:00
parent 6af9514a48
commit 44cf5bb4af
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
10 changed files with 1068 additions and 284 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
bin/packages/Date.py
View File

@ -40,3 +40,13 @@ class Date(object):
comp_month = str(computed_date.month).zfill(2)
comp_day = str(computed_date.day).zfill(2)
return comp_year + comp_month + comp_day
def date_add_day(date, num_day=1):
new_date = datetime.date(int(date[0:4]), int(date[4:6]), int(date[6:8])) + datetime.timedelta(num_day)
new_date = str(new_date).replace('-', '')
return new_date
def date_substract_day(date, num_day=1):
new_date = datetime.date(int(date[0:4]), int(date[4:6]), int(date[6:8])) - datetime.timedelta(num_day)
new_date = str(new_date).replace('-', '')
return new_date

4
bin/packages/Item.py
View File

@ -14,3 +14,7 @@ def exist_item(item_id):
return True
else:
return False
def get_item_date(item_id):
l_directory = item_id.split('/')
return '{}{}{}'.format(l_directory[-4], l_directory[-3], l_directory[-2])

210
bin/packages/Tag.py Executable file
View File

@ -0,0 +1,210 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
import os
import redis
import Flask_config
import Date
import Item
from pytaxonomies import Taxonomies
from pymispgalaxies import Galaxies, Clusters
r_serv_tags = Flask_config.r_serv_tags
r_serv_metadata = Flask_config.r_serv_metadata
def get_taxonomie_from_tag(tag):
return tag.split(':')[0]
def get_galaxy_from_tag(tag):
galaxy = tag.split(':')[1]
galaxy = galaxy.split('=')[0]
return galaxy
def get_active_taxonomies():
return r_serv_tags.smembers('active_taxonomies')
def get_active_galaxies():
return r_serv_tags.smembers('active_galaxies')
def is_taxonomie_tag_enabled(taxonomie, tag):
if tag in r_serv_tags.smembers('active_tag_' + taxonomie):
return True
else:
return False
def is_galaxy_tag_enabled(galaxy, tag):
if tag in r_serv_tags.smembers('active_tag_galaxies_' + galaxy):
return True
else:
return False
# Check if tags are enabled in AIL
def is_valid_tags_taxonomies_galaxy(list_tags, list_tags_galaxy):
print(list_tags)
print(list_tags_galaxy)
if list_tags:
active_taxonomies = get_active_taxonomies()
for tag in list_tags:
taxonomie = get_taxonomie_from_tag(tag)
if taxonomie not in active_taxonomies:
return False
if not is_taxonomie_tag_enabled(taxonomie, tag):
return False
if list_tags_galaxy:
active_galaxies = get_active_galaxies()
for tag in list_tags_galaxy:
galaxy = get_galaxy_from_tag(tag)
if galaxy not in active_galaxies:
return False
if not is_galaxy_tag_enabled(galaxy, tag):
return False
return True
def get_item_tags(item_id):
tags = r_serv_metadata.smembers('tag:'+item_id)
if tags:
return list(tags)
else:
return '[]'
# TEMPLATE + API QUERY
def add_items_tag(tags=[], galaxy_tags=[], item_id=None):
res_dict = {}
if item_id == None:
return ({'status': 'error', 'reason': 'Item id not found'}, 400)
if not tags and not galaxy_tags:
return ({'status': 'error', 'reason': 'Tags or Galaxy not specified'}, 400)
res_dict['tags'] = []
for tag in tags:
taxonomie = get_taxonomie_from_tag(tag)
if is_taxonomie_tag_enabled(taxonomie, tag):
add_item_tag(tag, item_id)
res_dict['tags'].append(tag)
else:
return ({'status': 'error', 'reason': 'Tags or Galaxy not enabled'}, 400)
for tag in galaxy_tags:
galaxy = get_galaxy_from_tag(tag)
if is_galaxy_tag_enabled(galaxy, tag):
add_item_tag(tag, item_id)
res_dict['tags'].append(tag)
else:
return ({'status': 'error', 'reason': 'Tags or Galaxy not enabled'}, 400)
res_dict['id'] = item_id
return (res_dict, 200)
def add_item_tag(tag, item_path):
item_date = int(Item.get_item_date(item_path))
#add tag
r_serv_metadata.sadd('tag:{}'.format(item_path), tag)
r_serv_tags.sadd('{}:{}'.format(tag, item_date), item_path)
r_serv_tags.hincrby('daily_tags:{}'.format(item_date), tag, 1)
tag_first_seen = r_serv_tags.hget('tag_metadata:{}'.format(tag), 'last_seen')
if tag_first_seen is None:
tag_first_seen = 99999999
else:
tag_first_seen = int(tag_first_seen)
tag_last_seen = r_serv_tags.hget('tag_metadata:{}'.format(tag), 'last_seen')
if tag_last_seen is None:
tag_last_seen = 0
else:
tag_last_seen = int(tag_last_seen)
#add new tag in list of all used tags
r_serv_tags.sadd('list_tags', tag)
# update fisrt_seen/last_seen
if item_date < tag_first_seen:
r_serv_tags.hset('tag_metadata:{}'.format(tag), 'first_seen', item_date)
# update metadata last_seen
if item_date > tag_last_seen:
r_serv_tags.hset('tag_metadata:{}'.format(tag), 'last_seen', item_date)
# API QUERY
def remove_item_tags(tags=[], item_id=None):
if item_id == None:
return ({'status': 'error', 'reason': 'Item id not found'}, 400)
if not tags:
return ({'status': 'error', 'reason': 'No Tag(s) specified'}, 400)
dict_res = {}
dict_res[tags] = []
for tag in tags:
res = remove_item_tag(tag, item_id)
if res[1] != 200:
return res
else:
dict_res[tags].append(tag)
dict_res[id] = item_id
return (dict_res, 200)
# TEMPLATE + API QUERY
def remove_item_tag(tag, item_id):
item_date = int(Item.get_item_date(item_id))
#remove tag
r_serv_metadata.srem('tag:{}'.format(item_id), tag)
res = r_serv_tags.srem('{}:{}'.format(tag, item_date), item_id)
if res ==1:
# no tag for this day
if int(r_serv_tags.hget('daily_tags:{}'.format(item_date), tag)) == 1:
r_serv_tags.hdel('daily_tags:{}'.format(item_date), tag)
else:
r_serv_tags.hincrby('daily_tags:{}'.format(item_date), tag, -1)
tag_first_seen = int(r_serv_tags.hget('tag_metadata:{}'.format(tag), 'last_seen'))
tag_last_seen = int(r_serv_tags.hget('tag_metadata:{}'.format(tag), 'last_seen'))
# update fisrt_seen/last_seen
if item_date == tag_first_seen:
update_tag_first_seen(tag, tag_first_seen, tag_last_seen)
if item_date == tag_last_seen:
update_tag_last_seen(tag, tag_first_seen, tag_last_seen)
return ({'status': 'success'}, 200)
else:
return ({'status': 'error', 'reason': 'Item id or tag not found'}, 400)
def update_tag_first_seen(tag, tag_first_seen, tag_last_seen):
if tag_first_seen == tag_last_seen:
if r_serv_tags.scard('{}:{}'.format(tag, tag_first_seen)) > 0:
r_serv_tags.hset('tag_metadata:{}'.format(tag), 'first_seen', tag_first_seen)
# no tag in db
else:
r_serv_tags.srem('list_tags', tag)
r_serv_tags.hdel('tag_metadata:{}'.format(tag), 'first_seen')
r_serv_tags.hdel('tag_metadata:{}'.format(tag), 'last_seen')
else:
if r_serv_tags.scard('{}:{}'.format(tag, tag_first_seen)) > 0:
r_serv_tags.hset('tag_metadata:{}'.format(tag), 'first_seen', tag_first_seen)
else:
tag_first_seen = Date.date_add_day(tag_first_seen)
update_tag_first_seen(tag, tag_first_seen, tag_last_seen)
def update_tag_last_seen(tag, tag_first_seen, tag_last_seen):
if tag_first_seen == tag_last_seen:
if r_serv_tags.scard('{}:{}'.format(tag, tag_last_seen)) > 0:
r_serv_tags.hset('tag_metadata:{}'.format(tag), 'last_seen', tag_last_seen)
# no tag in db
else:
r_serv_tags.srem('list_tags', tag)
r_serv_tags.hdel('tag_metadata:{}'.format(tag), 'first_seen')
r_serv_tags.hdel('tag_metadata:{}'.format(tag), 'last_seen')
else:
if r_serv_tags.scard('{}:{}'.format(tag, tag_last_seen)) > 0:
r_serv_tags.hset('tag_metadata:{}'.format(tag), 'last_seen', tag_last_seen)
else:
tag_last_seen = Date.date_substract_day(tag_last_seen)
update_tag_last_seen(tag, tag_first_seen, tag_last_seen)

71
bin/packages/Tags.py
View File

@ -1,71 +0,0 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
import os
import redis
import Flask_config
from pytaxonomies import Taxonomies
from pymispgalaxies import Galaxies, Clusters
r_serv_tags = Flask_config.r_serv_tags
r_serv_metadata = Flask_config.r_serv_metadata
def get_taxonomie_from_tag(tag):
return tag.split(':')[0]
def get_galaxy_from_tag(tag):
galaxy = tag.split(':')[1]
galaxy = galaxy.split('=')[0]
return galaxy
def get_active_taxonomies():
return r_serv_tags.smembers('active_taxonomies')
def get_active_galaxies():
return r_serv_tags.smembers('active_galaxies')
def is_taxonomie_tag_enabled(taxonomie, tag):
if tag in r_serv_tags.smembers('active_tag_' + taxonomie):
return True
else:
return False
def is_galaxy_tag_enabled(galaxy, tag):
if tag in r_serv_tags.smembers('active_tag_galaxies_' + galaxy):
return True
else:
return False
# Check if tags are enabled in AIL
def is_valid_tags_taxonomies_galaxy(list_tags, list_tags_galaxy):
print(list_tags)
print(list_tags_galaxy)
if list_tags:
active_taxonomies = get_active_taxonomies()
for tag in list_tags:
taxonomie = get_taxonomie_from_tag(tag)
if taxonomie not in active_taxonomies:
return False
if not is_taxonomie_tag_enabled(taxonomie, tag):
return False
if list_tags_galaxy:
active_galaxies = get_active_galaxies()
for tag in list_tags_galaxy:
galaxy = get_galaxy_from_tag(tag)
if galaxy not in active_galaxies:
return False
if not is_galaxy_tag_enabled(galaxy, tag):
return False
return True
def get_item_tags(item_id):
tags = r_serv_metadata.smembers('tag:'+item_id)
if tags:
return list(tags)
else:
return '[]'

464
doc/README.md Normal file
View File

@ -0,0 +1,464 @@
# API DOCUMENTATION
## General
### Automation key
The authentication of the automation is performed via a secure key available in the AIL UI interface. Make sure you keep that key secret as it gives access to the entire database! The API key is available in the ``Server Management`` menu under ``My Profile``.
The authorization is performed by using the following header:
~~~~
Authorization: YOUR API KEY
~~~~
### Accept and Content-Type headers
When submitting data in a POST, PUT or DELETE operation you need to specify in what content-type you encoded the payload. This is done by setting the below Content-Type headers:
~~~~
Content-Type: application/json
~~~~
Example:
~~~~
curl --header "Authorization: YOUR API KEY" --header "Content-Type: application/json" https://<AIL URL>/
~~~~
## Item management
### Get item: `api/get/item/info/<path:item_id>`
#### Description
Get a specific item information.
**Method** : `GET`
#### Parameters
- `id`
- item id
- *str - relative item path*
- mandatory
#### JSON response
- `content`
- item content
- *str*
- `id`
- item id
- *str*
- `date`
- item date
- *str - YYMMDD*
- `tags`
- item tags list
- *list*
#### Example
```
curl https://127.0.0.1:7000/api/get/item/info/submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json"
```
#### Expected Success Response
**HTTP Status Code** : `200`
```json
{
"content": "item content test",
"date": "20190726",
"id": "submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz",
"tags":
[
"misp-galaxy:backdoor=\"Rosenbridge\"",
"infoleak:automatic-detection=\"pgp-message\"",
"infoleak:automatic-detection=\"encrypted-private-key\"",
"infoleak:submission=\"manual\"",
"misp-galaxy:backdoor=\"SLUB\""
]
}
```
#### Expected Fail Response
**HTTP Status Code** : `400`
```
{'status': 'error', 'reason': 'Item not found'}
```
### Get item content: `api/get/item/content/<path:item_id>`
#### Description
Get a specific item content.
**Method** : `GET`
#### Parameters
- `id`
- item id
- *str - relative item path*
- mandatory
#### JSON response
- `content`
- item content
- *str*
- `id`
- item id
- *str*
#### Example
```
curl https://127.0.0.1:7000/api/get/item/content/submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json"
```
#### Expected Success Response
**HTTP Status Code** : `200`
```json
{
"content": "item content test",
"id": "submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz"
}
```
#### Expected Fail Response
**HTTP Status Code** : `400`
```
{'status': 'error', 'reason': 'Item not found'}
```
### Get item content: `api/get/item/tag/<path:item_id>`
#### Description
Get all tags from an item.
**Method** : `GET`
#### Parameters
- `id`
- item id
- *str - relative item path*
- mandatory
#### JSON response
- `content`
- item content
- *str*
- `tags`
- item tags list
- *list*
#### Example
```
curl https://127.0.0.1:7000/api/get/item/tag/submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json"
```
#### Expected Success Response
**HTTP Status Code** : `200`
```json
{
"id": "submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz",
"tags":
[
"misp-galaxy:backdoor=\"Rosenbridge\"",
"infoleak:automatic-detection=\"pgp-message\"",
"infoleak:automatic-detection=\"encrypted-private-key\"",
"infoleak:submission=\"manual\"",
"misp-galaxy:backdoor=\"SLUB\""
]
}
```
#### Expected Fail Response
**HTTP Status Code** : `400`
```
{'status': 'error', 'reason': 'Item not found'}
```
### add item tags: `api/add/item/tag`
#### Description
Add tags to an item.
**Method** : `POST`
#### Parameters
- `id`
- item id
- *str - relative item path*
- mandatory
- `tags`
- list of tags
- *list*
- default: `[]`
- `galaxy`
- list of galaxy
- *list*
- default: `[]`
#### JSON response
- `id`
- item id
- *str - relative item path*
- `tags`
- list of item tags added
- *list*
#### Example
```
curl https://127.0.0.1:7000/api/import/item --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json" --data @input.json -X POST
```
#### input.json Example
```json
{
"id": "submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz",
"tags": [
"infoleak:analyst-detection=\"private-key\"",
"infoleak:analyst-detection=\"api-key\""
],
"galaxy": [
"misp-galaxy:stealer=\"Vidar\""
]
}
```
#### Expected Success Response
**HTTP Status Code** : `200`
```json
{
"id": "submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz",
"tags": [
"infoleak:analyst-detection=\"private-key\"",
"infoleak:analyst-detection=\"api-key\"",
"misp-galaxy:stealer=\"Vidar\""
]
}
```
#### Expected Fail Response
**HTTP Status Code** : `400`
```
{'status': 'error', 'reason': 'Item id not found'}
{'status': 'error', 'reason': 'Tags or Galaxy not specified'}
{'status': 'error', 'reason': 'Tags or Galaxy not enabled'}
```
### Delete item tags: `api/delete/item/tag`
#### Description
Delete tags from an item.
**Method** : `DELETE`
#### Parameters
- `id`
- item id
- *str - relative item path*
- mandatory
- `tags`
- list of tags
- *list*
- default: `[]`
#### JSON response
- `id`
- item id
- *str - relative item path*
- `tags`
- list of item tags deleted
- *list*
#### Example
```
curl https://127.0.0.1:7000/api/delete/item/tag --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json" --data @input.json -X DELETE
```
#### input.json Example
```json
{
"id": "submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz",
"tags": [
"infoleak:analyst-detection=\"private-key\"",
"infoleak:analyst-detection=\"api-key\"",
"misp-galaxy:stealer=\"Vidar\""
]
}
```
#### Expected Success Response
**HTTP Status Code** : `200`
```json
{
"id": "submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz",
"tags": [
"infoleak:analyst-detection=\"private-key\"",
"infoleak:analyst-detection=\"api-key\"",
"misp-galaxy:stealer=\"Vidar\""
]
}
```
#### Expected Fail Response
**HTTP Status Code** : `400`
```
{'status': 'error', 'reason': 'Item id not found'}
{'status': 'error', 'reason': 'No Tag(s) specified'}
```
## Import management
### Import item (currently: text only): `api/import/item`
#### Description
Allows users to import new items. asynchronous function.
**Method** : `POST`
#### Parameters
- `type`
- import type
- *str*
- default: `text`
- `text`
- text to import
- *str*
- mandatory if type = text
- `default_tags`
- add default import tag
- *boolean*
- default: True
- `tags`
- list of tags
- *list*
- default: `[]`
- `galaxy`
- list of galaxy
- *list*
- default: `[]`
#### JSON response
- `uuid`
- import uuid
- *uuid4*
#### Example
```
curl https://127.0.0.1:7000/api/import/item --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json" --data @input.json -X POST
```
#### input.json Example
```json
{
"type": "text",
"tags": [
"infoleak:analyst-detection=\"private-key\""
],
"text": "text to import"
}
```
#### Expected Success Response
**HTTP Status Code** : `200`
```json
{
"uuid": "0c3d7b34-936e-4f01-9cdf-2070184b6016"
}
```
#### Expected Fail Response
**HTTP Status Code** : `400`
```
{'status': 'error', 'reason': 'Malformed JSON'}
{'status': 'error', 'reason': 'No text supplied'}
{'status': 'error', 'reason': 'Tags or Galaxy not enabled'}
{'status': 'error', 'reason': 'Size exceeds default'}
```
### GET Import item info: `api/import/item/<uuid4>`
#### Description
Get import status and all items imported by uuid
**Method** : `GET`
#### Parameters
- `uuid`
- import uuid
- *uuid4*
- mandatory
#### JSON response
- `status`
- import status
- *str*
- values: `in queue`, `in progress`, `imported`
- `items`
- list of imported items id
- *list*
- The full list of imported items is not complete until `status` = `"imported"`
#### Example
```
curl -k https://127.0.0.1:7000/api/import/item/b20a69f1-99ad-4cb3-b212-7ce24b763b50 --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json"
```
#### Expected Success Response
**HTTP Status Code** : `200`
```json
{
"items": [
"submitted/2019/07/26/b20a69f1-99ad-4cb3-b212-7ce24b763b50.gz"
],
"status": "imported"
}
```
#### Expected Fail Response
**HTTP Status Code** : `400`
```
{'status': 'error', 'reason': 'Invalid uuid'}
{'status': 'error', 'reason': 'Unknow uuid'}
```

53
doc/api/submit_paste.py
View File

@ -1,53 +0,0 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
'''
submit your own pastes in AIL
empty values must be initialized
'''
import requests
if __name__ == '__main__':
#AIL url
url = 'http://localhost:7000'
ail_url = url + '/PasteSubmit/submit'
# MIPS TAXONOMIE, need to be initialized (tags_taxonomies = '')
tags_taxonomies = 'CERT-XLM:malicious-code=\"ransomware\",CERT-XLM:conformity=\"standard\"'
# MISP GALAXY, need to be initialized (tags_galaxies = '')
tags_galaxies = 'misp-galaxy:cert-seu-gocsector=\"Constituency\",misp-galaxy:cert-seu-gocsector=\"EU-Centric\"'
# user paste input, need to be initialized (paste_content = '')
paste_content = 'paste content test'
#file full or relative path
file_to_submit = 'test_file.zip'
#compress file password, need to be initialized (password = '')
password = ''
'''
submit user text
'''
r = requests.post(ail_url, data={ 'password': password,
'paste_content': paste_content,
'tags_taxonomies': tags_taxonomies,
'tags_galaxies': tags_galaxies})
print(r.status_code, r.reason)
'''
submit a file
'''
with open(file_submit,'rb') as f:
r = requests.post(ail_url, data={ 'password': password,
'paste_content': paste_content,
'tags_taxonomies': tags_taxonomies,
'tags_galaxies': tags_galaxies}, files={'file': (file_to_submit, f.read() )})
print(r.status_code, r.reason)

BIN
doc/dia/ZMQ_Queuing_Tree.jpg
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 73 KiB

12
var/www/Flask_server.py
View File

@ -37,6 +37,8 @@ import Flask_config
from Role_Manager import create_user_db, check_password_strength, check_user_role_integrity
from Role_Manager import login_admin, login_analyst
Flask_dir = os.environ['AIL_FLASK']
# CONFIG #
cfg = Flask_config.cfg
baseUrl = cfg.get("Flask", "baseurl")
@ -81,7 +83,7 @@ if not os.path.isdir(log_dir):
# ========= TLS =========#
ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2)
ssl_context.load_cert_chain(certfile='server.crt', keyfile='server.key')
ssl_context.load_cert_chain(certfile=os.path.join(Flask_dir, 'server.crt'), keyfile=os.path.join(Flask_dir, 'server.key'))
#print(ssl_context.get_ciphers())
# ========= =========#
@ -112,13 +114,13 @@ try:
toIgnoreModule.add(line)
except IOError:
f = open('templates/ignored_modules.txt', 'w')
f = open(os.path.join(Flask_dir, 'templates', 'ignored_modules.txt'), 'w')
f.close()
# Dynamically import routes and functions from modules
# Also, prepare header.html
to_add_to_header_dico = {}
for root, dirs, files in os.walk('modules/'):
for root, dirs, files in os.walk(os.path.join(Flask_dir, 'modules')):
sys.path.append(join(root))
# Ignore the module
@ -140,7 +142,7 @@ for root, dirs, files in os.walk('modules/'):
#create header.html
complete_header = ""
with open('templates/header_base.html', 'r') as f:
with open(os.path.join(Flask_dir, 'templates', 'header_base.html'), 'r') as f:
complete_header = f.read()
modified_header = complete_header
@ -159,7 +161,7 @@ for module_name, txt in to_add_to_header_dico.items():
modified_header = modified_header.replace('<!--insert here-->', '\n'.join(to_add_to_header))
#Write the header.html file
with open('templates/header.html', 'w') as f:
with open(os.path.join(Flask_dir, 'templates', 'header.html'), 'w') as f:
f.write(modified_header)
# ========= JINJA2 FUNCTIONS ========

149
var/www/modules/Tags/Flask_Tags.py
View File

@ -20,6 +20,7 @@ from pymispgalaxies import Galaxies, Clusters
# ============ VARIABLES ============
import Flask_config
import Tag
app = Flask_config.app
cfg = Flask_config.cfg
@ -59,16 +60,6 @@ for name, tags in clusters.items(): #galaxie name + tags
def one():
return 1
def date_substract_day(date, num_day=1):
new_date = datetime.date(int(date[0:4]), int(date[4:6]), int(date[6:8])) - datetime.timedelta(num_day)
new_date = str(new_date).replace('-', '')
return new_date
def date_add_day(date, num_day=1):
new_date = datetime.date(int(date[0:4]), int(date[4:6]), int(date[6:8])) + datetime.timedelta(num_day)
new_date = str(new_date).replace('-', '')
return new_date
def get_tags_with_synonyms(tag):
str_synonyms = ' - synonyms: '
synonyms = r_serv_tags.smembers('synonym_tag_' + tag)
@ -131,93 +122,6 @@ def get_last_seen_from_tags_list(list_tags):
min_last_seen = tag_last_seen
return str(min_last_seen)
def add_item_tag(tag, item_path):
item_date = int(get_item_date(item_path))
#add tag
r_serv_metadata.sadd('tag:{}'.format(item_path), tag)
r_serv_tags.sadd('{}:{}'.format(tag, item_date), item_path)
r_serv_tags.hincrby('daily_tags:{}'.format(item_date), tag, 1)
tag_first_seen = r_serv_tags.hget('tag_metadata:{}'.format(tag), 'last_seen')
if tag_first_seen is None:
tag_first_seen = 99999999
else:
tag_first_seen = int(tag_first_seen)
tag_last_seen = r_serv_tags.hget('tag_metadata:{}'.format(tag), 'last_seen')
if tag_last_seen is None:
tag_last_seen = 0
else:
tag_last_seen = int(tag_last_seen)
#add new tag in list of all used tags
r_serv_tags.sadd('list_tags', tag)
# update fisrt_seen/last_seen
if item_date < tag_first_seen:
r_serv_tags.hset('tag_metadata:{}'.format(tag), 'first_seen', item_date)
# update metadata last_seen
if item_date > tag_last_seen:
r_serv_tags.hset('tag_metadata:{}'.format(tag), 'last_seen', item_date)
def remove_item_tag(tag, item_path):
item_date = int(get_item_date(item_path))
#remove tag
r_serv_metadata.srem('tag:{}'.format(item_path), tag)
res = r_serv_tags.srem('{}:{}'.format(tag, item_date), item_path)
if res ==1:
# no tag for this day
if int(r_serv_tags.hget('daily_tags:{}'.format(item_date), tag)) == 1:
r_serv_tags.hdel('daily_tags:{}'.format(item_date), tag)
else:
r_serv_tags.hincrby('daily_tags:{}'.format(item_date), tag, -1)
tag_first_seen = int(r_serv_tags.hget('tag_metadata:{}'.format(tag), 'last_seen'))
tag_last_seen = int(r_serv_tags.hget('tag_metadata:{}'.format(tag), 'last_seen'))
# update fisrt_seen/last_seen
if item_date == tag_first_seen:
update_tag_first_seen(tag, tag_first_seen, tag_last_seen)
if item_date == tag_last_seen:
update_tag_last_seen(tag, tag_first_seen, tag_last_seen)
else:
return 'Error incorrect tag'
def update_tag_first_seen(tag, tag_first_seen, tag_last_seen):
if tag_first_seen == tag_last_seen:
if r_serv_tags.scard('{}:{}'.format(tag, tag_first_seen)) > 0:
r_serv_tags.hset('tag_metadata:{}'.format(tag), 'first_seen', tag_first_seen)
# no tag in db
else:
r_serv_tags.srem('list_tags', tag)
r_serv_tags.hdel('tag_metadata:{}'.format(tag), 'first_seen')
r_serv_tags.hdel('tag_metadata:{}'.format(tag), 'last_seen')
else:
if r_serv_tags.scard('{}:{}'.format(tag, tag_first_seen)) > 0:
r_serv_tags.hset('tag_metadata:{}'.format(tag), 'first_seen', tag_first_seen)
else:
tag_first_seen = date_add_day(tag_first_seen)
update_tag_first_seen(tag, tag_first_seen, tag_last_seen)
def update_tag_last_seen(tag, tag_first_seen, tag_last_seen):
if tag_first_seen == tag_last_seen:
if r_serv_tags.scard('{}:{}'.format(tag, tag_last_seen)) > 0:
r_serv_tags.hset('tag_metadata:{}'.format(tag), 'last_seen', tag_last_seen)
# no tag in db
else:
r_serv_tags.srem('list_tags', tag)
r_serv_tags.hdel('tag_metadata:{}'.format(tag), 'first_seen')
r_serv_tags.hdel('tag_metadata:{}'.format(tag), 'last_seen')
else:
if r_serv_tags.scard('{}:{}'.format(tag, tag_last_seen)) > 0:
r_serv_tags.hset('tag_metadata:{}'.format(tag), 'last_seen', tag_last_seen)
else:
tag_last_seen = date_substract_day(tag_last_seen)
update_tag_last_seen(tag, tag_first_seen, tag_last_seen)
# ============= ROUTES ==============
@Tags.route("/tags/", methods=['GET'])
@ -472,8 +376,9 @@ def remove_tag():
path = request.args.get('paste')
tag = request.args.get('tag')
remove_item_tag(tag, path)
res = Tag.remove_item_tag(tag, path)
if res[1] != 200:
str(res[0])
return redirect(url_for('showsavedpastes.showsavedpaste', paste=path))
@Tags.route("/Tags/confirm_tag")
@ -486,11 +391,11 @@ def confirm_tag():
tag = request.args.get('tag')
if(tag[9:28] == 'automatic-detection'):
remove_item_tag(tag, path)
Tag.remove_item_tag(tag, path)
tag = tag.replace('automatic-detection','analyst-detection', 1)
#add analyst tag
add_item_tag(tag, path)
Tag.add_item_tag(tag, path)
return redirect(url_for('showsavedpastes.showsavedpaste', paste=path))
@ -530,42 +435,12 @@ def addTags():
list_tag = tags.split(',')
list_tag_galaxies = tagsgalaxies.split(',')
taxonomies = Taxonomies()
active_taxonomies = r_serv_tags.smembers('active_taxonomies')
active_galaxies = r_serv_tags.smembers('active_galaxies')
if not path:
return 'INCORRECT INPUT0'
if list_tag != ['']:
for tag in list_tag:
# verify input
tax = tag.split(':')[0]
if tax in active_taxonomies:
if tag in r_serv_tags.smembers('active_tag_' + tax):
add_item_tag(tag, path)
else:
return 'INCORRECT INPUT1'
else:
return 'INCORRECT INPUT2'
if list_tag_galaxies != ['']:
for tag in list_tag_galaxies:
# verify input
gal = tag.split(':')[1]
gal = gal.split('=')[0]
if gal in active_galaxies:
if tag in r_serv_tags.smembers('active_tag_galaxies_' + gal):
add_item_tag(tag, path)
else:
return 'INCORRECT INPUT3'
else:
return 'INCORRECT INPUT4'
res = Tag.add_items_tag(list_tag, list_tag_galaxies, path)
print(res)
# error
if res[1] != 200:
return str(res[0])
# success
return redirect(url_for('showsavedpastes.showsavedpaste', paste=path))

379
var/www/modules/restApi/Flask_restApi.py
View File

@ -16,7 +16,7 @@ import datetime
import Import_helper
import Item
import Paste
import Tags
import Tag
from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response
from flask_login import login_required
@ -146,24 +146,49 @@ def items():
return Response(json.dumps({'test': 2}), mimetype='application/json')
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# GET
#
# {
# "id": item_id, mandatory
# }
#
# response: {
# "id": "item_id",
# "date": "date",
# "tags": [],
# "content": "item content"
# }
#
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
@restApi.route("api/get/item/info/<path:item_id>", methods=['GET'])
@token_required('admin')
def get_item_id(item_id):
"""
**GET api/get/item/info/<item id>**
**Get item**
This function allows user to get a specific item information through their item_id.
:param id: id of the item
:type id: item id
:return: item's information in json and http status code
- Example::
curl -k https://127.0.0.1:7000/api/get/item/info/submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json"
- Expected Success Response::
HTTP Status Code: 200
{
"content": "item content test",
"date": "20190726",
"id": "submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz",
"tags":
[
"misp-galaxy:backdoor=\"Rosenbridge\"",
"infoleak:automatic-detection=\"pgp-message\"",
"infoleak:automatic-detection=\"encrypted-private-key\"",
"infoleak:submission=\"manual\"",
"misp-galaxy:backdoor=\"SLUB\""
]
}
- Expected Fail Response::
HTTP Status Code: 400
{'status': 'error', 'reason': 'Item not found'}
"""
try:
item_object = Paste.Paste(item_id)
except FileNotFoundError:
@ -188,14 +213,214 @@ def get_item_id(item_id):
@restApi.route("api/get/item/tag/<path:item_id>", methods=['GET'])
@token_required('admin')
def get_item_tag(item_id):
"""
**GET api/get/item/tag/<item id>**
**Get item tags**
This function allows user to get all items tags form a specified item id.
:param id: id of the item
:type id: item id
:return: item's tags list in json and http status code
- Example::
curl -k https://127.0.0.1:7000/api/get/item/tag/submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json"
- Expected Success Response::
HTTP Status Code: 200
{
"id": "submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz",
"tags":
[
"misp-galaxy:backdoor=\"Rosenbridge\"",
"infoleak:automatic-detection=\"pgp-message\"",
"infoleak:automatic-detection=\"encrypted-private-key\"",
"infoleak:submission=\"manual\"",
"misp-galaxy:backdoor=\"SLUB\""
]
}
- Expected Fail Response::
HTTP Status Code: 400
{'status': 'error', 'reason': 'Item not found'}
"""
if not Item.exist_item(item_id):
return Response(json.dumps({'status': 'error', 'reason': 'Item not found'}, indent=2, sort_keys=True), mimetype='application/json'), 400
tags = Tags.get_item_tags(item_id)
tags = Tag.get_item_tags(item_id)
dict_tags = {}
dict_tags['id'] = item_id
dict_tags['tags'] = tags
return Response(json.dumps(dict_tags, indent=2, sort_keys=True), mimetype='application/json')
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# POST
#
# {
# "id": item_id, mandatory
# "tags": [tags to add],
# "galaxy": [galaxy to add],
# }
#
# response: {
# "id": "item_id",
# "tags": [tags added],
# }
#
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
@restApi.route("api/add/item/tag", methods=['POST'])
@token_required('admin')
def add_item_tags():
"""
**POST api/add/item/tag**
**add tags to an item**
This function allows user to add tags and galaxy to an item.
:param id: id of the item
:type id: item id
:param tags: list of tags (default=[])
:type tags: list
:param galaxy: list of galaxy (default=[])
:type galaxy: list
:return: item id and tags added in json and http status code
- Example::
curl -k https://127.0.0.1:7000/api/add/item/tag --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json" --data @input.json -X POST
- input.json Example::
{
"id": "submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz",
"tags": [
"infoleak:analyst-detection=\"private-key\"",
"infoleak:analyst-detection=\"api-key\""
],
"galaxy": [
"misp-galaxy:stealer=\"Vidar\""
]
}
- Expected Success Response::
HTTP Status Code: 200
{
"id": "submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz",
"tags": [
"infoleak:analyst-detection=\"private-key\"",
"infoleak:analyst-detection=\"api-key\"",
"misp-galaxy:stealer=\"Vidar\""
]
}
- Expected Fail Response::
HTTP Status Code: 400
{'status': 'error', 'reason': 'Item id not found'}
{'status': 'error', 'reason': 'Tags or Galaxy not specified'}
{'status': 'error', 'reason': 'Tags or Galaxy not enabled'}
"""
data = request.get_json()
if not data:
return Response(json.dumps({'status': 'error', 'reason': 'Malformed JSON'}, indent=2, sort_keys=True), mimetype='application/json'), 400
item_id = data.get('id', None)
tags = data.get('tags', [])
galaxy = data.get('galaxy', [])
res = Tag.add_items_tag(tags, galaxy, item_id)
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# DELETE
#
# {
# "id": item_id, mandatory
# "tags": [tags to delete],
# }
#
# response: {
# "id": "item_id",
# "tags": [tags deleted],
# }
#
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
@restApi.route("api/delete/item/tag", methods=['DELETE'])
@token_required('admin')
def delete_item_tags():
"""
**DELET E api/delete/item/tag**
**delete tags from an item**
This function allows user to delete tags and galaxy from an item.
:param id: id of the item
:type id: item id
:param tags: list of tags (default=[])
:type tags: list
:return: item id and tags deleted in json and http status code
- Example::
curl -k https://127.0.0.1:7000/api/delete/item/tag --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json" --data @input.json -X DELET E
- input.json Example::
{
"id": "submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz",
"tags": [
"infoleak:analyst-detection=\"private-key\"",
"infoleak:analyst-detection=\"api-key\"",
"misp-galaxy:stealer=\"Vidar\""
]
}
- Expected Success Response::
HTTP Status Code: 200
{
"id": "submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz",
"tags": [
"infoleak:analyst-detection=\"private-key\"",
"infoleak:analyst-detection=\"api-key\"",
"misp-galaxy:stealer=\"Vidar\""
]
}
- Expected Fail Response::
HTTP Status Code: 400
{'status': 'error', 'reason': 'Item id not found'}
{'status': 'error', 'reason': 'No Tag(s) specified}
{'status': 'error', 'reason': 'Malformed JSON'}
"""
data = request.get_json()
if not data:
return Response(json.dumps({'status': 'error', 'reason': 'Malformed JSON'}, indent=2, sort_keys=True), mimetype='application/json'), 400
item_id = data.get('id', None)
tags = data.get('tags', [])
res = Tag.remove_item_tags(tags, item_id)
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# GET
#
@ -212,6 +437,37 @@ def get_item_tag(item_id):
@restApi.route("api/get/item/content/<path:item_id>", methods=['GET'])
@token_required('admin')
def get_item_content(item_id):
"""
**GET api/get/item/content/<item id>**
**Get item content**
This function allows user to get a specific item content.
:param id: id of the item
:type id: item id
:return: item's content in json and http status code
- Example::
curl -k https://127.0.0.1:7000/api/get/item/content/submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json"
- Expected Success Response::
HTTP Status Code: 200
{
"content": "item content test",
"id": "submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz"
}
- Expected Fail Response::
HTTP Status Code: 400
{'status': 'error', 'reason': 'Item not found'}
"""
try:
item_object = Paste.Paste(item_id)
except FileNotFoundError:
@ -240,6 +496,58 @@ def get_item_content(item_id):
@restApi.route("api/import/item", methods=['POST'])
@token_required('admin')
def import_item():
"""
**POST api/import/item**
**Import new item**
This function allows user to import new items. asynchronous function.
:param text: text to import
:type text: str
:param type: import type (default='text')
:type type: "text"
:param tags: list of tags (default=[])
:type tags: list
:param galaxy: list of galaxy (default=[])
:type galaxy: list
:param default_tags: add default tag (default=True)
:type default_tags: boolean
:return: imported uuid in json and http status code
- Example::
curl -k https://127.0.0.1:7000/api/import/item --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json" --data @input.json -X POST
- input.json Example::
{
"type": "text",
"tags": [
"infoleak:analyst-detection=\"private-key\""
],
"text": "text to import"
}
- Expected Success Response::
HTTP Status Code: 200
{
"uuid": "0c3d7b34-936e-4f01-9cdf-2070184b6016"
}
- Expected Fail Response::
HTTP Status Code: 400
{'status': 'error', 'reason': 'Malformed JSON'}
{'status': 'error', 'reason': 'No text supplied'}
{'status': 'error', 'reason': 'Tags or Galaxy not enabled'}
{'status': 'error', 'reason': 'Size exceeds default'}
"""
data = request.get_json()
if not data:
return Response(json.dumps({'status': 'error', 'reason': 'Malformed JSON'}, indent=2, sort_keys=True), mimetype='application/json'), 400
@ -256,7 +564,7 @@ def import_item():
if not type(galaxy) is list:
galaxy = []
if not Tags.is_valid_tags_taxonomies_galaxy(tags, galaxy):
if not Tag.is_valid_tags_taxonomies_galaxy(tags, galaxy):
return Response(json.dumps({'status': 'error', 'reason': 'Tags or Galaxy not enabled'}, indent=2, sort_keys=True), mimetype='application/json'), 400
default_tags = data.get('default_tags', True)
@ -287,6 +595,41 @@ def import_item():
@restApi.route("api/import/item/<UUID>", methods=['GET'])
@token_required('admin')
def import_item_uuid(UUID):
"""
**GET api/import/item/<uuid4>**
**Get import status and all items imported by uuid**
This return the import status and a list of imported items.
The full list of imported items is not complete until 'status'='imported'.
:param uuid: import uuid
:type uuid: uuid4
:return: json: import status + imported items list
- Example::
curl -k https://127.0.0.1:7000/api/import/item/b20a69f1-99ad-4cb3-b212-7ce24b763b50 --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json"
- Expected Success Response::
HTTP Status Code: 200
{
"items": [
"submitted/2019/07/26/b20a69f1-99ad-4cb3-b212-7ce24b763b50.gz"
],
"status": "in queue"/"in progress"/"imported"
}
- Expected Fail Response::
HTTP Status Code: 400
{'status': 'error', 'reason': 'Invalid uuid'}
{'status': 'error', 'reason': 'Unknow uuid'}
"""
# Verify uuid
if not is_valid_uuid_v4(UUID):