CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

chg: [API] import item (text)

pull/370/head
Terrtia 2019-07-26 14:28:02 +02:00
parent 3a8531cafa
commit 0a756294fe
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
8 changed files with 138 additions and 89 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
OVERVIEW.md
View File

@ -97,6 +97,18 @@ Redis and ARDB overview
| ------ | ------ | ------ |
| ail:all_role | **role** | **int, role priority (1=admin)** |
##### Item Import:
| Key | Value |
| ------ | ------ |
| **uuid**:isfile | **boolean** |
| **uuid**:paste_content | **item_content** |
| Set Key | Value |
| ------ | ------ |
| submitted:uuid | **uuid** |
| **uuid**:ltags | **tag** |
| **uuid**:ltagsgalaxies | **tag** |
## DB2 - TermFreq:
##### Set:

6
bin/LAUNCH.sh
View File

@ -66,8 +66,8 @@ function helptext {
"$DEFAULT"
This script launch:
"$CYAN"
- All the ZMQ queuing modules.
- All the ZMQ processing modules.
- All the queuing modules.
- All the processing modules.
- All Redis in memory servers.
- All ARDB on disk servers.
"$DEFAULT"
@ -143,7 +143,7 @@ function launching_scripts {
screen -dmS "Script_AIL"
sleep 0.1
echo -e $GREEN"\t* Launching ZMQ scripts"$DEFAULT
echo -e $GREEN"\t* Launching scripts"$DEFAULT
screen -S "Script_AIL" -X screen -t "ModuleInformation" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./ModulesInformationV2.py -k 0 -c 1; read x"
sleep 0.1

68
bin/packages/Import_helper.py
View File

@ -2,33 +2,75 @@
# -*-coding:UTF-8 -*
import os
import uuid
import redis
import Flask_config
r_serv_db = Flask_config.r_serv_db
r_serv_log = Flask_config.r_serv_log
r_serv_log_submit = Flask_config.r_serv_log_submit
def create_import_queue(ltags, ltagsgalaxies, paste_content, UUID, password, isfile = False):
def is_valid_uuid_v4(UUID):
UUID = UUID.replace('-', '')
try:
uuid_test = uuid.UUID(hex=UUID, version=4)
return uuid_test.hex == UUID
except:
return False
def create_import_queue(tags, galaxy, paste_content, UUID, password=None, isfile = False):
# save temp value on disk
r_serv_db.set(UUID + ':ltags', ltags)
r_serv_db.set(UUID + ':ltagsgalaxies', ltagsgalaxies)
for tag in tags:
r_serv_db.sadd(UUID + ':ltags', tag)
for tag in galaxy:
r_serv_db.sadd(UUID + ':ltagsgalaxies', tag)
r_serv_db.set(UUID + ':paste_content', paste_content)
if password:
r_serv_db.set(UUID + ':password', password)
r_serv_db.set(UUID + ':isfile', isfile)
r_serv_log.set(UUID + ':end', 0)
r_serv_log.set(UUID + ':processing', 0)
r_serv_log.set(UUID + ':nb_total', -1)
r_serv_log.set(UUID + ':nb_end', 0)
r_serv_log.set(UUID + ':nb_sucess', 0)
r_serv_log_submit.set(UUID + ':end', 0)
r_serv_log_submit.set(UUID + ':processing', 0)
r_serv_log_submit.set(UUID + ':nb_total', -1)
r_serv_log_submit.set(UUID + ':nb_end', 0)
r_serv_log_submit.set(UUID + ':nb_sucess', 0)
# save UUID on disk
r_serv_db.sadd('submitted:uuid', UUID)
return UUID
def import_text_item():
res = r_serv_db.smembers('submitted:uuid')
print(res)
return res
def check_import_status(UUID):
if not is_valid_uuid_v4(UUID):
return ({'status': 'error', 'reason': 'Invalid uuid'}, 400)
processing = r_serv_log_submit.get(UUID + ':processing')
if not processing:
return ({'status': 'error', 'reason': 'Unknow uuid'}, 400)
# nb_total = r_serv_log_submit.get(UUID + ':nb_total')
# nb_sucess = r_serv_log_submit.get(UUID + ':nb_sucess')
# nb_end = r_serv_log_submit.get(UUID + ':nb_end')
items_id = list(r_serv_log_submit.smembers(UUID + ':paste_submit_link'))
error = r_serv_log_submit.get(UUID + ':error')
end = r_serv_log_submit.get(UUID + ':end')
dict_import_status = {}
if items_id:
dict_import_status['items'] = items_id
if error:
dict_import_status['error'] = error
if processing == '0':
status = 'in queue'
else:
if end == '0':
status = 'in progress'
else:
status = 'imported'
dict_import_status['status'] = status
return (dict_import_status, 200)

8
bin/packages/Tags.py
View File

@ -31,7 +31,7 @@ def is_taxonomie_tag_enabled(taxonomie, tag):
else:
return False
def is_galaxy_tag_enabled(taxonomie, galaxy):
def is_galaxy_tag_enabled(galaxy, tag):
if tag in r_serv_tags.smembers('active_tag_galaxies_' + galaxy):
return True
else:
@ -39,8 +39,10 @@ def is_galaxy_tag_enabled(taxonomie, galaxy):
# Check if tags are enabled in AIL
def is_valid_tags_taxonomies_galaxy(list_tags, list_tags_galaxy):
print(list_tags)
print(list_tags_galaxy)
if list_tags:
active_taxonomies = Tags.get_active_taxonomies()
active_taxonomies = get_active_taxonomies()
for tag in list_tags:
taxonomie = get_taxonomie_from_tag(tag)
@ -50,7 +52,7 @@ def is_valid_tags_taxonomies_galaxy(list_tags, list_tags_galaxy):
return False
if list_tags_galaxy:
active_galaxies = Tags.get_active_galaxies()
active_galaxies = get_active_galaxies()
for tag in list_tags_galaxy:
galaxy = get_galaxy_from_tag(tag)

24
bin/submit_paste.py
View File

@ -47,7 +47,11 @@ def create_paste(uuid, paste_content, ltags, ltagsgalaxies, name):
r_serv_log_submit.hincrby("mixer_cache:list_feeder", "submitted", 1)
# add tags
add_tags(ltags, ltagsgalaxies, rel_item_path)
for tag in ltags:
add_item_tag(tag, rel_item_path)
for tag in ltagsgalaxies:
add_item_tag(tag, rel_item_path)
r_serv_log_submit.incr(uuid + ':nb_end')
r_serv_log_submit.incr(uuid + ':nb_sucess')
@ -133,18 +137,6 @@ def add_item_tag(tag, item_path):
if item_date > tag_last_seen:
r_serv_tags.hset('tag_metadata:{}'.format(tag), 'last_seen', item_date)
def add_tags(tags, tagsgalaxies, path):
list_tag = tags.split(',')
list_tag_galaxies = tagsgalaxies.split(',')
if list_tag != ['']:
for tag in list_tag:
add_item_tag(tag, path)
if list_tag_galaxies != ['']:
for tag in list_tag_galaxies:
add_item_tag(tag, path)
def verify_extention_filename(filename):
if not '.' in filename:
return True
@ -217,8 +209,8 @@ if __name__ == "__main__":
uuid = r_serv_db.srandmember('submitted:uuid')
# get temp value save on disk
ltags = r_serv_db.get(uuid + ':ltags')
ltagsgalaxies = r_serv_db.get(uuid + ':ltagsgalaxies')
ltags = r_serv_db.smembers(uuid + ':ltags')
ltagsgalaxies = r_serv_db.smembers(uuid + ':ltagsgalaxies')
paste_content = r_serv_db.get(uuid + ':paste_content')
isfile = r_serv_db.get(uuid + ':isfile')
password = r_serv_db.get(uuid + ':password')
@ -272,7 +264,7 @@ if __name__ == "__main__":
else:
#decompress file
try:
if password == '':
if password == None:
files = unpack(file_full_path.encode())
#print(files.children)
else:

2
var/www/modules/Flask_config.py
View File

@ -178,6 +178,8 @@ crawler_enabled = cfg.getboolean("Crawler", "activate_crawler")
email_regex = r'[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,6}'
email_regex = re.compile(email_regex)
IMPORT_MAX_TEXT_SIZE = 900000 # size in bytes
# VT
try:
from virusTotalKEYS import vt_key

45
var/www/modules/PasteSubmit/Flask_PasteSubmit.py
View File

@ -90,27 +90,6 @@ def clean_filename(filename, whitelist=valid_filename_chars, replace=' '):
# keep only whitelisted chars
return ''.join(c for c in cleaned_filename if c in whitelist)
def launch_submit(ltags, ltagsgalaxies, paste_content, UUID, password, isfile = False):
# save temp value on disk
r_serv_db.set(UUID + ':ltags', ltags)
r_serv_db.set(UUID + ':ltagsgalaxies', ltagsgalaxies)
r_serv_db.set(UUID + ':paste_content', paste_content)
r_serv_db.set(UUID + ':password', password)
r_serv_db.set(UUID + ':isfile', isfile)
r_serv_log_submit.set(UUID + ':end', 0)
r_serv_log_submit.set(UUID + ':processing', 0)
r_serv_log_submit.set(UUID + ':nb_total', -1)
r_serv_log_submit.set(UUID + ':nb_end', 0)
r_serv_log_submit.set(UUID + ':nb_sucess', 0)
r_serv_log_submit.set(UUID + ':error', 'error:')
r_serv_log_submit.sadd(UUID + ':paste_submit_link', '')
# save UUID on disk
r_serv_db.sadd('submitted:uuid', UUID)
def date_to_str(date):
return "{0}-{1}-{2}".format(date.year, date.month, date.day)
@ -264,9 +243,6 @@ def submit():
ltagsgalaxies = request.form['tags_galaxies']
paste_content = request.form['paste_content']
print(ltags)
print(ltagsgalaxies)
is_file = False
if 'file' in request.files:
file = request.files['file']
@ -283,8 +259,11 @@ def submit():
if ltags or ltagsgalaxies:
list_tag = tags.split(',')
list_tag_galaxies = tagsgalaxies.split(',')
ltags = ltags.split(',')
ltagsgalaxies = ltagsgalaxies.split(',')
print(ltags)
print(ltagsgalaxies)
if not Tags.is_valid_tags_taxonomies_galaxy(ltags, ltagsgalaxies):
content = 'INVALID TAGS'
@ -292,10 +271,9 @@ def submit():
return content, 400
# add submitted tags
if(ltags != ''):
ltags = ltags + ',' + submitted_tag
else:
ltags = submitted_tag
if not ltags:
ltags = []
ltags.append(submitted_tag)
if is_file:
if file:
@ -346,11 +324,6 @@ def submit():
# get id
UUID = str(uuid.uuid4())
#if paste_name:
# clean file name
#id = clean_filename(paste_name)
Import_helper.create_import_queue(ltags, ltagsgalaxies, paste_content, UUID, password)
return render_template("submit_items.html",
@ -385,7 +358,7 @@ def submit_status():
nb_sucess = r_serv_log_submit.get(UUID + ':nb_sucess')
paste_submit_link = list(r_serv_log_submit.smembers(UUID + ':paste_submit_link'))
if (end != None) and (nb_total != None) and (nb_end != None) and (error != None) and (processing != None) and (paste_submit_link != None):
if (end != None) and (nb_total != None) and (nb_end != None) and (processing != None):
link = ''
if paste_submit_link:

60
var/www/modules/restApi/Flask_restApi.py
View File

@ -14,6 +14,7 @@ import redis
import datetime
import Import_helper
import Tags
from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response
from flask_login import login_required
@ -151,24 +152,14 @@ def items():
# {
# "type": "text", (default value)
# "tags": [], (default value)
# "default_ags": True, (default value)
# "default_tags": True, (default value)
# "galaxy" [], (default value)
# "text": "", mandatory if type = text
# }
#
# response: {"uuid": "uuid"}
#
# # # #
# GET
#
# {
# "uuid": "uuid", mandatory
# }
#
# response: {"uuid": "uuid"}
#
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
@restApi.route("api/import/item", methods=['POST'])
@token_required('admin')
def import_item():
@ -176,24 +167,59 @@ def import_item():
if not data:
return Response(json.dumps({'status': 'error', 'reason': 'Malformed JSON'}, indent=2, sort_keys=True), mimetype='application/json'), 400
# TODO: add submitted tag
# unpack json
text_to_import = data.get('text', None)
if not text_to_import:
return Response(json.dumps({'status': 'error', 'reason': 'No text supplied'}, indent=2, sort_keys=True), mimetype='application/json'), 400
UUID = 'uuuuuuu'
tags = data.get('tags', [])
if not type(tags) is list:
tags = []
galaxy = data.get('galaxy', [])
if not type(galaxy) is list:
galaxy = []
if not Tags.is_valid_tags_taxonomies_galaxy(tags, galaxy):
return Response(json.dumps({'status': 'error', 'reason': 'Tags or Galaxy not enabled'}, indent=2, sort_keys=True), mimetype='application/json'), 400
default_tags = data.get('default_tags', True)
if default_tags:
tags.append('infoleak:submission="manual"')
if sys.getsizeof(text_to_import) > 900000:
return Response(json.dumps({'status': 'error', 'reason': 'Size exceeds default'}, indent=2, sort_keys=True), mimetype='application/json'), 400
UUID = str(uuid.uuid4())
Import_helper.create_import_queue(tags, galaxy, text_to_import, UUID)
return Response(json.dumps({'uuid': UUID}, indent=2, sort_keys=True), mimetype='application/json')
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# GET
#
# {
# "uuid": "uuid", mandatory
# }
#
# response: {
# "status": "in queue"/"in progress"/"imported",
# "items": [all item id]
# }
#
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
@restApi.route("api/import/item/<UUID>", methods=['GET'])
@token_required('admin')
def import_item_uuid(UUID):
# Verify uuid
if not is_valid_uuid_v4(UUID):
Response(json.dumps({'status': 'error', 'reason': 'Invalid uuid'}), mimetype='application/json'), 400
return Response(json.dumps({'status': 'error', 'reason': 'Invalid uuid'}), mimetype='application/json'), 400
data = Import_helper.check_import_status(UUID)
if data:
return Response(json.dumps(data[0]), mimetype='application/json'), data[1]
return Response(json.dumps({'item_id': 4}), mimetype='application/json')
return Response(json.dumps({'status': 'error', 'reason': 'Invalid response'}), mimetype='application/json'), 400
# ========= REGISTRATION =========
app.register_blueprint(restApi, url_prefix=baseUrl)