diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index ed05f676..704d64cc 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -163,6 +163,8 @@ function launching_scripts { screen -S "Script_AIL" -X screen -t "Tags" bash -c './Tags.py; read x' sleep 0.1 screen -S "Script_AIL" -X screen -t "SentimentAnalysis" bash -c './SentimentAnalysis.py; read x' + sleep 0.1 + screen -S "Script_AIL" -X screen -t "SubmitPaste" bash -c './submit_paste.py; read x' } diff --git a/bin/submit_paste.py b/bin/submit_paste.py index 048a4c43..12ae14af 100755 --- a/bin/submit_paste.py +++ b/bin/submit_paste.py @@ -9,8 +9,80 @@ import io import redis import base64 import datetime +import time + +from sflock.main import unpack +import sflock from Helper import Process +from pubsublogger import publisher + +def create_paste(uuid, paste_content, ltags, ltagsgalaxies, name): + + now = datetime.datetime.now() + save_path = 'submitted/' + now.strftime("%Y") + '/' + now.strftime("%m") + '/' + now.strftime("%d") + '/' + name + '.gz' + + full_path = filename = os.path.join(os.environ['AIL_HOME'], + p.config.get("Directories", "pastes"), save_path) + + if os.path.isfile(full_path): + addError(uuid, 'File: ' + save_path + ' already exist in submitted pastes') + return 1 + + gzipencoded = gzip.compress(paste_content) + gzip64encoded = base64.standard_b64encode(gzipencoded).decode() + + # send paste to Global module + relay_message = "{0} {1}".format(save_path, gzip64encoded) + p.populate_set_out(relay_message, 'Mixer') + + # add tags + add_tags(ltags, ltagsgalaxies, full_path) + + r_serv_log_submit.incr(uuid + ':nb_end') + r_serv_log_submit.incr(uuid + ':nb_sucess') + + if r_serv_log_submit.get(uuid + ':nb_end') == r_serv_log_submit.get(uuid + ':nb_total'): + r_serv_log_submit.set(uuid + ':end', 1) + + print(' {} send to Global'.format(save_path)) + r_serv_log_submit.sadd(uuid + ':paste_submit_link', full_path) + + return 0 + +def addError(uuid, errorMessage): + print(errorMessage) + error = r_serv_log_submit.get(uuid + ':error') + if error != None: + r_serv_log_submit.set(uuid + ':error', error + '

' + errorMessage) + r_serv_log_submit.incr(uuid + ':nb_end') + +def abord_file_submission(uuid, errorMessage): + addError(uuid, errorMessage) + r_serv_log_submit.set(uuid + ':end', 1) + remove_submit_uuid(uuid) + + +def remove_submit_uuid(uuid): + # save temp value on disk + r_serv_db.delete(uuid + ':ltags') + r_serv_db.delete(uuid + ':ltagsgalaxies') + r_serv_db.delete(uuid + ':paste_content') + r_serv_db.delete(uuid + ':isfile') + r_serv_db.delete(uuid + ':password') + + r_serv_log_submit.expire(uuid + ':end', expire_time) + r_serv_log_submit.expire(uuid + ':processing', expire_time) + r_serv_log_submit.expire(uuid + ':nb_total', expire_time) + r_serv_log_submit.expire(uuid + ':nb_sucess', expire_time) + r_serv_log_submit.expire(uuid + ':nb_end', expire_time) + r_serv_log_submit.expire(uuid + ':error', expire_time) + r_serv_log_submit.srem(uuid + ':paste_submit_link', '') + r_serv_log_submit.expire(uuid + ':paste_submit_link', expire_time) + + # delete uuid + r_serv_db.srem('submitted:uuid', uuid) + print('{} all file submitted'.format(uuid)) def add_tags(tags, tagsgalaxies, path): list_tag = tags.split(',') @@ -32,21 +104,22 @@ def add_tags(tags, tagsgalaxies, path): #add new tag in list of all used tags r_serv_tags.sadd('list_tags', tag) +def verify_extention_filename(filename): + if not '.' in filename: + return True + else: + file_type = filename.rsplit('.', 1)[1] + + #txt file + if file_type in ALLOWED_EXTENSIONS: + return True + else: + return False if __name__ == "__main__": - if len(sys.argv) != 6: - print('usage:', 'submit_paste.py', 'ltags', 'ltagsgalaxies', 'paste_content', 'paste_name', 'id') - exit(1) - try: - ltags = sys.argv[1] - ltagsgalaxies = sys.argv[2] - paste_content = sys.argv[3] - paste_name = sys.argv[4] - id = sys.argv[5] - except: - print('unable to get elements') - exit(1) + publisher.port = 6380 + publisher.channel = "Script" configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg') if not os.path.exists(configfile): @@ -57,6 +130,12 @@ if __name__ == "__main__": cfg = configparser.ConfigParser() cfg.read(configfile) + r_serv_db = redis.StrictRedis( + host=cfg.get("ARDB_DB", "host"), + port=cfg.getint("ARDB_DB", "port"), + db=cfg.getint("ARDB_DB", "db"), + decode_responses=True) + r_serv_log_submit = redis.StrictRedis( host=cfg.get("Redis_Log_submit", "host"), port=cfg.getint("Redis_Log_submit", "port"), @@ -75,43 +154,117 @@ if __name__ == "__main__": db=cfg.getint("ARDB_Metadata", "db"), decode_responses=True) - # TODO put on config - expire_time = 10200 - - r_serv_log_submit.expire(id + ':end', expire_time) - r_serv_log_submit.expire(id + ':nb_total', expire_time) - r_serv_log_submit.expire(id + ':nb_end', expire_time) - r_serv_log_submit.expire(id + ':error', expire_time) + expire_time = 120 + MAX_FILE_SIZE = 1000000000 + ALLOWED_EXTENSIONS = 'txt' config_section = 'submit_paste' p = Process(config_section) - now = datetime.datetime.now() - save_path = 'submitted/' + now.strftime("%Y") + '/' + now.strftime("%m") + '/' + now.strftime("%d") + '/' + id + '.gz' + while True: - full_path = filename = os.path.join(os.environ['AIL_HOME'], - p.config.get("Directories", "pastes"), save_path) + # paste submitted + if r_serv_db.scard('submitted:uuid') > 0: + uuid = r_serv_db.srandmember('submitted:uuid') - if os.path.isfile(full_path): - error = r_serv_log_submit.get(id + ':error') - r_serv_log_submit.set(id + ':error', error + '

File: ' + save_path + ' already exist in submitted pastes') - exit(1) + # get temp value save on disk + ltags = r_serv_db.get(uuid + ':ltags') + ltagsgalaxies = r_serv_db.get(uuid + ':ltagsgalaxies') + paste_content = r_serv_db.get(uuid + ':paste_content') + isfile = r_serv_db.get(uuid + ':isfile') + password = r_serv_db.get(uuid + ':password') + + # needed if redis is restarted + r_serv_log_submit.set(uuid + ':end', 0) + r_serv_log_submit.set(uuid + ':processing', 0) + r_serv_log_submit.set(uuid + ':nb_total', -1) + r_serv_log_submit.set(uuid + ':nb_end', 0) + r_serv_log_submit.set(uuid + ':nb_sucess', 0) + r_serv_log_submit.set(uuid + ':error', 'error:') + r_serv_log_submit.sadd(uuid + ':paste_submit_link', '') - gzipencoded = gzip.compress(paste_content.encode()) - gzip64encoded = base64.standard_b64encode(gzipencoded).decode() + r_serv_log_submit.set(uuid + ':processing', 1) - # send paste to Global module - relay_message = "{0} {1}".format(save_path, gzip64encoded) - p.populate_set_out(relay_message, 'Mixer') + if isfile == 'True': + file_full_path = paste_content - # add tags - add_tags(ltags, ltagsgalaxies, full_path) + if not os.path.exists(file_full_path): + abord_file_submission(uuid, "Server Error, the archive can't be found") + continue - r_serv_log_submit.incr(id + ':nb_end') + #verify file lengh + if os.stat(file_full_path).st_size > MAX_FILE_SIZE: + abord_file_submission(uuid, 'File :{} too large'.format(file_full_path)) + + else: + filename = file_full_path.split('/')[-1] + if not '.' in filename: + # read file + with open(file_full_path,'r') as f: + content = f.read() + r_serv_log_submit.set(uuid + ':nb_total', 1) + create_paste(uuid, content.encode(), ltags, ltagsgalaxies, uuid) + remove_submit_uuid(uuid) + + else: + file_type = filename.rsplit('.', 1)[1] + + #txt file + if file_type == 'txt': + with open(file_full_path,'r') as f: + content = f.read() + r_serv_log_submit.set(uuid + ':nb_total', 1) + create_paste(uuid, content.encode(), ltags, ltagsgalaxies, uuid) + remove_submit_uuid(uuid) + #compressed file + else: + #decompress file + try: + if password == '': + files = unpack(file_full_path.encode()) + #print(files.children) + else: + try: + files = unpack(file_full_path.encode(), password) + #print(files.children) + except sflock.exception.IncorrectUsageException: + abord_file_submission(uuid, "Wrong Password") + continue + except: + abord_file_submission(uuid, "file decompression error") + continue + print('unpacking {} file'.format(files.unpacker)) + if(not files.children): + abord_file_submission(uuid, "Empty compressed file") + continue + # set number of files to submit + r_serv_log_submit.set(uuid + ':nb_total', len(files.children)) + n = 1 + for child in files.children: + if verify_extention_filename(child.filename.decode()): + create_paste(uuid, child.contents, ltags, ltagsgalaxies, uuid+'_'+ str(n) ) + n = n + 1 + else: + print('bad extention') + addError(uuid, 'Bad file extension: {}'.format(child.filename.decode()) ) + + except FileNotFoundError: + print('file not found') + addError(uuid, 'File not found: {}'.format(file_full_path), uuid ) + + remove_submit_uuid(uuid) - if r_serv_log_submit.get(id + ':nb_end') == r_serv_log_submit.get(id + ':nb_total'): - r_serv_log_submit.set(id + ':end', 1) - exit(0) + # textarea input paste + else: + r_serv_log_submit.set(uuid + ':nb_total', 1) + create_paste(uuid, paste_content.encode(), ltags, ltagsgalaxies, uuid) + remove_submit_uuid(uuid) + time.sleep(0.5) + + # wait for paste + else: + publisher.debug("Script submit_paste is Idling 10s") + time.sleep(3) diff --git a/installing_deps.sh b/installing_deps.sh index 6bb73f6b..6948c5d6 100755 --- a/installing_deps.sh +++ b/installing_deps.sh @@ -33,6 +33,9 @@ sudo pip install nose -y sudo apt-get install libfuzzy-dev -y sudo apt-get install build-essential libffi-dev automake autoconf libtool -y +# sflock, gz requirement +sudo apt-get install p7zip-full -y + # REDIS # test ! -d redis/ && git clone https://github.com/antirez/redis.git pushd redis/ diff --git a/pip3_packages_requirement.txt b/pip3_packages_requirement.txt index 78b19d64..352c8679 100644 --- a/pip3_packages_requirement.txt +++ b/pip3_packages_requirement.txt @@ -56,6 +56,9 @@ pycountry # To fetch Onion urls PySocks +# decompress files +sflock + #ASN lookup requirements #https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/adns-python/adns-python-1.2.1.tar.gz https://github.com/trolldbois/python3-adns/archive/master.zip diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py index 60244814..85353a7d 100755 --- a/var/www/Flask_server.py +++ b/var/www/Flask_server.py @@ -7,7 +7,7 @@ import json import datetime import time import calendar -from flask import Flask, render_template, jsonify, request +from flask import Flask, render_template, jsonify, request, Request import flask import importlib import os @@ -28,7 +28,7 @@ cfg = Flask_config.cfg Flask_config.app = Flask(__name__, static_url_path='/static/') app = Flask_config.app -#app.secret_key = Flask_config.secret_key +app.config['MAX_CONTENT_LENGTH'] = 900 * 1024 * 1024 # ========= HEADER GENERATION ======== diff --git a/var/www/modules/Flask_config.py b/var/www/modules/Flask_config.py index db74928f..80ec5967 100644 --- a/var/www/modules/Flask_config.py +++ b/var/www/modules/Flask_config.py @@ -84,6 +84,12 @@ r_serv_metadata = redis.StrictRedis( db=cfg.getint("ARDB_Metadata", "db"), decode_responses=True) +r_serv_db = redis.StrictRedis( + host=cfg.get("ARDB_DB", "host"), + port=cfg.getint("ARDB_DB", "port"), + db=cfg.getint("ARDB_DB", "db"), + decode_responses=True) + # VARIABLES # max_preview_char = int(cfg.get("Flask", "max_preview_char")) # Maximum number of character to display in the tooltip max_preview_modal = int(cfg.get("Flask", "max_preview_modal")) # Maximum number of character to display in the modal @@ -91,3 +97,5 @@ max_preview_modal = int(cfg.get("Flask", "max_preview_modal")) # Maximum number DiffMaxLineLength = int(cfg.get("Flask", "DiffMaxLineLength"))#Use to display the estimated percentage instead of a raw value bootstrap_label = ['primary', 'success', 'danger', 'warning', 'info'] + +UPLOAD_FOLDER = os.path.join(os.environ['AIL_FLASK'], 'submitted') diff --git a/var/www/modules/PasteSubmit/Flask_PasteSubmit.py b/var/www/modules/PasteSubmit/Flask_PasteSubmit.py index 1816d85c..0afa8e06 100644 --- a/var/www/modules/PasteSubmit/Flask_PasteSubmit.py +++ b/var/www/modules/PasteSubmit/Flask_PasteSubmit.py @@ -5,9 +5,7 @@ Flask functions and routes for the trending modules page ''' import redis -from flask import Flask, render_template, jsonify, request, Blueprint - -'''import random''' +from flask import Flask, render_template, jsonify, request, Blueprint, url_for import unicodedata import string @@ -15,6 +13,7 @@ import subprocess import os import sys import datetime +import uuid from pytaxonomies import Taxonomies from pymispgalaxies import Galaxies, Clusters @@ -25,21 +24,26 @@ import Flask_config app = Flask_config.app cfg = Flask_config.cfg r_serv_tags = Flask_config.r_serv_tags +r_serv_metadata = Flask_config.r_serv_metadata +r_serv_db = Flask_config.r_serv_db r_serv_log_submit = Flask_config.r_serv_log_submit PasteSubmit = Blueprint('PasteSubmit', __name__, template_folder='templates') valid_filename_chars = "-_ %s%s" % (string.ascii_letters, string.digits) -ALLOWED_EXTENSIONS = set(['txt', 'zip', 'gzip']) +ALLOWED_EXTENSIONS = set(['txt', 'zip', 'gz', 'tar.gz']) +UPLOAD_FOLDER = Flask_config.UPLOAD_FOLDER # ============ FUNCTIONS ============ def one(): return 1 def allowed_file(filename): - return '.' in filename and \ - filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS + if not '.' in filename: + return True + else: + return filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS def clean_filename(filename, whitelist=valid_filename_chars, replace=' '): # replace characters @@ -52,23 +56,28 @@ def clean_filename(filename, whitelist=valid_filename_chars, replace=' '): # keep only whitelisted chars return ''.join(c for c in cleaned_filename if c in whitelist) -'''@app.before_request -def csrf_protect(): - if request.method == "POST": - token = session.pop('_csrf_token', None) - if not token or token != request.form.get('_csrf_token'): - abort(400) +def launch_submit(ltags, ltagsgalaxies, paste_content, UUID, password, isfile = False): -def generate_csrf_token(): - if '_csrf_token' not in session: - session['_csrf_token'] = some_random_string() - return session['_csrf_token'] + print(UUID) -app.jinja_env.globals['csrf_token'] = generate_csrf_token + # save temp value on disk + r_serv_db.set(UUID + ':ltags', ltags) + r_serv_db.set(UUID + ':ltagsgalaxies', ltagsgalaxies) + r_serv_db.set(UUID + ':paste_content', paste_content) + r_serv_db.set(UUID + ':password', password) + r_serv_db.set(UUID + ':isfile', isfile) -def some_random_string(): - N = 15 - return ''.join(random.SystemRandom().choice(string.ascii_uppercase + string.digits) for _ in range(N))''' + r_serv_log_submit.set(UUID + ':end', 0) + r_serv_log_submit.set(UUID + ':processing', 0) + r_serv_log_submit.set(UUID + ':nb_total', -1) + r_serv_log_submit.set(UUID + ':nb_end', 0) + r_serv_log_submit.set(UUID + ':nb_sucess', 0) + r_serv_log_submit.set(UUID + ':error', 'error:') + r_serv_log_submit.sadd(UUID + ':paste_submit_link', '') + + + # save UUID on disk + r_serv_db.sadd('submitted:uuid', UUID) def addTagsVerification(tags, tagsgalaxies): @@ -124,7 +133,9 @@ def PasteSubmit_page(): @PasteSubmit.route("/PasteSubmit/submit", methods=['POST']) def submit(): - paste_name = request.form['paste_name'] + #paste_name = request.form['paste_name'] + + password = request.form['password'] ltags = request.form['tags_taxonomies'] ltagsgalaxies = request.form['tags_galaxies'] paste_content = request.form['paste_content'] @@ -133,68 +144,107 @@ def submit(): if not addTagsVerification(ltags, ltagsgalaxies): return 'INVALID TAGS' - if 'file' not in request.files: + # add submitted tags + if(ltags != ''): + ltags = ltags + ',submitted' + else: + ltags ='submitted' + + print(request.files) + + if 'file' in request.files: + print(request.files) file = request.files['file'] + if file: - if file.filename == '': + if file and allowed_file(file.filename): - if paste_content != '': - if sys.getsizeof(paste_content) < 900000: + # get UUID + UUID = str(uuid.uuid4()) - to_launch = os.environ['AIL_BIN'] + 'submit_paste.py' - # get id - id = str(r_serv_tags.get('submit_id')) - - if paste_name: - # clean file name - id = clean_filename(paste_name) - - # create logs - r_serv_log_submit.set(id + ':end', 0) - r_serv_log_submit.set(id + ':nb_total', 1) - r_serv_log_submit.set(id + ':nb_end', 0) - r_serv_log_submit.set(id + ':error', 'error:') - - #incr id - r_serv_tags.incr('submit_id') - - # add submitted tags - if(ltags != ''): - ltags = ltags + ',submitted' - else: - ltags ='submitted' - - # launch process - process = subprocess.Popen(["python", to_launch, ltags, ltagsgalaxies, paste_content, paste_name, id], - stdout=subprocess.PIPE) - - return render_template("submiting.html", - id = id) + '''if paste_name: + # clean file name + UUID = clean_filename(paste_name)''' + if not '.' in file.filename: + full_path = os.path.join(UPLOAD_FOLDER, UUID) else: - return 'size error' + if file.filename[-6:] == 'tar.gz': + file_type = 'tar.gz' + else: + file_type = file.filename.rsplit('.', 1)[1] + name = UUID + '.' + file_type + full_path = os.path.join(UPLOAD_FOLDER, name) - return 'submit' + #Flask verify the file size + file.save(full_path) - if file and allowed_file(file.filename): - print(file.read()) + paste_content = full_path - return 'error' + launch_submit(ltags, ltagsgalaxies, paste_content, UUID, password ,True) + + return render_template("submiting.html", + UUID = UUID) + + else: + print('wrong file type') + + + if paste_content != '': + if sys.getsizeof(paste_content) < 900000: + + # get id + UUID = str(uuid.uuid4()) + print(UUID) + + #if paste_name: + # clean file name + #id = clean_filename(paste_name) + + launch_submit(ltags, ltagsgalaxies, paste_content, UUID, password) + + return render_template("submiting.html", + UUID = UUID) + + else: + return 'size error' + + return 'submit' + + + return PasteSubmit_page() @PasteSubmit.route("/PasteSubmit/submit_status", methods=['GET']) def submit_status(): - id = request.args.get('id') + UUID = request.args.get('UUID') - if id: - end = r_serv_log_submit.get(id + ':end') - nb_total = r_serv_log_submit.get(id + ':nb_total') - nb_end = r_serv_log_submit.get(id + ':nb_end') - error = r_serv_log_submit.get(id + ':error') - if (end != None) and (nb_total != None) and (nb_end != None) and (error != None): + if UUID: + end = r_serv_log_submit.get(UUID + ':end') + nb_total = r_serv_log_submit.get(UUID + ':nb_total') + nb_end = r_serv_log_submit.get(UUID + ':nb_end') + error = r_serv_log_submit.get(UUID + ':error') + processing = r_serv_log_submit.get(UUID + ':processing') + nb_sucess = r_serv_log_submit.get(UUID + ':nb_sucess') + paste_submit_link = list(r_serv_log_submit.smembers(UUID + ':paste_submit_link')) - in_progress = nb_end + ' / ' + nb_total - prog = int(int(nb_end) * 100 / int(nb_total)) + if (end != None) and (nb_total != None) and (nb_end != None) and (error != None) and (processing != None) and (paste_submit_link != None): + + link = '' + if paste_submit_link: + for paste in paste_submit_link: + url = url_for('showsavedpastes.showsavedpaste') + '?paste=' + paste + link += '' + paste +'' + + if nb_total == '-1': + in_progress = nb_sucess + ' / ' + else: + in_progress = nb_sucess + ' / ' + nb_total + + if int(nb_total) != 0: + prog = int(int(nb_end) * 100 / int(nb_total)) + else: + prog = 0 if error == 'error:': isError = False @@ -206,15 +256,29 @@ def submit_status(): else: end = True + if processing == '0': + processing = False + else: + processing = True + return jsonify(end=end, in_progress=in_progress, prog=prog, + link=link, + processing=processing, isError=isError, error=error) else: + # FIXME TODO + print(end) + print(nb_total) + print(nb_end) + print(error) + print(processing) + print(nb_sucess) return 'to do' else: - return 'INVALID ID' + return 'INVALID UUID' # ========= REGISTRATION ========= app.register_blueprint(PasteSubmit) diff --git a/var/www/modules/PasteSubmit/templates/PasteSubmit.html b/var/www/modules/PasteSubmit/templates/PasteSubmit.html index b2d01a6e..3a463694 100644 --- a/var/www/modules/PasteSubmit/templates/PasteSubmit.html +++ b/var/www/modules/PasteSubmit/templates/PasteSubmit.html @@ -36,22 +36,18 @@
-
paste info +
Files submission
-
- +
+ +
- - -
- -
- - + +
@@ -129,6 +125,9 @@ var ltagsgalaxies $(document).ready(function(){ + activePage = "page-PasteSubmit" + $("#"+activePage).addClass("active"); + $.getJSON('/Tags/get_all_tags_taxonomies', function(data) { @@ -148,9 +147,6 @@ name: 'ltagsgalaxies' }); }); - - activePage = "page-PasteSubmit" - $("#"+activePage).addClass("active"); }); diff --git a/var/www/modules/PasteSubmit/templates/submiting.html b/var/www/modules/PasteSubmit/templates/submiting.html index 31062b9b..9ab52465 100644 --- a/var/www/modules/PasteSubmit/templates/submiting.html +++ b/var/www/modules/PasteSubmit/templates/submiting.html @@ -22,99 +22,111 @@ + + - {% include 'navbar.html' %} + {% include 'navbar.html' %} -
-
- +
+
+ -
+ - - + + -
-
paste info -
-
+
+
Files submission +
+
-
- -
+
+ + +
-
- - -
+
+ + +
+
+
-
- - -
-
-
+
+
Tags : +
    +
  • -
    -
    Tags : -
      -
    • +
      + +
      -
      - -
      - -
      - - -
      - -
    • -
    • - -
      - -
      - -
      - - -
      - -
    • +
      + + -
    -
    +
  • +
  • -
    - -
    - -
    - -
    +
    +
    +
    + + +
    + +
  • +
+
+
+ +
+ +
+ +
+ +
+ +
+ @@ -136,16 +148,21 @@