mirror of https://github.com/CIRCL/AIL-framework
fix: [submit_paste] restrict source name
Terrtia
parent
75bc585242
commit
a2ebd09c2a
|
|
@ -79,6 +79,9 @@ class SubmitPaste(AbstractModule):
|
|||
password = self.r_serv_db.get(f'{uuid}:password')
|
||||
source = self.r_serv_db.get(f'{uuid}:source')
|
||||
|
||||
if source in ['crawled', 'tests']:
|
||||
source = 'submitted'
|
||||
|
||||
self.redis_logger.debug(f'isfile UUID {isfile}')
|
||||
self.redis_logger.debug(f'source UUID {source}')
|
||||
self.redis_logger.debug(f'paste_content UUID {paste_content}')
|
||||
|
|
@ -100,7 +103,7 @@ class SubmitPaste(AbstractModule):
|
|||
# textarea input paste
|
||||
self._manage_text(uuid, paste_content, ltags, ltagsgalaxies, source)
|
||||
|
||||
# new paste created from file, remove uuid ref
|
||||
# new paste created from file, remove uuid ref
|
||||
self.remove_submit_uuid(uuid)
|
||||
|
||||
|
||||
|
|
@ -108,12 +111,12 @@ class SubmitPaste(AbstractModule):
|
|||
"""
|
||||
Run Module endless process
|
||||
"""
|
||||
|
||||
|
||||
# Endless loop processing messages from the input queue
|
||||
while self.proceed:
|
||||
# Get one message (paste) from the QueueIn (copy of Redis_Global publish)
|
||||
nb_submit = self.r_serv_db.scard('submitted:uuid')
|
||||
|
||||
|
||||
if nb_submit > 0:
|
||||
try:
|
||||
uuid = self.r_serv_db.srandmember('submitted:uuid')
|
||||
|
|
@ -122,7 +125,7 @@ class SubmitPaste(AbstractModule):
|
|||
self.compute(uuid)
|
||||
except Exception as err:
|
||||
self.redis_logger.error(f'Error in module {self.module_name}: {err}')
|
||||
# Remove uuid ref
|
||||
# Remove uuid ref
|
||||
self.remove_submit_uuid(uuid)
|
||||
else:
|
||||
# Wait before next process
|
||||
|
|
@ -150,7 +153,7 @@ class SubmitPaste(AbstractModule):
|
|||
|
||||
if os.path.exists(file_full_path):
|
||||
self.redis_logger.debug(f'file exists {file_full_path}')
|
||||
|
||||
|
||||
file_size = os.stat(file_full_path).st_size
|
||||
self.redis_logger.debug(f'file size {file_size}')
|
||||
# Verify file length
|
||||
|
|
@ -168,7 +171,6 @@ class SubmitPaste(AbstractModule):
|
|||
content = f.read()
|
||||
self.r_serv_log_submit.set(uuid + ':nb_total', 1)
|
||||
self.create_paste(uuid, content.encode(), ltags, ltagsgalaxies, uuid, source)
|
||||
self.remove_submit_uuid(uuid)
|
||||
except:
|
||||
self.abord_file_submission(uuid, "file error")
|
||||
|
||||
|
|
@ -261,15 +263,17 @@ class SubmitPaste(AbstractModule):
|
|||
# delete uuid
|
||||
self.r_serv_db.srem('submitted:uuid', uuid)
|
||||
self.redis_logger.debug(f'{uuid} all file submitted')
|
||||
print(f'{uuid} all file submitted')
|
||||
|
||||
|
||||
def create_paste(self, uuid, paste_content, ltags, ltagsgalaxies, name, source=None):
|
||||
|
||||
# # TODO: Use Item create
|
||||
|
||||
result = False
|
||||
|
||||
now = datetime.datetime.now()
|
||||
source = source if source else 'submitted'
|
||||
save_path = source + '/' + now.strftime("%Y") + '/' + now.strftime("%m") + '/' + now.strftime("%d") + '/' + name + '.gz'
|
||||
save_path = source + '/' + now.strftime("%Y") + '/' + now.strftime("%m") + '/' + now.strftime("%d") + '/submitted_' + name + '.gz'
|
||||
|
||||
full_path = filename = os.path.join(os.environ['AIL_HOME'],
|
||||
self.process.config.get("Directories", "pastes"), save_path)
|
||||
|
|
@ -309,6 +313,7 @@ class SubmitPaste(AbstractModule):
|
|||
self.r_serv_log_submit.set(f'{uuid}:end', 1)
|
||||
|
||||
self.redis_logger.debug(f' {rel_item_path} send to Global')
|
||||
print(f' {rel_item_path} send to Global')
|
||||
self.r_serv_log_submit.sadd(f'{uuid}:paste_submit_link', rel_item_path)
|
||||
|
||||
curr_date = datetime.date.today()
|
||||
|
|
@ -328,13 +333,13 @@ class SubmitPaste(AbstractModule):
|
|||
gzip64encoded = base64.standard_b64encode(gzipencoded).decode()
|
||||
except:
|
||||
self.abord_file_submission(uuid, "file error")
|
||||
|
||||
|
||||
return gzip64encoded
|
||||
|
||||
|
||||
def addError(self, uuid, errorMessage):
|
||||
self.redis_logger.debug(errorMessage)
|
||||
|
||||
print(errorMessage)
|
||||
error = self.r_serv_log_submit.get(f'{uuid}:error')
|
||||
if error != None:
|
||||
self.r_serv_log_submit.set(f'{uuid}:error', error + '<br></br>' + errorMessage)
|
||||
|
|
@ -351,7 +356,7 @@ class SubmitPaste(AbstractModule):
|
|||
self.serv_statistics.hincrby(curr_date.strftime("%Y%m%d"),'submit_abord', 1)
|
||||
self.remove_submit_uuid(uuid)
|
||||
|
||||
|
||||
# # TODO: use Item function
|
||||
def get_item_date(self, item_filename):
|
||||
l_directory = item_filename.split('/')
|
||||
return f'{l_directory[-4]}{l_directory[-3]}{l_directory[-2]}'
|
||||
|
|
@ -371,6 +376,6 @@ class SubmitPaste(AbstractModule):
|
|||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
|
||||
module = SubmitPaste()
|
||||
module.run()
|
||||
|
|
|
|||
|
|
@ -45,7 +45,7 @@ redis_logger = publisher
|
|||
# Port of the redis instance used by pubsublogger
|
||||
redis_logger.port = 6380
|
||||
# Channel name to publish logs
|
||||
redis_logger.channel = 'front'
|
||||
redis_logger.channel = 'Flask'
|
||||
|
||||
|
||||
sys.path.append('../../configs/keys')
|
||||
|
|
|
|||
|
|
@ -11,7 +11,6 @@ import os
|
|||
import sys
|
||||
import json
|
||||
import string
|
||||
import subprocess
|
||||
import datetime
|
||||
import redis
|
||||
import unicodedata
|
||||
|
|
@ -19,8 +18,10 @@ import uuid
|
|||
from io import BytesIO
|
||||
from Date import Date
|
||||
|
||||
from flask import Flask, render_template, jsonify, request, Blueprint, url_for, redirect, abort
|
||||
from functools import wraps
|
||||
|
||||
# Flask
|
||||
from flask import Flask, render_template, jsonify, request, Blueprint, url_for, redirect, abort
|
||||
from Role_Manager import login_admin, login_analyst
|
||||
from flask_login import login_required
|
||||
|
||||
|
|
@ -257,7 +258,7 @@ def PasteSubmit_page():
|
|||
|
||||
return render_template("submit_items.html",
|
||||
active_taxonomies = active_taxonomies,
|
||||
active_galaxies = active_galaxies,
|
||||
active_galaxies = active_galaxies,
|
||||
text_max_size = text_max_size,
|
||||
file_max_size = file_max_size,
|
||||
allowed_extensions = allowed_extensions)
|
||||
|
|
@ -277,6 +278,13 @@ def submit():
|
|||
paste_content = request.form['paste_content']
|
||||
paste_source = request.form['paste_source']
|
||||
|
||||
# limit source length
|
||||
paste_source = paste_source.replace('/', '')[:80]
|
||||
if paste_source in ['crawled', 'tests']:
|
||||
content = f'Invalid source'
|
||||
logger.info(paste_source)
|
||||
return content, 400
|
||||
|
||||
is_file = False
|
||||
if 'file' in request.files:
|
||||
file_import = request.files['file']
|
||||
|
|
@ -343,7 +351,7 @@ def submit():
|
|||
#Flask verify the file size
|
||||
file_import.save(full_path)
|
||||
logger.debug('file saved')
|
||||
|
||||
|
||||
Import_helper.create_import_queue(ltags, ltagsgalaxies, full_path, UUID, password, True)
|
||||
|
||||
return render_template("submit_items.html",
|
||||
|
|
|
|||
|
|
@ -100,7 +100,7 @@
|
|||
<div class="col-12 col-lg-10" id="core_content">
|
||||
|
||||
{% if message %}
|
||||
<p>{{ message }}</p>
|
||||
<p>{{ message }}</p>
|
||||
{% endif %}
|
||||
|
||||
<form action="{{ url_for('PasteSubmit.submit') }}" id="pasteSubmitForm" method="post"
|
||||
|
|
@ -112,7 +112,7 @@
|
|||
|
||||
<div class="card mt-2 mb-4">
|
||||
<div class="card-header py-3 d-flex flex-row align-items-center justify-content-between">
|
||||
<h6 class="m-0 font-weight-bold text-primary">Submit Paste</h6>
|
||||
<h6 class="m-0 font-weight-bold text-primary">Submit Item</h6>
|
||||
</div>
|
||||
<div class="card-body">
|
||||
|
||||
|
|
@ -150,8 +150,8 @@
|
|||
Tags <i class="fas fa-tags"></i></button>
|
||||
<div class="dropdown-divider"></div>
|
||||
{% for taxo in active_taxonomies %}
|
||||
<button class="dropdown-item" type="button"
|
||||
id="{{ taxo }}-id{{ loop.index0 }}">{{ taxo }}</button>
|
||||
<button class="dropdown-item" type="button"
|
||||
id="{{ taxo }}-id{{ loop.index0 }}">{{ taxo }}</button>
|
||||
{% endfor %}
|
||||
</div>
|
||||
</div>
|
||||
|
|
@ -174,8 +174,8 @@
|
|||
<i class="fas fa-tags"></i></button>
|
||||
<div class="dropdown-divider"></div>
|
||||
{% for galaxy in active_galaxies %}
|
||||
<button class="dropdown-item" type="button"
|
||||
id="{{ galaxy }}-idgalax{{ loop.index0 }}">{{ galaxy }}</button>
|
||||
<button class="dropdown-item" type="button"
|
||||
id="{{ galaxy }}-idgalax{{ loop.index0 }}">{{ galaxy }}</button>
|
||||
{% endfor %}
|
||||
</div>
|
||||
</div>
|
||||
|
|
@ -320,22 +320,22 @@
|
|||
});
|
||||
|
||||
{% for taxo in active_taxonomies %}
|
||||
jQuery("#{{ taxo }}-id{{ loop.index0 }}").on("click", function (e) {
|
||||
$.getJSON("{{ url_for('Tags.get_tags_taxonomie') }}?taxonomie={{ taxo }}",
|
||||
function (data) {
|
||||
ltags.setData(data)
|
||||
});
|
||||
});
|
||||
jQuery("#{{ taxo }}-id{{ loop.index0 }}").on("click", function (e) {
|
||||
$.getJSON("{{ url_for('Tags.get_tags_taxonomie') }}?taxonomie={{ taxo }}",
|
||||
function (data) {
|
||||
ltags.setData(data)
|
||||
});
|
||||
});
|
||||
{% endfor %}
|
||||
</script>
|
||||
<script>
|
||||
{% for galaxy in active_galaxies %}
|
||||
jQuery("#{{ galaxy }}-idgalax{{ loop.index0 }}").on("click", function (e) {
|
||||
$.getJSON("{{ url_for('Tags.get_tags_galaxy') }}?galaxy={{ galaxy }}",
|
||||
function (data) {
|
||||
ltagsgalaxies.setData(data)
|
||||
});
|
||||
});
|
||||
jQuery("#{{ galaxy }}-idgalax{{ loop.index0 }}").on("click", function (e) {
|
||||
$.getJSON("{{ url_for('Tags.get_tags_galaxy') }}?galaxy={{ galaxy }}",
|
||||
function (data) {
|
||||
ltagsgalaxies.setData(data)
|
||||
});
|
||||
});
|
||||
{% endfor %}
|
||||
</script>
|
||||
|
||||
|
|
@ -417,4 +417,4 @@
|
|||
</script>
|
||||
|
||||
|
||||
</html>
|
||||
</html>
|
||||
|
|
|
|||
Loading…
Reference in New Issue