chg: [server UI v0.2] add analyzer management

server/web/Flask_server.py

@@ -24,6 +24,7 @@ host_redis_stream = "localhost"
 port_redis_stream = 6379
 
 default_max_entries_by_stream = 10000
+analyzer_list_max_default_size = 10000
 
 json_type_description_path = os.path.join(os.environ['D4_HOME'], 'web/static/json/type.json')
 
@@ -197,8 +198,24 @@ def server_management():
     json_type_description = get_json_type_description()
 
     list_accepted_types = []
+    list_analyzer_types = []
     for type in redis_server_metadata.smembers('server:accepted_type'):
-        list_accepted_types.append({"id": int(type), "description": json_type_description[int(type)]['description']})
+        try:
+            description = json_type_description[int(type)]['description']
+        except:
+            description = 'Please update your web server'
+
+        list_analyzer_uuid = []
+        for analyzer_uuid in redis_server_metadata.smembers('analyzer:{}'.format(type)):
+            size_limit = redis_server_metadata.hget('analyzer:{}'.format(analyzer_uuid), 'max_size')
+            if size_limit is None:
+                size_limit = analyzer_list_max_default_size
+            last_updated = redis_server_metadata.hget('analyzer:{}'.format(analyzer_uuid), 'last_updated')
+            if last_updated is None:
+                last_updated = 'Never'
+            list_analyzer_uuid.append({'uuid': analyzer_uuid, 'size_limit': size_limit,'last_updated': last_updated})
+
+        list_accepted_types.append({"id": int(type), "description": description, 'list_analyzer_uuid': list_analyzer_uuid})
 
     return render_template("server_management.html", list_accepted_types=list_accepted_types,
                             blacklisted_ip=blacklisted_ip, unblacklisted_ip=unblacklisted_ip,
@@ -323,6 +340,61 @@ def uuid_change_stream_max_size():
     else:
         return 'Invalid uuid'
 
+@app.route('/add_new_analyzer')
+def add_new_analyzer():
+    type = request.args.get('type')
+    user = request.args.get('redirect')
+    analyzer_uuid = request.args.get('analyzer_uuid')
+    if is_valid_uuid_v4(analyzer_uuid):
+        try:
+            type = int(type)
+            if type < 0:
+                return 'type, Invalid Integer'
+        except:
+            return 'type, Invalid Integer'
+        redis_server_metadata.sadd('analyzer:{}'.format(type), analyzer_uuid)
+        if user:
+            return redirect(url_for('server_management'))
+    else:
+        return 'Invalid uuid'
+
+@app.route('/remove_analyzer')
+def remove_analyzer():
+    analyzer_uuid = request.args.get('analyzer_uuid')
+    type = request.args.get('type')
+    user = request.args.get('redirect')
+    if is_valid_uuid_v4(analyzer_uuid):
+        try:
+            type = int(type)
+            if type < 0:
+                return 'type, Invalid Integer'
+        except:
+            return 'type, Invalid Integer'
+        redis_server_metadata.srem('analyzer:{}'.format(type), analyzer_uuid)
+        print(user)
+        if user:
+            return redirect(url_for('server_management'))
+    else:
+        return 'Invalid uuid'
+
+@app.route('/analyzer_change_max_size')
+def analyzer_change_max_size():
+    analyzer_uuid = request.args.get('analyzer_uuid')
+    user = request.args.get('redirect')
+    max_size_analyzer = request.args.get('max_size_analyzer')
+    if is_valid_uuid_v4(analyzer_uuid):
+        try:
+            max_size_analyzer = int(max_size_analyzer)
+            if max_size_analyzer < 0:
+                return 'analyzer max size, Invalid Integer'
+        except:
+            return 'analyzer max size, Invalid Integer'
+        redis_server_metadata.hset('analyzer:{}'.format(analyzer_uuid), 'max_size', max_size_analyzer)
+        if user:
+            return redirect(url_for('server_management'))
+    else:
+        return 'Invalid uuid'
+
 @app.route('/blacklist_uuid')
 def blacklist_uuid():
     uuid_sensor = request.args.get('uuid')

server/web/templates/server_management.html

@@ -14,11 +14,6 @@
 	<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
 	<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
 
-  <style>
-
-  </style>
-
-
 </head>
 
 <body>
@@ -182,7 +177,7 @@
 						<div class="card border-dark mt-3" style="max-width: 18rem;">
 						  <div class="card-body text-dark">
 						    <h5 class="card-title">Add New Types</h5>
-									<input class="form-control" type="number" id="accepted_type" value="1" min="1" max="7" required>
+									<input class="form-control" type="number" id="accepted_type" value="1" min="1" max="8" required>
 									<button type="button" class="btn btn-outline-primary mt-1" onclick="window.location.href ='{{ url_for('add_accepted_type') }}?redirect=1&type='+$('#accepted_type').val();">Add New Type</button>
 							</div>
 						</div>
@@ -194,6 +189,69 @@
 
 	</div>
 
+
+	<div class="card-deck justify-content-center ml-0 mr-0">
+		<div class="card border-dark mt-3 ml-4 mr-4">
+			<div class="card-header bg-dark text-white">
+				Analyzer Management
+			</div>
+		  <div class="card-body text-dark">
+
+				<div class="row">
+			    <div class="col-sm-8">
+						<table class="table table-striped table-bordered table-hover" id="myTable_">
+							<thead class="thead-dark">
+								<tr>
+									<th>Type</th>
+									<th style="max-width: 800px;">uuid</th>
+									<th style="max-width: 800px;">last updated</th>
+									<th style="max-width: 800px;">Change max size limit</th>
+									<th style="max-width: 800px;">Delete Analyzer Queue</th>
+								</tr>
+							</thead>
+							<tbody>
+								{% for type in list_accepted_types %}
+									{% if type['list_analyzer_uuid'] %}
+										{% for analyzer in type['list_analyzer_uuid'] %}
+											<tr>
+												<td>{{type['id']}}</td>
+												<td>{{analyzer['uuid']}}</td>
+												<td>{{analyzer['last_updated']}}</td>
+												<td>
+													<div class="d-flex justify-content-center">
+														<input class="form-control mt-1 mr-1" style="max-width: 100px;" type="number" id="max_size_analyzer_{{analyzer['uuid']}}" value="{{analyzer['size_limit']}}" min="0" required="">
+														<button type="button" class="btn btn-outline-secondary mt-1" onclick="window.location.href ='{{ url_for('analyzer_change_max_size') }}?analyzer_uuid={{analyzer['uuid']}}&redirect=0&max_size_analyzer='+$('#max_size_analyzer_{{analyzer['uuid']}}').val();">Change Max Size</button>
+													</div>
+												</td>
+												<td>
+													<a href="{{ url_for('remove_analyzer') }}?redirect=1&type={{type['id']}}&analyzer_uuid={{analyzer['uuid']}}">
+														<button type="button mt-2" class="btn btn-outline-danger">Remove</button>
+													</a>
+												</td>
+											</tr>
+										{% endfor %}
+									{% endif %}
+								{% endfor %}
+							</tbody>
+						</table>
+					</div>
+			    <div class="col-sm-4">
+						<div class="card border-dark mt-3" style="max-width: 18rem;">
+						  <div class="card-body text-dark">
+						    <h5 class="card-title">Add New Analyzer Queue</h5>
+									<input class="form-control" type="number" id="analyzer_type" value="1" min="1" max="8" required>
+									<input class="form-control" type="text" id="analyzer_uuid" required>
+									<button type="button" class="btn btn-outline-primary mt-1" onclick="window.location.href ='{{ url_for('add_new_analyzer') }}?redirect=1&type='+$('#analyzer_type').val()+'&analyzer_uuid='+$('#analyzer_uuid').val();">Add New Analyzer</button>
+							</div>
+						</div>
+					</div>
+			  </div>
+
+			</div>
+		</div>
+
+	</div>
+
 	{% include 'navfooter.html' %}
 </body>
 

server/workers/workers_8/worker.py

@@ -33,7 +33,7 @@ redis_server_analyzer = redis.StrictRedis(
                     db=2)
 
 type = 8
-rotation_save_cycle = 10 #seconds
+rotation_save_cycle = 300 #seconds
 
 save_to_file = True
 
@@ -96,10 +96,10 @@ if __name__ == "__main__":
                     if b'\n' in data[b'message']:
                         all_line = data[b'message'].split(b'\n')
                         for line in all_line[:-1]:
-                            for analyzer_uuid in redis_server_metadata.smembers('analyser:{}'.format(type)):
+                            for analyzer_uuid in redis_server_metadata.smembers('analyzer:{}'.format(type)):
                                 analyzer_uuid = analyzer_uuid.decode()
                                 redis_server_analyzer.lpush('analyzer:{}:{}'.format(type, analyzer_uuid), line)
-                                redis_server_metadata.hset('analyser:{}'.format(analyzer_uuid), 'last_updated', time.time())
+                                redis_server_metadata.hset('analyzer:{}'.format(analyzer_uuid), 'last_updated', time.time())
                         # keep incomplete line
                         if all_line[-1] != b'':
                             buffer += data[b'message']