new: Add monitoring for pickle cache status

pull/267/head
Raphaël Vinot 2021-09-24 12:02:28 +02:00
parent 1d3bb55343
commit 48fc807e7d
9 changed files with 21 additions and 3 deletions

View File

@ -20,7 +20,6 @@ from lookyloo.abstractmanager import AbstractManager
from lookyloo.helpers import (get_captures_dir, get_config, get_socket_path,
get_splash_url, load_cookies, safe_create_dir,
splash_status)
from lookyloo.lookyloo import Lookyloo
logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s:%(message)s',
level=logging.INFO)
@ -30,7 +29,6 @@ class AsyncCapture(AbstractManager):
def __init__(self, loglevel: int=logging.INFO):
super().__init__(loglevel)
self.lookyloo = Lookyloo()
self.script_name = 'async_capture'
self.only_global_lookups: bool = get_config('generic', 'only_global_lookups')
self.capture_dir: Path = get_captures_dir()

View File

@ -2,6 +2,7 @@
# -*- coding: utf-8 -*-
import logging
import os
from datetime import datetime, timedelta
from lookyloo.abstractmanager import AbstractManager
@ -25,6 +26,7 @@ class BackgroundIndexer(AbstractManager):
def _to_run_forever(self):
self._build_missing_pickles()
self._check_indexes()
self.lookyloo.update_tree_cache_info(os.getpid(), self.script_name)
def _build_missing_pickles(self):
for uuid_path in sorted(self.lookyloo.capture_dir.glob('**/uuid'), reverse=True):

View File

@ -14,6 +14,7 @@ def main():
p.wait()
r = Redis(unix_socket_path=get_socket_path('cache'), db=1)
r.delete('shutdown')
r.delete('tree_cache')
Popen(['run_backend', '--stop'])

View File

@ -137,6 +137,9 @@ class CapturesIndex(Mapping):
self.redis.flushdb()
self.__cache = {}
def lru_cache_status(self):
return load_pickle_tree.cache_info()
def _quick_init(self) -> None:
'''Initialize the cache with a list of UUIDs, with less back and forth with redis.
Only get recent captures.'''

View File

@ -287,6 +287,9 @@ class Lookyloo():
(self._get_capture_dir(capture_uuid) / 'no_index').touch()
self._captures_index.reload_cache(capture_uuid)
def update_tree_cache_info(self, process_id: int, classname: str) -> None:
self.redis.hset('tree_cache', f'{process_id}|{classname}', str(self._captures_index.lru_cache_status()))
@property
def capture_uuids(self) -> List[str]:
'''All the capture UUIDs present in the cache.'''

View File

@ -25,7 +25,6 @@ classifiers = [
start = "bin.start:main"
stop = "bin.stop:main"
update = "bin.update:main"
rebuild_caches = "bin.rebuild_caches:main"
shutdown = "bin.shutdown:main"
run_backend = "bin.run_backend:main"
async_capture = "bin.async_capture:main"

View File

@ -74,6 +74,10 @@ class Monitoring():
return to_return
@property
def tree_cache(self):
return self.redis_cache.hgetall('tree_cache')
if __name__ == '__main__':
@ -95,6 +99,11 @@ if __name__ == '__main__':
s = Padding(f'{service} ({int(number)} service(s))', (0, 2))
console.print(s)
console.print('Current cache status:')
for name, status in m.tree_cache.items():
s = Padding(f'{name}: {status}', (0, 2))
console.print(s)
console.print('Current queues:')
for q, priority in m.queues:
s = Padding(f'{q} Recently enqueued captures: {int(priority)}', (0, 2))

View File

@ -5,6 +5,7 @@ import calendar
import http
import json
import logging
import os
import time
from datetime import date, datetime, timedelta, timezone
from io import BytesIO, StringIO
@ -646,6 +647,8 @@ def tree(tree_uuid: str, node_uuid: Optional[str]=None):
except NoValidHarFile as e:
return render_template('error.html', error_message=e)
finally:
lookyloo.update_tree_cache_info(os.getpid(), 'website')
@app.route('/tree/<string:tree_uuid>/mark_as_legitimate', methods=['POST'])