new: Add monitoring for pickle cache status

bin/async_capture.py

@@ -20,7 +20,6 @@ from lookyloo.abstractmanager import AbstractManager
 from lookyloo.helpers import (get_captures_dir, get_config, get_socket_path,
                               get_splash_url, load_cookies, safe_create_dir,
                               splash_status)
-from lookyloo.lookyloo import Lookyloo
 
 logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s:%(message)s',
                     level=logging.INFO)
@@ -30,7 +29,6 @@ class AsyncCapture(AbstractManager):
 
     def __init__(self, loglevel: int=logging.INFO):
         super().__init__(loglevel)
-        self.lookyloo = Lookyloo()
         self.script_name = 'async_capture'
         self.only_global_lookups: bool = get_config('generic', 'only_global_lookups')
         self.capture_dir: Path = get_captures_dir()

bin/background_indexer.py

@@ -2,6 +2,7 @@
 # -*- coding: utf-8 -*-
 
 import logging
+import os
 from datetime import datetime, timedelta
 
 from lookyloo.abstractmanager import AbstractManager
@@ -25,6 +26,7 @@ class BackgroundIndexer(AbstractManager):
     def _to_run_forever(self):
         self._build_missing_pickles()
         self._check_indexes()
+        self.lookyloo.update_tree_cache_info(os.getpid(), self.script_name)
 
     def _build_missing_pickles(self):
         for uuid_path in sorted(self.lookyloo.capture_dir.glob('**/uuid'), reverse=True):

bin/stop.py

@@ -14,6 +14,7 @@ def main():
     p.wait()
     r = Redis(unix_socket_path=get_socket_path('cache'), db=1)
     r.delete('shutdown')
+    r.delete('tree_cache')
     Popen(['run_backend', '--stop'])
 
 

lookyloo/capturecache.py

@@ -137,6 +137,9 @@ class CapturesIndex(Mapping):
         self.redis.flushdb()
         self.__cache = {}
 
+    def lru_cache_status(self):
+        return load_pickle_tree.cache_info()
+
     def _quick_init(self) -> None:
         '''Initialize the cache with a list of UUIDs, with less back and forth with redis.
         Only get recent captures.'''

lookyloo/lookyloo.py

@@ -287,6 +287,9 @@ class Lookyloo():
         (self._get_capture_dir(capture_uuid) / 'no_index').touch()
         self._captures_index.reload_cache(capture_uuid)
 
+    def update_tree_cache_info(self, process_id: int, classname: str) -> None:
+        self.redis.hset('tree_cache', f'{process_id}|{classname}', str(self._captures_index.lru_cache_status()))
+
     @property
     def capture_uuids(self) -> List[str]:
         '''All the capture UUIDs present in the cache.'''

pyproject.toml

@@ -25,7 +25,6 @@ classifiers = [
 start = "bin.start:main"
 stop = "bin.stop:main"
 update = "bin.update:main"
-rebuild_caches = "bin.rebuild_caches:main"
 shutdown = "bin.shutdown:main"
 run_backend = "bin.run_backend:main"
 async_capture = "bin.async_capture:main"

tools/monitoring.py

@@ -74,6 +74,10 @@ class Monitoring():
 
         return to_return
 
+    @property
+    def tree_cache(self):
+        return self.redis_cache.hgetall('tree_cache')
+
 
 if __name__ == '__main__':
 
@@ -95,6 +99,11 @@ if __name__ == '__main__':
         s = Padding(f'{service} ({int(number)} service(s))', (0, 2))
         console.print(s)
 
+    console.print('Current cache status:')
+    for name, status in m.tree_cache.items():
+        s = Padding(f'{name}: {status}', (0, 2))
+        console.print(s)
+
     console.print('Current queues:')
     for q, priority in m.queues:
         s = Padding(f'{q} Recently enqueued captures: {int(priority)}', (0, 2))

website/web/__init__.py

@@ -5,6 +5,7 @@ import calendar
 import http
 import json
 import logging
+import os
 import time
 from datetime import date, datetime, timedelta, timezone
 from io import BytesIO, StringIO
@@ -646,6 +647,8 @@ def tree(tree_uuid: str, node_uuid: Optional[str]=None):
 
     except NoValidHarFile as e:
         return render_template('error.html', error_message=e)
+    finally:
+        lookyloo.update_tree_cache_info(os.getpid(), 'website')
 
 
 @app.route('/tree/<string:tree_uuid>/mark_as_legitimate', methods=['POST'])