mirror of https://github.com/CIRCL/lookyloo
new: Add monitoring for pickle cache status
parent
1d3bb55343
commit
48fc807e7d
|
@ -20,7 +20,6 @@ from lookyloo.abstractmanager import AbstractManager
|
|||
from lookyloo.helpers import (get_captures_dir, get_config, get_socket_path,
|
||||
get_splash_url, load_cookies, safe_create_dir,
|
||||
splash_status)
|
||||
from lookyloo.lookyloo import Lookyloo
|
||||
|
||||
logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s:%(message)s',
|
||||
level=logging.INFO)
|
||||
|
@ -30,7 +29,6 @@ class AsyncCapture(AbstractManager):
|
|||
|
||||
def __init__(self, loglevel: int=logging.INFO):
|
||||
super().__init__(loglevel)
|
||||
self.lookyloo = Lookyloo()
|
||||
self.script_name = 'async_capture'
|
||||
self.only_global_lookups: bool = get_config('generic', 'only_global_lookups')
|
||||
self.capture_dir: Path = get_captures_dir()
|
||||
|
|
|
@ -2,6 +2,7 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
import logging
|
||||
import os
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from lookyloo.abstractmanager import AbstractManager
|
||||
|
@ -25,6 +26,7 @@ class BackgroundIndexer(AbstractManager):
|
|||
def _to_run_forever(self):
|
||||
self._build_missing_pickles()
|
||||
self._check_indexes()
|
||||
self.lookyloo.update_tree_cache_info(os.getpid(), self.script_name)
|
||||
|
||||
def _build_missing_pickles(self):
|
||||
for uuid_path in sorted(self.lookyloo.capture_dir.glob('**/uuid'), reverse=True):
|
||||
|
|
|
@ -14,6 +14,7 @@ def main():
|
|||
p.wait()
|
||||
r = Redis(unix_socket_path=get_socket_path('cache'), db=1)
|
||||
r.delete('shutdown')
|
||||
r.delete('tree_cache')
|
||||
Popen(['run_backend', '--stop'])
|
||||
|
||||
|
||||
|
|
|
@ -137,6 +137,9 @@ class CapturesIndex(Mapping):
|
|||
self.redis.flushdb()
|
||||
self.__cache = {}
|
||||
|
||||
def lru_cache_status(self):
|
||||
return load_pickle_tree.cache_info()
|
||||
|
||||
def _quick_init(self) -> None:
|
||||
'''Initialize the cache with a list of UUIDs, with less back and forth with redis.
|
||||
Only get recent captures.'''
|
||||
|
|
|
@ -287,6 +287,9 @@ class Lookyloo():
|
|||
(self._get_capture_dir(capture_uuid) / 'no_index').touch()
|
||||
self._captures_index.reload_cache(capture_uuid)
|
||||
|
||||
def update_tree_cache_info(self, process_id: int, classname: str) -> None:
|
||||
self.redis.hset('tree_cache', f'{process_id}|{classname}', str(self._captures_index.lru_cache_status()))
|
||||
|
||||
@property
|
||||
def capture_uuids(self) -> List[str]:
|
||||
'''All the capture UUIDs present in the cache.'''
|
||||
|
|
|
@ -25,7 +25,6 @@ classifiers = [
|
|||
start = "bin.start:main"
|
||||
stop = "bin.stop:main"
|
||||
update = "bin.update:main"
|
||||
rebuild_caches = "bin.rebuild_caches:main"
|
||||
shutdown = "bin.shutdown:main"
|
||||
run_backend = "bin.run_backend:main"
|
||||
async_capture = "bin.async_capture:main"
|
||||
|
|
|
@ -74,6 +74,10 @@ class Monitoring():
|
|||
|
||||
return to_return
|
||||
|
||||
@property
|
||||
def tree_cache(self):
|
||||
return self.redis_cache.hgetall('tree_cache')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
|
@ -95,6 +99,11 @@ if __name__ == '__main__':
|
|||
s = Padding(f'{service} ({int(number)} service(s))', (0, 2))
|
||||
console.print(s)
|
||||
|
||||
console.print('Current cache status:')
|
||||
for name, status in m.tree_cache.items():
|
||||
s = Padding(f'{name}: {status}', (0, 2))
|
||||
console.print(s)
|
||||
|
||||
console.print('Current queues:')
|
||||
for q, priority in m.queues:
|
||||
s = Padding(f'{q} Recently enqueued captures: {int(priority)}', (0, 2))
|
||||
|
|
|
@ -5,6 +5,7 @@ import calendar
|
|||
import http
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from datetime import date, datetime, timedelta, timezone
|
||||
from io import BytesIO, StringIO
|
||||
|
@ -646,6 +647,8 @@ def tree(tree_uuid: str, node_uuid: Optional[str]=None):
|
|||
|
||||
except NoValidHarFile as e:
|
||||
return render_template('error.html', error_message=e)
|
||||
finally:
|
||||
lookyloo.update_tree_cache_info(os.getpid(), 'website')
|
||||
|
||||
|
||||
@app.route('/tree/<string:tree_uuid>/mark_as_legitimate', methods=['POST'])
|
||||
|
|
Loading…
Reference in New Issue