CIRCL
/
lookyloo
mirror of https://github.com/CIRCL/lookyloo
2
0
Fork 0
Code Issues Releases Wiki Activity

chg: Try using redis connection pool

pull/251/head
Raphaël Vinot 2021-08-18 16:07:39 +02:00
parent 725a43cc44
commit 3a96e27f8c
1 changed files with 8 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
lookyloo/lookyloo.py
View File

@ -31,7 +31,8 @@ from pymisp import MISPEvent, MISPAttribute, MISPObject
from pymisp.tools import URLObject, FileObject from pymisp.tools import URLObject, FileObject
import requests import requests
from requests.exceptions import HTTPError from requests.exceptions import HTTPError
from redis import Redis from redis import Redis, ConnectionPool
from redis.connection import UnixDomainSocketConnection
from scrapysplashwrapper import crawl from scrapysplashwrapper import crawl
from werkzeug.useragents import UserAgent from werkzeug.useragents import UserAgent
@ -56,7 +57,8 @@ class Lookyloo():
self.public_domain = get_config('generic', 'public_domain') self.public_domain = get_config('generic', 'public_domain')
self.taxonomies = get_taxonomies() self.taxonomies = get_taxonomies()
self.redis: Redis = Redis(unix_socket_path=get_socket_path('cache'), decode_responses=True) self.redis_pool: ConnectionPool = ConnectionPool(connection_class=UnixDomainSocketConnection,
path=get_socket_path('cache'), decode_responses=True)
self.capture_dir: Path = get_homedir() / 'scraped' self.capture_dir: Path = get_homedir() / 'scraped'
if os.environ.get('SPLASH_URL_DOCKER'): if os.environ.get('SPLASH_URL_DOCKER'):
# In order to have a working default for the docker image, it is easier to use an environment variable # In order to have a working default for the docker image, it is easier to use an environment variable
@ -100,6 +102,10 @@ class Lookyloo():
if not self.redis.exists('cache_loaded'): if not self.redis.exists('cache_loaded'):
self._init_existing_dumps() self._init_existing_dumps()
@property
def redis(self):
return Redis(connection_pool=self.redis_pool)
def _get_priority(self, source: str, user: str, authenticated: bool) -> int: def _get_priority(self, source: str, user: str, authenticated: bool) -> int:
src_prio: int = self._priority['sources'][source] if source in self._priority['sources'] else -1 src_prio: int = self._priority['sources'][source] if source in self._priority['sources'] else -1
if not authenticated: if not authenticated: