OpenCloud
/
MatrixSynapse
mirror of https://github.com/matrix-org/synapse
1
0
Fork 0
Code Issues Releases Wiki Activity

Some variable cleanups in cache.py

pull/6391/head
Andrew Morgan 2020-04-27 20:08:45 +01:00
parent fcc9c3ac01
commit 5acca12064
4 changed files with 24 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
synapse/config/cache.py
View File

@ -21,13 +21,15 @@ from ._base import Config, ConfigError
# The prefix for all cache factor-related environment variables # The prefix for all cache factor-related environment variables
_CACHES = {} _CACHES = {}
_CACHE_PREFIX = "SYNAPSE_CACHE_FACTOR" _CACHE_PREFIX = "SYNAPSE_CACHE_FACTOR"
_DEFAULT_FACTOR_SIZE = 0.5
_DEFAULT_EVENT_CACHE_SIZE = "10K"
class CacheProperties(object): class CacheProperties(object):
def __init__(self): def __init__(self):
# The default size factor for all caches # The default factor size for all caches
self.default_size_factor = 0.5 self.default_factor_size = _DEFAULT_FACTOR_SIZE
self.resize_all_caches = None self.resize_all_caches_func = None
properties = CacheProperties() properties = CacheProperties()
@ -43,11 +45,12 @@ def add_resizable_cache(cache_name: str, cache_resize_callback: Callable):
""" """
_CACHES[cache_name.lower()] = cache_resize_callback _CACHES[cache_name.lower()] = cache_resize_callback
# Ensure all loaded caches are resized # Ensure all loaded caches are sized appropriately
#
# This method should only run once the config has been read, # This method should only run once the config has been read,
# as it uses variables from it # as it uses values read from it
if properties.resize_all_caches: if properties.resize_all_caches_func:
properties.resize_all_caches() properties.resize_all_caches_func()
class CacheConfig(Config): class CacheConfig(Config):
@ -57,8 +60,8 @@ class CacheConfig(Config):
@staticmethod @staticmethod
def reset(): def reset():
"""Resets the caches to their defaults. Used for tests.""" """Resets the caches to their defaults. Used for tests."""
properties.default_size_factor = float(os.environ.get(_CACHE_PREFIX, 0.5)) properties.default_factor_size = float(os.environ.get(_CACHE_PREFIX, _DEFAULT_FACTOR_SIZE))
properties.resize_all_caches = None properties.resize_all_caches_func = None
_CACHES.clear() _CACHES.clear()
def generate_config_section(self, **kwargs): def generate_config_section(self, **kwargs):
@ -79,19 +82,18 @@ class CacheConfig(Config):
""" """
def read_config(self, config, **kwargs): def read_config(self, config, **kwargs):
self.event_cache_size = self.parse_size(config.get("event_cache_size", "10K")) self.event_cache_size = self.parse_size(config.get("event_cache_size", _DEFAULT_EVENT_CACHE_SIZE))
self.cache_factors = {} # type: Dict[str, float] self.cache_factors = {} # type: Dict[str, float]
cache_config = config.get("caches", {}) cache_config = config.get("caches", {})
self.global_factor = cache_config.get( self.global_factor = cache_config.get(
"global_factor", properties.default_size_factor "global_factor", properties.default_factor_size
) )
if not isinstance(self.global_factor, (int, float)): if not isinstance(self.global_factor, (int, float)):
raise ConfigError("caches.global_factor must be a number.") raise ConfigError("caches.global_factor must be a number.")
# Set the global one so that it's reflected in new caches # Set the global one so that it's reflected in new caches
properties.default_size_factor = self.global_factor properties.default_factor_size = self.global_factor
# Load cache factors from the environment, but override them with the # Load cache factors from the environment, but override them with the
# ones in the config file if they exist # ones in the config file if they exist
@ -113,9 +115,12 @@ class CacheConfig(Config):
self.cache_factors[cache.lower()] = factor self.cache_factors[cache.lower()] = factor
# Resize all caches (if necessary) with the new factors we've loaded # Resize all caches (if necessary) with the new factors we've loaded
properties.resize_all_caches = self.resize_all_caches
self.resize_all_caches() self.resize_all_caches()
# Store this function so that it can be called from other classes without
# needing an instance of Config
properties.resize_all_caches_func = self.resize_all_caches
def resize_all_caches(self): def resize_all_caches(self):
"""Ensure all cache sizes are up to date """Ensure all cache sizes are up to date

4
synapse/http/client.py
View File

@ -240,7 +240,9 @@ class SimpleHttpClient(object):
# tends to do so in batches, so we need to allow the pool to keep # tends to do so in batches, so we need to allow the pool to keep
# lots of idle connections around. # lots of idle connections around.
pool = HTTPConnectionPool(self.reactor) pool = HTTPConnectionPool(self.reactor)
# XXX: Why does this use the cache factor???? # XXX: The justification for using the cache factor here is that larger instances
# will need both more cache and more connections.
# Still, this should probably be a separate dial
pool.maxPersistentPerHost = max((100 * hs.config.caches.global_factor, 5)) pool.maxPersistentPerHost = max((100 * hs.config.caches.global_factor, 5))
pool.cachedConnectionTimeout = 2 * 60 pool.cachedConnectionTimeout = 2 * 60

2
synapse/util/caches/expiringcache.py
View File

@ -57,7 +57,7 @@ class ExpiringCache(object):
self._cache_name = cache_name self._cache_name = cache_name
self._original_max_size = max_len self._original_max_size = max_len
self._max_size = int(max_len * cache_config.properties.default_size_factor) self._max_size = int(max_len * cache_config.properties.default_factor_size)
self._clock = clock self._clock = clock

2
synapse/util/caches/lrucache.py
View File

@ -79,7 +79,7 @@ class LruCache(object):
# Save the original max size, and apply the default size factor. # Save the original max size, and apply the default size factor.
self._original_max_size = max_size self._original_max_size = max_size
self.max_size = int(max_size * cache_config.properties.default_size_factor) self.max_size = int(max_size * cache_config.properties.default_factor_size)
list_root = _Node(None, None, None, None) list_root = _Node(None, None, None, None)
list_root.next_node = list_root list_root.next_node = list_root