MatrixSynapse/synapse/util/caches/__init__.py

175 lines
5.2 KiB
Python
Raw Normal View History

# -*- coding: utf-8 -*-
2016-01-07 05:26:29 +01:00
# Copyright 2015, 2016 OpenMarket Ltd
# Copyright 2019 The Matrix.org Foundation C.I.C.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
2018-09-26 13:40:20 +02:00
import logging
2016-03-23 17:13:05 +01:00
import os
from typing import Dict
2016-03-23 17:13:05 +01:00
import six
2018-07-09 08:09:20 +02:00
from six.moves import intern
from prometheus_client.core import REGISTRY, Gauge, GaugeMetricFamily
2018-09-26 13:40:20 +02:00
logger = logging.getLogger(__name__)
2017-07-04 10:58:32 +02:00
CACHE_SIZE_FACTOR = float(os.environ.get("SYNAPSE_CACHE_FACTOR", 0.5))
2018-02-21 21:49:55 +01:00
def get_cache_factor_for(cache_name):
env_var = "SYNAPSE_CACHE_FACTOR_" + cache_name.upper()
factor = os.environ.get(env_var)
if factor:
return float(factor)
return CACHE_SIZE_FACTOR
caches_by_name = {}
collectors_by_name = {} # type: Dict
2018-05-22 02:47:37 +02:00
2018-05-22 23:28:23 +02:00
cache_size = Gauge("synapse_util_caches_cache:size", "", ["name"])
cache_hits = Gauge("synapse_util_caches_cache:hits", "", ["name"])
cache_evicted = Gauge("synapse_util_caches_cache:evicted_size", "", ["name"])
cache_total = Gauge("synapse_util_caches_cache:total", "", ["name"])
response_cache_size = Gauge("synapse_util_caches_response_cache:size", "", ["name"])
response_cache_hits = Gauge("synapse_util_caches_response_cache:hits", "", ["name"])
2018-05-22 23:54:22 +02:00
response_cache_evicted = Gauge(
"synapse_util_caches_response_cache:evicted_size", "", ["name"]
)
2018-05-22 23:28:23 +02:00
response_cache_total = Gauge("synapse_util_caches_response_cache:total", "", ["name"])
2018-05-22 23:54:22 +02:00
def register_cache(cache_type, cache_name, cache, collect_callback=None):
"""Register a cache object for metric collection.
Args:
cache_type (str):
cache_name (str): name of the cache
cache (object): cache itself
collect_callback (callable|None): if not None, a function which is called during
metric collection to update additional metrics.
Returns:
CacheMetric: an object which provides inc_{hits,misses,evictions} methods
"""
2018-05-22 02:47:37 +02:00
# Check if the metric is already registered. Unregister it, if so.
2018-05-28 11:14:41 +02:00
# This usually happens during tests, as at runtime these caches are
# effectively singletons.
2018-05-22 23:54:22 +02:00
metric_name = "cache_%s_%s" % (cache_type, cache_name)
2018-05-22 02:47:37 +02:00
if metric_name in collectors_by_name.keys():
REGISTRY.unregister(collectors_by_name[metric_name])
class CacheMetric(object):
hits = 0
misses = 0
evicted_size = 0
def inc_hits(self):
self.hits += 1
def inc_misses(self):
self.misses += 1
def inc_evictions(self, size=1):
self.evicted_size += size
2018-05-22 23:28:23 +02:00
def describe(self):
return []
2018-05-22 02:47:37 +02:00
def collect(self):
2018-09-26 13:40:20 +02:00
try:
if cache_type == "response_cache":
response_cache_size.labels(cache_name).set(len(cache))
response_cache_hits.labels(cache_name).set(self.hits)
response_cache_evicted.labels(cache_name).set(self.evicted_size)
response_cache_total.labels(cache_name).set(self.hits + self.misses)
else:
cache_size.labels(cache_name).set(len(cache))
cache_hits.labels(cache_name).set(self.hits)
cache_evicted.labels(cache_name).set(self.evicted_size)
cache_total.labels(cache_name).set(self.hits + self.misses)
if collect_callback:
collect_callback()
2018-09-26 13:40:20 +02:00
except Exception as e:
logger.warning("Error calculating metrics for %s: %s", cache_name, e)
2018-09-26 13:40:20 +02:00
raise
2018-05-22 23:28:23 +02:00
yield GaugeMetricFamily("__unused", "")
2018-05-22 02:47:37 +02:00
metric = CacheMetric()
REGISTRY.register(metric)
caches_by_name[cache_name] = cache
collectors_by_name[metric_name] = metric
return metric
2016-03-22 19:22:52 +01:00
2018-05-22 23:28:23 +02:00
2016-03-23 17:13:05 +01:00
KNOWN_KEYS = {
2019-06-20 11:32:02 +02:00
key: key
for key in (
2016-03-23 17:13:05 +01:00
"auth_events",
"content",
"depth",
"event_id",
"hashes",
"origin",
"origin_server_ts",
"prev_events",
"room_id",
"sender",
"signatures",
"state_key",
"type",
"unsigned",
"user_id",
)
}
2016-03-22 19:22:52 +01:00
def intern_string(string):
2017-04-24 15:07:27 +02:00
"""Takes a (potentially) unicode string and interns it if it's ascii
2016-03-23 17:13:05 +01:00
"""
2017-04-24 15:07:27 +02:00
if string is None:
return None
try:
if six.PY2:
string = string.encode("ascii")
2017-04-24 15:07:27 +02:00
return intern(string)
except UnicodeEncodeError:
return string
2016-03-23 17:13:05 +01:00
def intern_dict(dictionary):
"""Takes a dictionary and interns well known keys and their values
"""
2016-03-23 17:34:59 +01:00
return {
KNOWN_KEYS.get(key, key): _intern_known_values(key, value)
for key, value in dictionary.items()
}
2016-03-23 17:13:05 +01:00
2016-03-23 17:34:59 +01:00
def _intern_known_values(key, value):
2019-06-20 11:32:02 +02:00
intern_keys = ("event_id", "room_id", "sender", "user_id", "type", "state_key")
2016-03-23 17:13:05 +01:00
2017-04-24 15:07:27 +02:00
if key in intern_keys:
2016-03-23 17:34:59 +01:00
return intern_string(value)
2016-03-23 17:13:05 +01:00
2016-03-23 17:34:59 +01:00
return value