Merge pull request #2760 from Valodim/pypy

Synapse on PyPy
erikj/state_delta_writeup
Richard van der Hoff 2018-04-11 11:20:01 +01:00 committed by GitHub
commit 233699c42e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 24 additions and 13 deletions

View File

@ -35,7 +35,7 @@ from synapse.storage.presence import UserPresenceState
from synapse.util.metrics import Measure from synapse.util.metrics import Measure
import synapse.metrics import synapse.metrics
from blist import sorteddict from sortedcontainers import SortedDict
from collections import namedtuple from collections import namedtuple
import logging import logging
@ -56,19 +56,19 @@ class FederationRemoteSendQueue(object):
self.is_mine_id = hs.is_mine_id self.is_mine_id = hs.is_mine_id
self.presence_map = {} # Pending presence map user_id -> UserPresenceState self.presence_map = {} # Pending presence map user_id -> UserPresenceState
self.presence_changed = sorteddict() # Stream position -> user_id self.presence_changed = SortedDict() # Stream position -> user_id
self.keyed_edu = {} # (destination, key) -> EDU self.keyed_edu = {} # (destination, key) -> EDU
self.keyed_edu_changed = sorteddict() # stream position -> (destination, key) self.keyed_edu_changed = SortedDict() # stream position -> (destination, key)
self.edus = sorteddict() # stream position -> Edu self.edus = SortedDict() # stream position -> Edu
self.failures = sorteddict() # stream position -> (destination, Failure) self.failures = SortedDict() # stream position -> (destination, Failure)
self.device_messages = sorteddict() # stream position -> destination self.device_messages = SortedDict() # stream position -> destination
self.pos = 1 self.pos = 1
self.pos_time = sorteddict() self.pos_time = SortedDict()
# EVERYTHING IS SAD. In particular, python only makes new scopes when # EVERYTHING IS SAD. In particular, python only makes new scopes when
# we make a new function, so we need to make a new function so the inner # we make a new function, so we need to make a new function so the inner

View File

@ -17,6 +17,7 @@ import logging
import functools import functools
import time import time
import gc import gc
import platform
from twisted.internet import reactor from twisted.internet import reactor
@ -30,6 +31,7 @@ from .process_collector import register_process_collector
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
running_on_pypy = platform.python_implementation() == 'PyPy'
all_metrics = [] all_metrics = []
all_collectors = [] all_collectors = []
@ -174,6 +176,9 @@ def runUntilCurrentTimer(func):
tick_time.inc_by(end - start) tick_time.inc_by(end - start)
pending_calls_metric.inc_by(num_pending) pending_calls_metric.inc_by(num_pending)
if running_on_pypy:
return ret
# Check if we need to do a manual GC (since its been disabled), and do # Check if we need to do a manual GC (since its been disabled), and do
# one if necessary. # one if necessary.
threshold = gc.get_threshold() threshold = gc.get_threshold()
@ -206,6 +211,7 @@ try:
# We manually run the GC each reactor tick so that we can get some metrics # We manually run the GC each reactor tick so that we can get some metrics
# about time spent doing GC, # about time spent doing GC,
gc.disable() if not running_on_pypy:
gc.disable()
except AttributeError: except AttributeError:
pass pass

View File

@ -34,8 +34,8 @@ REQUIREMENTS = {
"bcrypt": ["bcrypt>=3.1.0"], "bcrypt": ["bcrypt>=3.1.0"],
"pillow": ["PIL"], "pillow": ["PIL"],
"pydenticon": ["pydenticon"], "pydenticon": ["pydenticon"],
"blist": ["blist"],
"pysaml2>=3.0.0": ["saml2>=3.0.0"], "pysaml2>=3.0.0": ["saml2>=3.0.0"],
"sortedcontainers": ["sortedcontainers"],
"pymacaroons-pynacl": ["pymacaroons"], "pymacaroons-pynacl": ["pymacaroons"],
"msgpack-python>=0.3.0": ["msgpack"], "msgpack-python>=0.3.0": ["msgpack"],
"phonenumbers>=8.2.0": ["phonenumbers"], "phonenumbers>=8.2.0": ["phonenumbers"],

View File

@ -376,7 +376,7 @@ class SQLBaseStore(object):
Returns: Returns:
A list of dicts where the key is the column header. A list of dicts where the key is the column header.
""" """
col_headers = list(intern(column[0]) for column in cursor.description) col_headers = list(intern(str(column[0])) for column in cursor.description)
results = list( results = list(
dict(zip(col_headers, row)) for row in cursor dict(zip(col_headers, row)) for row in cursor
) )

View File

@ -18,6 +18,7 @@ from .postgres import PostgresEngine
from .sqlite3 import Sqlite3Engine from .sqlite3 import Sqlite3Engine
import importlib import importlib
import platform
SUPPORTED_MODULE = { SUPPORTED_MODULE = {
@ -31,6 +32,10 @@ def create_engine(database_config):
engine_class = SUPPORTED_MODULE.get(name, None) engine_class = SUPPORTED_MODULE.get(name, None)
if engine_class: if engine_class:
# pypy requires psycopg2cffi rather than psycopg2
if (name == "psycopg2" and
platform.python_implementation() == "PyPy"):
name = "psycopg2cffi"
module = importlib.import_module(name) module = importlib.import_module(name)
return engine_class(module, database_config) return engine_class(module, database_config)

View File

@ -169,7 +169,7 @@ class DomainSpecificString(
except Exception: except Exception:
return False return False
__str__ = to_string __repr__ = to_string
class UserID(DomainSpecificString): class UserID(DomainSpecificString):

View File

@ -16,7 +16,7 @@
from synapse.util.caches import register_cache, CACHE_SIZE_FACTOR from synapse.util.caches import register_cache, CACHE_SIZE_FACTOR
from blist import sorteddict from sortedcontainers import SortedDict
import logging import logging
@ -35,7 +35,7 @@ class StreamChangeCache(object):
def __init__(self, name, current_stream_pos, max_size=10000, prefilled_cache={}): def __init__(self, name, current_stream_pos, max_size=10000, prefilled_cache={}):
self._max_size = int(max_size * CACHE_SIZE_FACTOR) self._max_size = int(max_size * CACHE_SIZE_FACTOR)
self._entity_to_key = {} self._entity_to_key = {}
self._cache = sorteddict() self._cache = SortedDict()
self._earliest_known_stream_pos = current_stream_pos self._earliest_known_stream_pos = current_stream_pos
self.name = name self.name = name
self.metrics = register_cache(self.name, self._cache) self.metrics = register_cache(self.name, self._cache)