Mark Haines
commit
e815763b7f
|
|
@ -0,0 +1,14 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
# Copyright 2016 OpenMarket Ltd
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
# Copyright 2016 OpenMarket Ltd
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
# Copyright 2016 OpenMarket Ltd
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from synapse.storage._base import SQLBaseStore
|
||||
from twisted.internet import defer
|
||||
|
||||
|
||||
class BaseSlavedStore(SQLBaseStore):
|
||||
def __init__(self, db_conn, hs):
|
||||
super(BaseSlavedStore, self).__init__(hs)
|
||||
|
||||
def stream_positions(self):
|
||||
return {}
|
||||
|
||||
def process_replication(self, result):
|
||||
return defer.succeed(None)
|
||||
|
|
@ -0,0 +1,30 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
# Copyright 2016 OpenMarket Ltd
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from synapse.storage.util.id_generators import _load_current_id
|
||||
|
||||
|
||||
class SlavedIdTracker(object):
|
||||
def __init__(self, db_conn, table, column, extra_tables=[], step=1):
|
||||
self.step = step
|
||||
self._current = _load_current_id(db_conn, table, column, step)
|
||||
for table, column in extra_tables:
|
||||
self.advance(_load_current_id(db_conn, table, column))
|
||||
|
||||
def advance(self, new_id):
|
||||
self._current = (max if self.step > 0 else min)(self._current, new_id)
|
||||
|
||||
def get_current_token(self):
|
||||
return self._current
|
||||
|
|
@ -0,0 +1,198 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
# Copyright 2016 OpenMarket Ltd
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
from ._base import BaseSlavedStore
|
||||
from ._slaved_id_tracker import SlavedIdTracker
|
||||
|
||||
from synapse.api.constants import EventTypes
|
||||
from synapse.events import FrozenEvent
|
||||
from synapse.storage import DataStore
|
||||
from synapse.storage.room import RoomStore
|
||||
from synapse.storage.roommember import RoomMemberStore
|
||||
from synapse.storage.event_federation import EventFederationStore
|
||||
from synapse.storage.state import StateStore
|
||||
from synapse.util.caches.stream_change_cache import StreamChangeCache
|
||||
|
||||
import ujson as json
|
||||
|
||||
# So, um, we want to borrow a load of functions intended for reading from
|
||||
# a DataStore, but we don't want to take functions that either write to the
|
||||
# DataStore or are cached and don't have cache invalidation logic.
|
||||
#
|
||||
# Rather than write duplicate versions of those functions, or lift them to
|
||||
# a common base class, we going to grab the underlying __func__ object from
|
||||
# the method descriptor on the DataStore and chuck them into our class.
|
||||
|
||||
|
||||
class SlavedEventStore(BaseSlavedStore):
|
||||
|
||||
def __init__(self, db_conn, hs):
|
||||
super(SlavedEventStore, self).__init__(db_conn, hs)
|
||||
self._stream_id_gen = SlavedIdTracker(
|
||||
db_conn, "events", "stream_ordering",
|
||||
)
|
||||
self._backfill_id_gen = SlavedIdTracker(
|
||||
db_conn, "events", "stream_ordering", step=-1
|
||||
)
|
||||
events_max = self._stream_id_gen.get_current_token()
|
||||
event_cache_prefill, min_event_val = self._get_cache_dict(
|
||||
db_conn, "events",
|
||||
entity_column="room_id",
|
||||
stream_column="stream_ordering",
|
||||
max_value=events_max,
|
||||
)
|
||||
self._events_stream_cache = StreamChangeCache(
|
||||
"EventsRoomStreamChangeCache", min_event_val,
|
||||
prefilled_cache=event_cache_prefill,
|
||||
)
|
||||
|
||||
# Cached functions can't be accessed through a class instance so we need
|
||||
# to reach inside the __dict__ to extract them.
|
||||
get_room_name_and_aliases = RoomStore.__dict__["get_room_name_and_aliases"]
|
||||
get_rooms_for_user = RoomMemberStore.__dict__["get_rooms_for_user"]
|
||||
get_users_in_room = RoomMemberStore.__dict__["get_users_in_room"]
|
||||
get_latest_event_ids_in_room = EventFederationStore.__dict__[
|
||||
"get_latest_event_ids_in_room"
|
||||
]
|
||||
_get_current_state_for_key = StateStore.__dict__[
|
||||
"_get_current_state_for_key"
|
||||
]
|
||||
|
||||
get_current_state = DataStore.get_current_state.__func__
|
||||
get_current_state_for_key = DataStore.get_current_state_for_key.__func__
|
||||
_get_rooms_for_user_where_membership_is_txn = (
|
||||
DataStore._get_rooms_for_user_where_membership_is_txn.__func__
|
||||
)
|
||||
get_rooms_for_user_where_membership_is = (
|
||||
DataStore.get_rooms_for_user_where_membership_is.__func__
|
||||
)
|
||||
get_membership_changes_for_user = (
|
||||
DataStore.get_membership_changes_for_user.__func__
|
||||
)
|
||||
get_room_events_max_id = DataStore.get_room_events_max_id.__func__
|
||||
get_room_events_stream_for_room = (
|
||||
DataStore.get_room_events_stream_for_room.__func__
|
||||
)
|
||||
_set_before_and_after = DataStore._set_before_and_after
|
||||
|
||||
_get_events = DataStore._get_events.__func__
|
||||
_get_events_from_cache = DataStore._get_events_from_cache.__func__
|
||||
|
||||
_invalidate_get_event_cache = DataStore._invalidate_get_event_cache.__func__
|
||||
_parse_events_txn = DataStore._parse_events_txn.__func__
|
||||
_get_events_txn = DataStore._get_events_txn.__func__
|
||||
_fetch_events_txn = DataStore._fetch_events_txn.__func__
|
||||
_fetch_event_rows = DataStore._fetch_event_rows.__func__
|
||||
_get_event_from_row_txn = DataStore._get_event_from_row_txn.__func__
|
||||
|
||||
def stream_positions(self):
|
||||
result = super(SlavedEventStore, self).stream_positions()
|
||||
result["events"] = self._stream_id_gen.get_current_token()
|
||||
result["backfilled"] = self._backfill_id_gen.get_current_token()
|
||||
return result
|
||||
|
||||
def process_replication(self, result):
|
||||
state_resets = set(
|
||||
r[0] for r in result.get("state_resets", {"rows": []})["rows"]
|
||||
)
|
||||
|
||||
stream = result.get("events")
|
||||
if stream:
|
||||
self._stream_id_gen.advance(stream["position"])
|
||||
for row in stream["rows"]:
|
||||
self._process_replication_row(
|
||||
row, backfilled=False, state_resets=state_resets
|
||||
)
|
||||
|
||||
stream = result.get("backfill")
|
||||
if stream:
|
||||
self._backfill_id_gen.advance(stream["position"])
|
||||
for row in stream["rows"]:
|
||||
self._process_replication_row(
|
||||
row, backfilled=True, state_resets=state_resets
|
||||
)
|
||||
|
||||
stream = result.get("forward_ex_outliers")
|
||||
if stream:
|
||||
for row in stream["rows"]:
|
||||
event_id = row[1]
|
||||
self._invalidate_get_event_cache(event_id)
|
||||
|
||||
stream = result.get("backward_ex_outliers")
|
||||
if stream:
|
||||
for row in stream["rows"]:
|
||||
event_id = row[1]
|
||||
self._invalidate_get_event_cache(event_id)
|
||||
|
||||
return super(SlavedEventStore, self).process_replication(result)
|
||||
|
||||
def _process_replication_row(self, row, backfilled, state_resets):
|
||||
position = row[0]
|
||||
internal = json.loads(row[1])
|
||||
event_json = json.loads(row[2])
|
||||
|
||||
event = FrozenEvent(event_json, internal_metadata_dict=internal)
|
||||
self._invalidate_caches_for_event(
|
||||
event, backfilled, reset_state=position in state_resets
|
||||
)
|
||||
|
||||
def _invalidate_caches_for_event(self, event, backfilled, reset_state):
|
||||
if reset_state:
|
||||
self._get_current_state_for_key.invalidate_all()
|
||||
self.get_rooms_for_user.invalidate_all()
|
||||
self.get_users_in_room.invalidate((event.room_id,))
|
||||
# self.get_joined_hosts_for_room.invalidate((event.room_id,))
|
||||
self.get_room_name_and_aliases.invalidate((event.room_id,))
|
||||
|
||||
self._invalidate_get_event_cache(event.event_id)
|
||||
|
||||
if not backfilled:
|
||||
self._events_stream_cache.entity_has_changed(
|
||||
event.room_id, event.internal_metadata.stream_ordering
|
||||
)
|
||||
|
||||
# self.get_unread_event_push_actions_by_room_for_user.invalidate_many(
|
||||
# (event.room_id,)
|
||||
# )
|
||||
|
||||
if event.type == EventTypes.Redaction:
|
||||
self._invalidate_get_event_cache(event.redacts)
|
||||
|
||||
if event.type == EventTypes.Member:
|
||||
self.get_rooms_for_user.invalidate((event.state_key,))
|
||||
# self.get_joined_hosts_for_room.invalidate((event.room_id,))
|
||||
self.get_users_in_room.invalidate((event.room_id,))
|
||||
# self._membership_stream_cache.entity_has_changed(
|
||||
# event.state_key, event.internal_metadata.stream_ordering
|
||||
# )
|
||||
|
||||
if not event.is_state():
|
||||
return
|
||||
|
||||
if backfilled:
|
||||
return
|
||||
|
||||
if (not event.internal_metadata.is_invite_from_remote()
|
||||
and event.internal_metadata.is_outlier()):
|
||||
return
|
||||
|
||||
self._get_current_state_for_key.invalidate((
|
||||
event.room_id, event.type, event.state_key
|
||||
))
|
||||
|
||||
if event.type in [EventTypes.Name, EventTypes.Aliases]:
|
||||
self.get_room_name_and_aliases.invalidate(
|
||||
(event.room_id,)
|
||||
)
|
||||
pass
|
||||
|
|
@ -1134,7 +1134,7 @@ class EventsStore(SQLBaseStore):
|
|||
upper_bound = current_forward_id
|
||||
|
||||
sql = (
|
||||
"SELECT -event_stream_ordering FROM current_state_resets"
|
||||
"SELECT event_stream_ordering FROM current_state_resets"
|
||||
" WHERE ? < event_stream_ordering"
|
||||
" AND event_stream_ordering <= ?"
|
||||
" ORDER BY event_stream_ordering ASC"
|
||||
|
|
@ -1143,7 +1143,7 @@ class EventsStore(SQLBaseStore):
|
|||
state_resets = txn.fetchall()
|
||||
|
||||
sql = (
|
||||
"SELECT -event_stream_ordering, event_id, state_group"
|
||||
"SELECT event_stream_ordering, event_id, state_group"
|
||||
" FROM ex_outlier_stream"
|
||||
" WHERE ? > event_stream_ordering"
|
||||
" AND event_stream_ordering >= ?"
|
||||
|
|
|
|||
|
|
@ -0,0 +1,14 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
# Copyright 2016 OpenMarket Ltd
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
# Copyright 2016 OpenMarket Ltd
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
|
@ -0,0 +1,57 @@
|
|||
# Copyright 2016 OpenMarket Ltd
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from twisted.internet import defer
|
||||
from tests import unittest
|
||||
|
||||
from synapse.replication.slave.storage.events import SlavedEventStore
|
||||
|
||||
from mock import Mock, NonCallableMock
|
||||
from tests.utils import setup_test_homeserver
|
||||
from synapse.replication.resource import ReplicationResource
|
||||
|
||||
|
||||
class BaseSlavedStoreTestCase(unittest.TestCase):
|
||||
@defer.inlineCallbacks
|
||||
def setUp(self):
|
||||
self.hs = yield setup_test_homeserver(
|
||||
"blue",
|
||||
http_client=None,
|
||||
replication_layer=Mock(),
|
||||
ratelimiter=NonCallableMock(spec_set=[
|
||||
"send_message",
|
||||
]),
|
||||
)
|
||||
self.hs.get_ratelimiter().send_message.return_value = (True, 0)
|
||||
|
||||
self.replication = ReplicationResource(self.hs)
|
||||
|
||||
self.master_store = self.hs.get_datastore()
|
||||
self.slaved_store = SlavedEventStore(self.hs.get_db_conn(), self.hs)
|
||||
self.event_id = 0
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def replicate(self):
|
||||
streams = self.slaved_store.stream_positions()
|
||||
result = yield self.replication.replicate(streams, 100)
|
||||
yield self.slaved_store.process_replication(result)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def check(self, method, args, expected_result=None):
|
||||
master_result = yield getattr(self.master_store, method)(*args)
|
||||
slaved_result = yield getattr(self.slaved_store, method)(*args)
|
||||
self.assertEqual(master_result, slaved_result)
|
||||
if expected_result is not None:
|
||||
self.assertEqual(master_result, expected_result)
|
||||
self.assertEqual(slaved_result, expected_result)
|
||||
|
|
@ -0,0 +1,114 @@
|
|||
# Copyright 2016 OpenMarket Ltd
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from ._base import BaseSlavedStoreTestCase
|
||||
|
||||
from synapse.types import UserID
|
||||
from synapse.events import FrozenEvent
|
||||
from synapse.events.snapshot import EventContext
|
||||
|
||||
from twisted.internet import defer
|
||||
|
||||
USER_ID = "@feeling:blue"
|
||||
USER = UserID.from_string(USER_ID)
|
||||
OUTLIER = {"outlier": True}
|
||||
ROOM_ID = "!room:blue"
|
||||
|
||||
|
||||
class SlavedEventStoreTestCase(BaseSlavedStoreTestCase):
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_room_name_and_aliases(self):
|
||||
create = yield self.persist(type="m.room.create", key="", creator=USER_ID)
|
||||
yield self.persist(type="m.room.member", key=USER_ID, membership="join")
|
||||
yield self.persist(type="m.room.name", key="", name="name1")
|
||||
yield self.persist(
|
||||
type="m.room.aliases", key="blue", aliases=["#1:blue"]
|
||||
)
|
||||
yield self.replicate()
|
||||
yield self.check(
|
||||
"get_room_name_and_aliases", (ROOM_ID,), ("name1", ["#1:blue"])
|
||||
)
|
||||
|
||||
# Set the room name.
|
||||
yield self.persist(type="m.room.name", key="", name="name2")
|
||||
yield self.replicate()
|
||||
yield self.check(
|
||||
"get_room_name_and_aliases", (ROOM_ID,), ("name2", ["#1:blue"])
|
||||
)
|
||||
|
||||
# Set the room aliases.
|
||||
yield self.persist(
|
||||
type="m.room.aliases", key="blue", aliases=["#2:blue"]
|
||||
)
|
||||
yield self.replicate()
|
||||
yield self.check(
|
||||
"get_room_name_and_aliases", (ROOM_ID,), ("name2", ["#2:blue"])
|
||||
)
|
||||
|
||||
# Leave and join the room clobbering the state.
|
||||
yield self.persist(type="m.room.member", key=USER_ID, membership="leave")
|
||||
yield self.persist(
|
||||
type="m.room.member", key=USER_ID, membership="join",
|
||||
reset_state=[create]
|
||||
)
|
||||
yield self.replicate()
|
||||
|
||||
yield self.check(
|
||||
"get_room_name_and_aliases", (ROOM_ID,), (None, [])
|
||||
)
|
||||
|
||||
event_id = 0
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def persist(
|
||||
self, sender=USER_ID, room_id=ROOM_ID, type={}, key=None,
|
||||
internal={},
|
||||
state=None, reset_state=False, backfill=False,
|
||||
depth=None, prev_events=[], auth_events=[], prev_state=[],
|
||||
**content
|
||||
):
|
||||
if depth is None:
|
||||
depth = self.event_id
|
||||
|
||||
event_dict = {
|
||||
"sender": sender,
|
||||
"type": type,
|
||||
"content": content,
|
||||
"event_id": "$%d:blue" % (self.event_id,),
|
||||
"room_id": room_id,
|
||||
"depth": depth,
|
||||
"origin_server_ts": self.event_id,
|
||||
"prev_events": prev_events,
|
||||
"auth_events": auth_events,
|
||||
}
|
||||
if key is not None:
|
||||
event_dict["state_key"] = key
|
||||
event_dict["prev_state"] = prev_state
|
||||
|
||||
event = FrozenEvent(event_dict, internal_metadata_dict=internal)
|
||||
|
||||
self.event_id += 1
|
||||
|
||||
context = EventContext(current_state=state)
|
||||
|
||||
if backfill:
|
||||
yield self.master_store.persist_events(
|
||||
[(event, context)], backfilled=True
|
||||
)
|
||||
else:
|
||||
yield self.master_store.persist_event(
|
||||
event, context, current_state=reset_state
|
||||
)
|
||||
defer.returnValue(event)
|
||||
Loading…
Reference in New Issue