OpenCloud
/
MatrixSynapse
mirror of https://github.com/matrix-org/synapse
1
0
Fork 0
Code Issues Releases Wiki Activity

Add `rooms.room_version` column (#6729) 

This is so that we don't have to rely on pulling it out from `current_state_events` table.
pull/6788/head
Erik Johnston 2020-01-27 14:30:57 +00:00 committed by GitHub
parent d5275fc55f
commit 8df862e45d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 270 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
changelog.d/6729.misc Normal file
View File

@ -0,0 +1 @@
Record room versions in the `rooms` table.

50
synapse/federation/federation_client.py
View File

@ -17,6 +17,7 @@
import copy
import itertools
import logging
from typing import Dict, Iterable
from prometheus_client import Counter
@ -29,6 +30,7 @@ from synapse.api.errors import (
FederationDeniedError,
HttpResponseException,
SynapseError,
UnsupportedRoomVersionError,
)
from synapse.api.room_versions import (
KNOWN_ROOM_VERSIONS,
@ -385,6 +387,8 @@ class FederationClient(FederationBase):
return res
except InvalidResponseError as e:
logger.warning("Failed to %s via %s: %s", description, destination, e)
except UnsupportedRoomVersionError:
raise
except HttpResponseException as e:
if not 500 <= e.code < 600:
raise e.to_synapse_error()
@ -404,7 +408,13 @@ class FederationClient(FederationBase):
raise SynapseError(502, "Failed to %s via any server" % (description,))
def make_membership_event(
self, destinations, room_id, user_id, membership, content, params
self,
destinations: Iterable[str],
room_id: str,
user_id: str,
membership: str,
content: dict,
params: Dict[str, str],
):
"""
Creates an m.room.member event, with context, without participating in the room.
@ -417,21 +427,23 @@ class FederationClient(FederationBase):
Note that this does not append any events to any graphs.
Args:
destinations (Iterable[str]): Candidate homeservers which are probably
destinations: Candidate homeservers which are probably
participating in the room.
room_id (str): The room in which the event will happen.
user_id (str): The user whose membership is being evented.
membership (str): The "membership" property of the event. Must be
one of "join" or "leave".
content (dict): Any additional data to put into the content field
of the event.
params (dict[str, str|Iterable[str]]): Query parameters to include in the
request.
room_id: The room in which the event will happen.
user_id: The user whose membership is being evented.
membership: The "membership" property of the event. Must be one of
"join" or "leave".
content: Any additional data to put into the content field of the
event.
params: Query parameters to include in the request.
Return:
Deferred[tuple[str, FrozenEvent, int]]: resolves to a tuple of
`(origin, event, event_format)` where origin is the remote
homeserver which generated the event, and event_format is one of
`synapse.api.room_versions.EventFormatVersions`.
Deferred[Tuple[str, FrozenEvent, RoomVersion]]: resolves to a tuple of
`(origin, event, room_version)` where origin is the remote
homeserver which generated the event, and room_version is the
version of the room.
Fails with a `UnsupportedRoomVersionError` if remote responds with
a room version we don't understand.
Fails with a ``SynapseError`` if the chosen remote server
returns a 300/400 code.
@ -453,8 +465,12 @@ class FederationClient(FederationBase):
# Note: If not supplied, the room version may be either v1 or v2,
# however either way the event format version will be v1.
room_version = ret.get("room_version", RoomVersions.V1.identifier)
event_format = room_version_to_event_format(room_version)
room_version_id = ret.get("room_version", RoomVersions.V1.identifier)
room_version = KNOWN_ROOM_VERSIONS.get(room_version_id)
if not room_version:
raise UnsupportedRoomVersionError()
event_format = room_version_to_event_format(room_version_id)
pdu_dict = ret.get("event", None)
if not isinstance(pdu_dict, dict):
@ -478,7 +494,7 @@ class FederationClient(FederationBase):
event_dict=pdu_dict,
)
return (destination, ev, event_format)
return (destination, ev, room_version)
return self._try_destination_list(
"make_" + membership, destinations, send_request

65
synapse/handlers/federation.py
View File

@ -44,10 +44,10 @@ from synapse.api.errors import (
StoreError,
SynapseError,
)
from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, RoomVersions
from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, RoomVersion, RoomVersions
from synapse.crypto.event_signing import compute_event_signature
from synapse.event_auth import auth_types_for_event
from synapse.events import EventBase
from synapse.events import EventBase, room_version_to_event_format
from synapse.events.snapshot import EventContext
from synapse.events.validator import EventValidator
from synapse.logging.context import (
@ -703,8 +703,20 @@ class FederationHandler(BaseHandler):
if not room:
try:
prev_state_ids = await context.get_prev_state_ids()
create_event = await self.store.get_event(
prev_state_ids[(EventTypes.Create, "")]
)
room_version_id = create_event.content.get(
"room_version", RoomVersions.V1.identifier
)
await self.store.store_room(
room_id=room_id, room_creator_user_id="", is_public=False
room_id=room_id,
room_creator_user_id="",
is_public=False,
room_version=KNOWN_ROOM_VERSIONS[room_version_id],
)
except StoreError:
logger.exception("Failed to store room.")
@ -1186,7 +1198,7 @@ class FederationHandler(BaseHandler):
"""
logger.debug("Joining %s to %s", joinee, room_id)
origin, event, event_format_version = yield self._make_and_verify_event(
origin, event, room_version = yield self._make_and_verify_event(
target_hosts,
room_id,
joinee,
@ -1214,6 +1226,8 @@ class FederationHandler(BaseHandler):
target_hosts.insert(0, origin)
except ValueError:
pass
event_format_version = room_version_to_event_format(room_version.identifier)
ret = yield self.federation_client.send_join(
target_hosts, event, event_format_version
)
@ -1234,13 +1248,18 @@ class FederationHandler(BaseHandler):
try:
yield self.store.store_room(
room_id=room_id, room_creator_user_id="", is_public=False
room_id=room_id,
room_creator_user_id="",
is_public=False,
room_version=room_version,
)
except Exception:
# FIXME
pass
yield self._persist_auth_tree(origin, auth_chain, state, event)
yield self._persist_auth_tree(
origin, auth_chain, state, event, room_version
)
# Check whether this room is the result of an upgrade of a room we already know
# about. If so, migrate over user information
@ -1486,7 +1505,7 @@ class FederationHandler(BaseHandler):
@defer.inlineCallbacks
def do_remotely_reject_invite(self, target_hosts, room_id, user_id, content):
origin, event, event_format_version = yield self._make_and_verify_event(
origin, event, room_version = yield self._make_and_verify_event(
target_hosts, room_id, user_id, "leave", content=content
)
# Mark as outlier as we don't have any state for this event; we're not
@ -1513,7 +1532,11 @@ class FederationHandler(BaseHandler):
def _make_and_verify_event(
self, target_hosts, room_id, user_id, membership, content={}, params=None
):
origin, event, format_ver = yield self.federation_client.make_membership_event(
(
origin,
event,
room_version,
) = yield self.federation_client.make_membership_event(
target_hosts, room_id, user_id, membership, content, params=params
)
@ -1525,7 +1548,7 @@ class FederationHandler(BaseHandler):
assert event.user_id == user_id
assert event.state_key == user_id
assert event.room_id == room_id
return origin, event, format_ver
return origin, event, room_version
@defer.inlineCallbacks
@log_function
@ -1810,7 +1833,14 @@ class FederationHandler(BaseHandler):
)
@defer.inlineCallbacks
def _persist_auth_tree(self, origin, auth_events, state, event):
def _persist_auth_tree(
self,
origin: str,
auth_events: List[EventBase],
state: List[EventBase],
event: EventBase,
room_version: RoomVersion,
):
"""Checks the auth chain is valid (and passes auth checks) for the
state and event. Then persists the auth chain and state atomically.
Persists the event separately. Notifies about the persisted events
@ -1819,10 +1849,12 @@ class FederationHandler(BaseHandler):
Will attempt to fetch missing auth events.
Args:
origin (str): Where the events came from
auth_events (list)
state (list)
event (Event)
origin: Where the events came from
auth_events
state
event
room_version: The room version we expect this room to have, and
will raise if it doesn't match the version in the create event.
Returns:
Deferred
@ -1848,10 +1880,13 @@ class FederationHandler(BaseHandler):
# invalid, and it would fail auth checks anyway.
raise SynapseError(400, "No create event in state")
room_version = create_event.content.get(
room_version_id = create_event.content.get(
"room_version", RoomVersions.V1.identifier
)
if room_version.identifier != room_version_id:
raise SynapseError(400, "Room version mismatch")
missing_auth_events = set()
for e in itertools.chain(auth_events, state, [event]):
for e_id in e.auth_event_ids():

52
synapse/handlers/room.py
View File

@ -29,7 +29,7 @@ from twisted.internet import defer
from synapse.api.constants import EventTypes, JoinRules, RoomCreationPreset
from synapse.api.errors import AuthError, Codes, NotFoundError, StoreError, SynapseError
from synapse.api.room_versions import KNOWN_ROOM_VERSIONS
from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, RoomVersion
from synapse.http.endpoint import parse_and_validate_server_name
from synapse.storage.state import StateFilter
from synapse.types import (
@ -100,13 +100,15 @@ class RoomCreationHandler(BaseHandler):
self.third_party_event_rules = hs.get_third_party_event_rules()
@defer.inlineCallbacks
def upgrade_room(self, requester, old_room_id, new_version):
def upgrade_room(
self, requester: Requester, old_room_id: str, new_version: RoomVersion
):
"""Replace a room with a new room with a different version
Args:
requester (synapse.types.Requester): the user requesting the upgrade
old_room_id (unicode): the id of the room to be replaced
new_version (unicode): the new room version to use
requester: the user requesting the upgrade
old_room_id: the id of the room to be replaced
new_version: the new room version to use
Returns:
Deferred[unicode]: the new room id
@ -151,7 +153,7 @@ class RoomCreationHandler(BaseHandler):
if r is None:
raise NotFoundError("Unknown room id %s" % (old_room_id,))
new_room_id = yield self._generate_room_id(
creator_id=user_id, is_public=r["is_public"]
creator_id=user_id, is_public=r["is_public"], room_version=new_version,
)
logger.info("Creating new room %s to replace %s", new_room_id, old_room_id)
@ -299,18 +301,22 @@ class RoomCreationHandler(BaseHandler):
@defer.inlineCallbacks
def clone_existing_room(
self, requester, old_room_id, new_room_id, new_room_version, tombstone_event_id
self,
requester: Requester,
old_room_id: str,
new_room_id: str,
new_room_version: RoomVersion,
tombstone_event_id: str,
):
"""Populate a new room based on an old room
Args:
requester (synapse.types.Requester): the user requesting the upgrade
old_room_id (unicode): the id of the room to be replaced
new_room_id (unicode): the id to give the new room (should already have been
requester: the user requesting the upgrade
old_room_id : the id of the room to be replaced
new_room_id: the id to give the new room (should already have been
created with _gemerate_room_id())
new_room_version (unicode): the new room version to use
tombstone_event_id (unicode|str): the ID of the tombstone event in the old
room.
new_room_version: the new room version to use
tombstone_event_id: the ID of the tombstone event in the old room.
Returns:
Deferred
"""
@ -320,7 +326,7 @@ class RoomCreationHandler(BaseHandler):
raise SynapseError(403, "You are not permitted to create rooms")
creation_content = {
"room_version": new_room_version,
"room_version": new_room_version.identifier,
"predecessor": {"room_id": old_room_id, "event_id": tombstone_event_id},
}
@ -577,14 +583,15 @@ class RoomCreationHandler(BaseHandler):
if ratelimit:
yield self.ratelimit(requester)
room_version = config.get(
room_version_id = config.get(
"room_version", self.config.default_room_version.identifier
)
if not isinstance(room_version, string_types):
if not isinstance(room_version_id, string_types):
raise SynapseError(400, "room_version must be a string", Codes.BAD_JSON)
if room_version not in KNOWN_ROOM_VERSIONS:
room_version = KNOWN_ROOM_VERSIONS.get(room_version_id)
if room_version is None:
raise SynapseError(
400,
"Your homeserver does not support this room version",
@ -631,7 +638,9 @@ class RoomCreationHandler(BaseHandler):
visibility = config.get("visibility", None)
is_public = visibility == "public"
room_id = yield self._generate_room_id(creator_id=user_id, is_public=is_public)
room_id = yield self._generate_room_id(
creator_id=user_id, is_public=is_public, room_version=room_version,
)
directory_handler = self.hs.get_handlers().directory_handler
if room_alias:
@ -660,7 +669,7 @@ class RoomCreationHandler(BaseHandler):
creation_content = config.get("creation_content", {})
# override any attempt to set room versions via the creation_content
creation_content["room_version"] = room_version
creation_content["room_version"] = room_version.identifier
yield self._send_events_for_new_room(
requester,
@ -849,7 +858,9 @@ class RoomCreationHandler(BaseHandler):
yield send(etype=etype, state_key=state_key, content=content)
@defer.inlineCallbacks
def _generate_room_id(self, creator_id, is_public):
def _generate_room_id(
self, creator_id: str, is_public: str, room_version: RoomVersion,
):
# autogen room IDs and try to create it. We may clash, so just
# try a few times till one goes through, giving up eventually.
attempts = 0
@ -863,6 +874,7 @@ class RoomCreationHandler(BaseHandler):
room_id=gen_room_id,
room_creator_user_id=creator_id,
is_public=is_public,
room_version=room_version,
)
return gen_room_id
except StoreError:

3
synapse/rest/client/v2_alpha/room_upgrade_rest_servlet.py
View File

@ -64,7 +64,8 @@ class RoomUpgradeRestServlet(RestServlet):
assert_params_in_dict(content, ("new_version",))
new_version = content["new_version"]
if new_version not in KNOWN_ROOM_VERSIONS:
new_version = KNOWN_ROOM_VERSIONS.get(content["new_version"])
if new_version is None:
raise SynapseError(
400,
"Your homeserver does not support this room version",

94
synapse/storage/data_stores/main/room.py
View File

@ -29,9 +29,10 @@ from twisted.internet import defer
from synapse.api.constants import EventTypes
from synapse.api.errors import StoreError
from synapse.api.room_versions import RoomVersion, RoomVersions
from synapse.storage._base import SQLBaseStore
from synapse.storage.data_stores.main.search import SearchStore
from synapse.storage.database import Database
from synapse.storage.database import Database, LoggingTransaction
from synapse.types import ThirdPartyInstanceID
from synapse.util.caches.descriptors import cached, cachedInlineCallbacks
@ -734,6 +735,7 @@ class RoomWorkerStore(SQLBaseStore):
class RoomBackgroundUpdateStore(SQLBaseStore):
REMOVE_TOMESTONED_ROOMS_BG_UPDATE = "remove_tombstoned_rooms_from_directory"
ADD_ROOMS_ROOM_VERSION_COLUMN = "add_rooms_room_version_column"
def __init__(self, database: Database, db_conn, hs):
super(RoomBackgroundUpdateStore, self).__init__(database, db_conn, hs)
@ -749,6 +751,11 @@ class RoomBackgroundUpdateStore(SQLBaseStore):
self._remove_tombstoned_rooms_from_directory,
)
self.db.updates.register_background_update_handler(
self.ADD_ROOMS_ROOM_VERSION_COLUMN,
self._background_add_rooms_room_version_column,
)
@defer.inlineCallbacks
def _background_insert_retention(self, progress, batch_size):
"""Retrieves a list of all rooms within a range and inserts an entry for each of
@ -817,6 +824,73 @@ class RoomBackgroundUpdateStore(SQLBaseStore):
defer.returnValue(batch_size)
async def _background_add_rooms_room_version_column(
self, progress: dict, batch_size: int
):
"""Background update to go and add room version inforamtion to `rooms`
table from `current_state_events` table.
"""
last_room_id = progress.get("room_id", "")
def _background_add_rooms_room_version_column_txn(txn: LoggingTransaction):
sql = """
SELECT room_id, json FROM current_state_events
INNER JOIN event_json USING (room_id, event_id)
WHERE room_id > ? AND type = 'm.room.create' AND state_key = ''
ORDER BY room_id
LIMIT ?
"""
txn.execute(sql, (last_room_id, batch_size))
updates = []
for room_id, event_json in txn:
event_dict = json.loads(event_json)
room_version_id = event_dict.get("content", {}).get(
"room_version", RoomVersions.V1.identifier
)
creator = event_dict.get("content").get("creator")
updates.append((room_id, creator, room_version_id))
if not updates:
return True
new_last_room_id = ""
for room_id, creator, room_version_id in updates:
# We upsert here just in case we don't already have a row,
# mainly for paranoia as much badness would happen if we don't
# insert the row and then try and get the room version for the
# room.
self.db.simple_upsert_txn(
txn,
table="rooms",
keyvalues={"room_id": room_id},
values={"room_version": room_version_id},
insertion_values={"is_public": False, "creator": creator},
)
new_last_room_id = room_id
self.db.updates._background_update_progress_txn(
txn, self.ADD_ROOMS_ROOM_VERSION_COLUMN, {"room_id": new_last_room_id}
)
return False
end = await self.db.runInteraction(
"_background_add_rooms_room_version_column",
_background_add_rooms_room_version_column_txn,
)
if end:
await self.db.updates._end_background_update(
self.ADD_ROOMS_ROOM_VERSION_COLUMN
)
return batch_size
async def _remove_tombstoned_rooms_from_directory(
self, progress, batch_size
) -> int:
@ -881,14 +955,21 @@ class RoomStore(RoomBackgroundUpdateStore, RoomWorkerStore, SearchStore):
self.config = hs.config
@defer.inlineCallbacks
def store_room(self, room_id, room_creator_user_id, is_public):
def store_room(
self,
room_id: str,
room_creator_user_id: str,
is_public: bool,
room_version: RoomVersion,
):
"""Stores a room.
Args:
room_id (str): The desired room ID, can be None.
room_creator_user_id (str): The user ID of the room creator.
is_public (bool): True to indicate that this room should appear in
public room lists.
room_id: The desired room ID, can be None.
room_creator_user_id: The user ID of the room creator.
is_public: True to indicate that this room should appear in
public room lists.
room_version: The version of the room
Raises:
StoreError if the room could not be stored.
"""
@ -902,6 +983,7 @@ class RoomStore(RoomBackgroundUpdateStore, RoomWorkerStore, SearchStore):
"room_id": room_id,
"creator": room_creator_user_id,
"is_public": is_public,
"room_version": room_version.identifier,
},
)
if is_public:

24
synapse/storage/data_stores/main/schema/delta/57/rooms_version_column.sql Normal file
View File

@ -0,0 +1,24 @@
/* Copyright 2020 The Matrix.org Foundation C.I.C
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-- We want to start storing the room version independently of
-- `current_state_events` so that we can delete stale entries from it without
-- losing the information.
ALTER TABLE rooms ADD COLUMN room_version TEXT;
INSERT into background_updates (update_name, progress_json)
VALUES ('add_rooms_room_version_column', '{}');

34
synapse/storage/data_stores/main/state.py
View File

@ -60,24 +60,34 @@ class StateGroupWorkerStore(EventsWorkerStore, SQLBaseStore):
def __init__(self, database: Database, db_conn, hs):
super(StateGroupWorkerStore, self).__init__(database, db_conn, hs)
@defer.inlineCallbacks
def get_room_version(self, room_id):
@cached(max_entries=10000)
async def get_room_version(self, room_id: str) -> str:
"""Get the room_version of a given room
Args:
room_id (str)
Returns:
Deferred[str]
Raises:
NotFoundError if the room is unknown
NotFoundError: if the room is unknown
"""
# for now we do this by looking at the create event. We may want to cache this
# more intelligently in future.
# First we try looking up room version from the database, but for old
# rooms we might not have added the room version to it yet so we fall
# back to previous behaviour and look in current state events.
# We really should have an entry in the rooms table for every room we
# care about, but let's be a bit paranoid (at least while the background
# update is happening) to avoid breaking existing rooms.
version = await self.db.simple_select_one_onecol(
table="rooms",
keyvalues={"room_id": room_id},
retcol="room_version",
desc="get_room_version",
allow_none=True,
)
if version is not None:
return version
# Retrieve the room's create event
create_event = yield self.get_create_event_for_room(room_id)
create_event = await self.get_create_event_for_room(room_id)
return create_event.content.get("room_version", "1")
@defer.inlineCallbacks

7
tests/storage/test_room.py
View File

@ -17,6 +17,7 @@
from twisted.internet import defer
from synapse.api.constants import EventTypes
from synapse.api.room_versions import RoomVersions
from synapse.types import RoomAlias, RoomID, UserID
from tests import unittest
@ -40,6 +41,7 @@ class RoomStoreTestCase(unittest.TestCase):
self.room.to_string(),
room_creator_user_id=self.u_creator.to_string(),
is_public=True,
room_version=RoomVersions.V1,
)
@defer.inlineCallbacks
@ -68,7 +70,10 @@ class RoomEventsStoreTestCase(unittest.TestCase):
self.room = RoomID.from_string("!abcde:test")
yield self.store.store_room(
self.room.to_string(), room_creator_user_id="@creator:text", is_public=True
self.room.to_string(),
room_creator_user_id="@creator:text",
is_public=True,
room_version=RoomVersions.V1,
)
@defer.inlineCallbacks

5
tests/storage/test_state.py
View File

@ -45,7 +45,10 @@ class StateStoreTestCase(tests.unittest.TestCase):
self.room = RoomID.from_string("!abc123:test")
yield self.store.store_room(
self.room.to_string(), room_creator_user_id="@creator:text", is_public=True
self.room.to_string(),
room_creator_user_id="@creator:text",
is_public=True,
room_version=RoomVersions.V1,
)
@defer.inlineCallbacks

8
tests/utils.py
View File

@ -639,9 +639,17 @@ def create_room(hs, room_id, creator_id):
"""
persistence_store = hs.get_storage().persistence
store = hs.get_datastore()
event_builder_factory = hs.get_event_builder_factory()
event_creation_handler = hs.get_event_creation_handler()
yield store.store_room(
room_id=room_id,
room_creator_user_id=creator_id,
is_public=False,
room_version=RoomVersions.V1,
)
builder = event_builder_factory.for_room_version(
RoomVersions.V1,
{