OpenCloud
/
MatrixSynapse
mirror of https://github.com/matrix-org/synapse
1
0
Fork 0
Code Issues Releases Wiki Activity

Replace `room_depth.min_depth` with a BIGINT (#10289) 

while I'm dealing with INTEGERs and BIGINTs, let's replace room_depth.min_depth
with a BIGINT.
pull/10391/head
Richard van der Hoff 2021-07-12 17:22:54 +01:00 committed by GitHub
parent 19d0401c56
commit c2c364f27f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 186 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
changelog.d/10289.misc Normal file
View File

@ -0,0 +1 @@
Convert `room_depth.min_depth` column to a `BIGINT`.

104
synapse/storage/databases/main/room.py
View File

@ -25,6 +25,7 @@ from synapse.api.room_versions import RoomVersion, RoomVersions
from synapse.storage._base import SQLBaseStore, db_to_json
from synapse.storage.database import DatabasePool, LoggingTransaction
from synapse.storage.databases.main.search import SearchStore
from synapse.storage.types import Cursor
from synapse.types import JsonDict, ThirdPartyInstanceID
from synapse.util import json_encoder
from synapse.util.caches.descriptors import cached
@ -1022,10 +1023,22 @@ class RoomWorkerStore(SQLBaseStore):
)
class RoomBackgroundUpdateStore(SQLBaseStore):
class _BackgroundUpdates:
REMOVE_TOMESTONED_ROOMS_BG_UPDATE = "remove_tombstoned_rooms_from_directory"
ADD_ROOMS_ROOM_VERSION_COLUMN = "add_rooms_room_version_column"
POPULATE_ROOM_DEPTH_MIN_DEPTH2 = "populate_room_depth_min_depth2"
REPLACE_ROOM_DEPTH_MIN_DEPTH = "replace_room_depth_min_depth"
_REPLACE_ROOM_DEPTH_SQL_COMMANDS = (
"DROP TRIGGER populate_min_depth2_trigger ON room_depth",
"DROP FUNCTION populate_min_depth2()",
"ALTER TABLE room_depth DROP COLUMN min_depth",
"ALTER TABLE room_depth RENAME COLUMN min_depth2 TO min_depth",
)
class RoomBackgroundUpdateStore(SQLBaseStore):
def __init__(self, database: DatabasePool, db_conn, hs):
super().__init__(database, db_conn, hs)
@ -1037,15 +1050,25 @@ class RoomBackgroundUpdateStore(SQLBaseStore):
)
self.db_pool.updates.register_background_update_handler(
self.REMOVE_TOMESTONED_ROOMS_BG_UPDATE,
_BackgroundUpdates.REMOVE_TOMESTONED_ROOMS_BG_UPDATE,
self._remove_tombstoned_rooms_from_directory,
)
self.db_pool.updates.register_background_update_handler(
self.ADD_ROOMS_ROOM_VERSION_COLUMN,
_BackgroundUpdates.ADD_ROOMS_ROOM_VERSION_COLUMN,
self._background_add_rooms_room_version_column,
)
# BG updates to change the type of room_depth.min_depth
self.db_pool.updates.register_background_update_handler(
_BackgroundUpdates.POPULATE_ROOM_DEPTH_MIN_DEPTH2,
self._background_populate_room_depth_min_depth2,
)
self.db_pool.updates.register_background_update_handler(
_BackgroundUpdates.REPLACE_ROOM_DEPTH_MIN_DEPTH,
self._background_replace_room_depth_min_depth,
)
async def _background_insert_retention(self, progress, batch_size):
"""Retrieves a list of all rooms within a range and inserts an entry for each of
them into the room_retention table.
@ -1164,7 +1187,9 @@ class RoomBackgroundUpdateStore(SQLBaseStore):
new_last_room_id = room_id
self.db_pool.updates._background_update_progress_txn(
txn, self.ADD_ROOMS_ROOM_VERSION_COLUMN, {"room_id": new_last_room_id}
txn,
_BackgroundUpdates.ADD_ROOMS_ROOM_VERSION_COLUMN,
{"room_id": new_last_room_id},
)
return False
@ -1176,7 +1201,7 @@ class RoomBackgroundUpdateStore(SQLBaseStore):
if end:
await self.db_pool.updates._end_background_update(
self.ADD_ROOMS_ROOM_VERSION_COLUMN
_BackgroundUpdates.ADD_ROOMS_ROOM_VERSION_COLUMN
)
return batch_size
@ -1215,7 +1240,7 @@ class RoomBackgroundUpdateStore(SQLBaseStore):
if not rooms:
await self.db_pool.updates._end_background_update(
self.REMOVE_TOMESTONED_ROOMS_BG_UPDATE
_BackgroundUpdates.REMOVE_TOMESTONED_ROOMS_BG_UPDATE
)
return 0
@ -1224,7 +1249,7 @@ class RoomBackgroundUpdateStore(SQLBaseStore):
await self.set_room_is_public(room_id, False)
await self.db_pool.updates._background_update_progress(
self.REMOVE_TOMESTONED_ROOMS_BG_UPDATE, {"room_id": rooms[-1]}
_BackgroundUpdates.REMOVE_TOMESTONED_ROOMS_BG_UPDATE, {"room_id": rooms[-1]}
)
return len(rooms)
@ -1268,6 +1293,71 @@ class RoomBackgroundUpdateStore(SQLBaseStore):
return max_ordering is None
async def _background_populate_room_depth_min_depth2(
self, progress: JsonDict, batch_size: int
) -> int:
"""Populate room_depth.min_depth2
This is to deal with the fact that min_depth was initially created as a
32-bit integer field.
"""
def process(txn: Cursor) -> int:
last_room = progress.get("last_room", "")
txn.execute(
"""
UPDATE room_depth SET min_depth2=min_depth
WHERE room_id IN (
SELECT room_id FROM room_depth WHERE room_id > ?
ORDER BY room_id LIMIT ?
)
RETURNING room_id;
""",
(last_room, batch_size),
)
row_count = txn.rowcount
if row_count == 0:
return 0
last_room = max(row[0] for row in txn)
logger.info("populated room_depth up to %s", last_room)
self.db_pool.updates._background_update_progress_txn(
txn,
_BackgroundUpdates.POPULATE_ROOM_DEPTH_MIN_DEPTH2,
{"last_room": last_room},
)
return row_count
result = await self.db_pool.runInteraction(
"_background_populate_min_depth2", process
)
if result != 0:
return result
await self.db_pool.updates._end_background_update(
_BackgroundUpdates.POPULATE_ROOM_DEPTH_MIN_DEPTH2
)
return 0
async def _background_replace_room_depth_min_depth(
self, progress: JsonDict, batch_size: int
) -> int:
"""Drop the old 'min_depth' column and rename 'min_depth2' into its place."""
def process(txn: Cursor) -> None:
for sql in _REPLACE_ROOM_DEPTH_SQL_COMMANDS:
logger.info("completing room_depth migration: %s", sql)
txn.execute(sql)
await self.db_pool.runInteraction("_background_replace_room_depth", process)
await self.db_pool.updates._end_background_update(
_BackgroundUpdates.REPLACE_ROOM_DEPTH_MIN_DEPTH,
)
return 0
class RoomStore(RoomBackgroundUpdateStore, RoomWorkerStore, SearchStore):
def __init__(self, database: DatabasePool, db_conn, hs):

18
synapse/storage/schema/main/delta/61/02drop_redundant_room_depth_index.sql Normal file
View File

@ -0,0 +1,18 @@
/* Copyright 2021 The Matrix.org Foundation C.I.C
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-- this index is redundant; there is another UNIQUE index on this table.
DROP INDEX IF EXISTS room_depth_room;

70
synapse/storage/schema/main/delta/61/03recreate_min_depth.py Normal file
View File

@ -0,0 +1,70 @@
# Copyright 2021 The Matrix.org Foundation C.I.C.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
This migration handles the process of changing the type of `room_depth.min_depth` to
a BIGINT.
"""
from synapse.storage.engines import BaseDatabaseEngine, PostgresEngine
from synapse.storage.types import Cursor
def run_create(cur: Cursor, database_engine: BaseDatabaseEngine, *args, **kwargs):
if not isinstance(database_engine, PostgresEngine):
# this only applies to postgres - sqlite does not distinguish between big and
# little ints.
return
# First add a new column to contain the bigger min_depth
cur.execute("ALTER TABLE room_depth ADD COLUMN min_depth2 BIGINT")
# Create a trigger which will keep it populated.
cur.execute(
"""
CREATE OR REPLACE FUNCTION populate_min_depth2() RETURNS trigger AS $BODY$
BEGIN
new.min_depth2 := new.min_depth;
RETURN NEW;
END;
$BODY$ LANGUAGE plpgsql
"""
)
cur.execute(
"""
CREATE TRIGGER populate_min_depth2_trigger BEFORE INSERT OR UPDATE ON room_depth
FOR EACH ROW
EXECUTE PROCEDURE populate_min_depth2()
"""
)
# Start a bg process to populate it for old rooms
cur.execute(
"""
INSERT INTO background_updates (ordering, update_name, progress_json) VALUES
(6103, 'populate_room_depth_min_depth2', '{}')
"""
)
# and another to switch them over once it completes.
cur.execute(
"""
INSERT INTO background_updates (ordering, update_name, progress_json, depends_on) VALUES
(6103, 'replace_room_depth_min_depth', '{}', 'populate_room_depth2')
"""
)
def run_upgrade(cur: Cursor, database_engine: BaseDatabaseEngine, *args, **kwargs):
pass