From 11e15d79b8a0af593fd9467e0cc7f8a9dfcb6c4f Mon Sep 17 00:00:00 2001 From: reivilibre Date: Wed, 31 May 2023 13:59:56 +0000 Subject: [PATCH 1/4] Fix a performance issue introduced in Synapse v1.83.0 which meant that purging rooms was very slow and database-intensive. (#15693) * Add indices required to efficiently validate new foreign key constraints on stream_ordering * Newsfile Signed-off-by: Olivier Wilkinson (reivilibre) --------- Signed-off-by: Olivier Wilkinson (reivilibre) --- changelog.d/15693.bugfix | 1 + synapse/storage/databases/state/bg_updates.py | 31 +++++++++++++++++++ .../77/14bg_indices_event_stream_ordering.sql | 20 ++++++++++++ 3 files changed, 52 insertions(+) create mode 100644 changelog.d/15693.bugfix create mode 100644 synapse/storage/schema/main/delta/77/14bg_indices_event_stream_ordering.sql diff --git a/changelog.d/15693.bugfix b/changelog.d/15693.bugfix new file mode 100644 index 0000000000..d0325de007 --- /dev/null +++ b/changelog.d/15693.bugfix @@ -0,0 +1 @@ +Fix a performance issue introduced in Synapse v1.83.0 which meant that purging rooms was very slow and database-intensive. \ No newline at end of file diff --git a/synapse/storage/databases/state/bg_updates.py b/synapse/storage/databases/state/bg_updates.py index 86eb1a8a08..5b8ba436d4 100644 --- a/synapse/storage/databases/state/bg_updates.py +++ b/synapse/storage/databases/state/bg_updates.py @@ -261,6 +261,16 @@ class StateBackgroundUpdateStore(StateGroupBackgroundUpdateStore): STATE_GROUPS_ROOM_INDEX_UPDATE_NAME = "state_groups_room_id_idx" STATE_GROUP_EDGES_UNIQUE_INDEX_UPDATE_NAME = "state_group_edges_unique_idx" + CURRENT_STATE_EVENTS_STREAM_ORDERING_INDEX_UPDATE_NAME = ( + "current_state_events_stream_ordering_idx" + ) + ROOM_MEMBERSHIPS_STREAM_ORDERING_INDEX_UPDATE_NAME = ( + "room_memberships_stream_ordering_idx" + ) + LOCAL_CURRENT_MEMBERSHIP_STREAM_ORDERING_INDEX_UPDATE_NAME = ( + "local_current_membership_stream_ordering_idx" + ) + def __init__( self, database: DatabasePool, @@ -297,6 +307,27 @@ class StateBackgroundUpdateStore(StateGroupBackgroundUpdateStore): replaces_index="state_group_edges_idx", ) + # These indices are needed to validate the foreign key constraint + # when events are deleted. + self.db_pool.updates.register_background_index_update( + self.CURRENT_STATE_EVENTS_STREAM_ORDERING_INDEX_UPDATE_NAME, + index_name="current_state_events_stream_ordering_idx", + table="current_state_events", + columns=["event_stream_ordering"], + ) + self.db_pool.updates.register_background_index_update( + self.ROOM_MEMBERSHIPS_STREAM_ORDERING_INDEX_UPDATE_NAME, + index_name="room_memberships_stream_ordering_idx", + table="room_memberships", + columns=["event_stream_ordering"], + ) + self.db_pool.updates.register_background_index_update( + self.LOCAL_CURRENT_MEMBERSHIP_STREAM_ORDERING_INDEX_UPDATE_NAME, + index_name="local_current_membership_stream_ordering_idx", + table="local_current_membership", + columns=["event_stream_ordering"], + ) + async def _background_deduplicate_state( self, progress: dict, batch_size: int ) -> int: diff --git a/synapse/storage/schema/main/delta/77/14bg_indices_event_stream_ordering.sql b/synapse/storage/schema/main/delta/77/14bg_indices_event_stream_ordering.sql new file mode 100644 index 0000000000..ec8cd522ec --- /dev/null +++ b/synapse/storage/schema/main/delta/77/14bg_indices_event_stream_ordering.sql @@ -0,0 +1,20 @@ +/* Copyright 2023 The Matrix.org Foundation C.I.C. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +INSERT INTO background_updates (ordering, update_name, progress_json) + VALUES + (7714, 'current_state_events_stream_ordering_idx', '{}'), + (7714, 'local_current_membership_stream_ordering_idx', '{}'), + (7714, 'room_memberships_stream_ordering_idx', '{}'); From 6d9e2fd8782a6610d6daf499d141e67f476b2f8c Mon Sep 17 00:00:00 2001 From: Shay Date: Wed, 31 May 2023 15:13:48 -0700 Subject: [PATCH 2/4] Speed up background jobs populate_full_user_id_user_filters and populate_full_user_id_profiles (#15700) --- changelog.d/15700.misc | 1 + synapse/storage/databases/main/filtering.py | 2 +- synapse/storage/databases/main/profile.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 changelog.d/15700.misc diff --git a/changelog.d/15700.misc b/changelog.d/15700.misc new file mode 100644 index 0000000000..e96bc681aa --- /dev/null +++ b/changelog.d/15700.misc @@ -0,0 +1 @@ +Speed up background jobs `populate_full_user_id_user_filters` and `populate_full_user_id_profiles`. \ No newline at end of file diff --git a/synapse/storage/databases/main/filtering.py b/synapse/storage/databases/main/filtering.py index da31eb44dc..f777777cbf 100644 --- a/synapse/storage/databases/main/filtering.py +++ b/synapse/storage/databases/main/filtering.py @@ -71,7 +71,7 @@ class FilteringWorkerStore(SQLBaseStore): SELECT user_id FROM user_filters WHERE user_id > ? ORDER BY user_id - LIMIT 1 OFFSET 50 + LIMIT 1 OFFSET 1000 """ txn.execute(sql, (lower_bound_id,)) res = txn.fetchone() diff --git a/synapse/storage/databases/main/profile.py b/synapse/storage/databases/main/profile.py index 65c92bef51..21d54c7a7a 100644 --- a/synapse/storage/databases/main/profile.py +++ b/synapse/storage/databases/main/profile.py @@ -65,7 +65,7 @@ class ProfileWorkerStore(SQLBaseStore): SELECT user_id FROM profiles WHERE user_id > ? ORDER BY user_id - LIMIT 1 OFFSET 50 + LIMIT 1 OFFSET 1000 """ txn.execute(sql, (lower_bound_id,)) res = txn.fetchone() From a273561c2247ee433f97a31961a30ab00ab19574 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Thu, 1 Jun 2023 08:21:37 -0400 Subject: [PATCH 3/4] Add a note about deprecating /register with a user property. (#15703) Application services providing a "user" property (instead of "username") for the /register endpoint was never specified. Deprecate this very old fallback. --- changelog.d/15703.removal | 1 + docs/upgrade.md | 11 +++++++++++ 2 files changed, 12 insertions(+) create mode 100644 changelog.d/15703.removal diff --git a/changelog.d/15703.removal b/changelog.d/15703.removal new file mode 100644 index 0000000000..95a2d8e484 --- /dev/null +++ b/changelog.d/15703.removal @@ -0,0 +1 @@ +Deprecate calling the `/register` endpoint with an unspecced `user` property for application services. diff --git a/docs/upgrade.md b/docs/upgrade.md index af999dd91f..49ab00c057 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -88,6 +88,17 @@ process, for example: dpkg -i matrix-synapse-py3_1.3.0+stretch1_amd64.deb ``` +# Upgrading to v1.85.0 + +## Application service registration with "user" property deprecation + +Application services should ensure they call the `/register` endpoint with a +`username` property. The legacy `user` property is considered deprecated and +should no longer be included. + +A future version of Synapse (v1.88.0 or later) will remove support for legacy +application service login. + # Upgrading to v1.84.0 ## Deprecation of `worker_replication_*` configuration settings From 4c0bffaca5ded573cc26d99bd5831f136f8acacc Mon Sep 17 00:00:00 2001 From: "H. Shay" Date: Thu, 1 Jun 2023 09:16:35 -0700 Subject: [PATCH 4/4] 1.85.0rc2 --- CHANGES.md | 21 +++++++++++++++++++++ changelog.d/15693.bugfix | 1 - changelog.d/15700.misc | 1 - changelog.d/15703.removal | 1 - debian/changelog | 6 ++++++ pyproject.toml | 2 +- 6 files changed, 28 insertions(+), 4 deletions(-) delete mode 100644 changelog.d/15693.bugfix delete mode 100644 changelog.d/15700.misc delete mode 100644 changelog.d/15703.removal diff --git a/CHANGES.md b/CHANGES.md index 14aac9f14e..f0885a2f1e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,24 @@ +Synapse 1.85.0rc2 (2023-06-01) +============================== + +Bugfixes +-------- + +- Fix a performance issue introduced in Synapse v1.83.0 which meant that purging rooms was very slow and database-intensive. ([\#15693](https://github.com/matrix-org/synapse/issues/15693)) + + +Deprecations and Removals +------------------------- + +- Deprecate calling the `/register` endpoint with an unspecced `user` property for application services. ([\#15703](https://github.com/matrix-org/synapse/issues/15703)) + + +Internal Changes +---------------- + +- Speed up background jobs `populate_full_user_id_user_filters` and `populate_full_user_id_profiles`. ([\#15700](https://github.com/matrix-org/synapse/issues/15700)) + + Synapse 1.85.0rc1 (2023-05-30) ============================== diff --git a/changelog.d/15693.bugfix b/changelog.d/15693.bugfix deleted file mode 100644 index d0325de007..0000000000 --- a/changelog.d/15693.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a performance issue introduced in Synapse v1.83.0 which meant that purging rooms was very slow and database-intensive. \ No newline at end of file diff --git a/changelog.d/15700.misc b/changelog.d/15700.misc deleted file mode 100644 index e96bc681aa..0000000000 --- a/changelog.d/15700.misc +++ /dev/null @@ -1 +0,0 @@ -Speed up background jobs `populate_full_user_id_user_filters` and `populate_full_user_id_profiles`. \ No newline at end of file diff --git a/changelog.d/15703.removal b/changelog.d/15703.removal deleted file mode 100644 index 95a2d8e484..0000000000 --- a/changelog.d/15703.removal +++ /dev/null @@ -1 +0,0 @@ -Deprecate calling the `/register` endpoint with an unspecced `user` property for application services. diff --git a/debian/changelog b/debian/changelog index 2d88cd9d29..ae348ce4df 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.85.0~rc2) stable; urgency=medium + + * New Synapse release 1.85.0rc2. + + -- Synapse Packaging team Thu, 01 Jun 2023 09:16:18 -0700 + matrix-synapse-py3 (1.85.0~rc1) stable; urgency=medium * New Synapse release 1.85.0rc1. diff --git a/pyproject.toml b/pyproject.toml index 7227bc7523..4ed4214f34 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -89,7 +89,7 @@ manifest-path = "rust/Cargo.toml" [tool.poetry] name = "matrix-synapse" -version = "1.85.0rc1" +version = "1.85.0rc2" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] license = "Apache-2.0"