From 99a4e5222d82cfcf8686b71d8fa94c8a81dfcbd6 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Thu, 14 Oct 2021 10:59:27 +0100 Subject: [PATCH 1/9] 1.45.0rc2 --- CHANGES.md | 21 +++++++++++++++++++++ changelog.d/10825.misc | 1 - changelog.d/10970.misc | 1 - changelog.d/10996.misc | 1 - changelog.d/11036.misc | 1 - changelog.d/11045.bugfix | 1 - changelog.d/11053.bugfix | 2 -- changelog.d/11061.bugfix | 1 - debian/changelog | 6 ++++++ synapse/__init__.py | 2 +- 10 files changed, 28 insertions(+), 9 deletions(-) delete mode 100644 changelog.d/10825.misc delete mode 100644 changelog.d/10970.misc delete mode 100644 changelog.d/10996.misc delete mode 100644 changelog.d/11036.misc delete mode 100644 changelog.d/11045.bugfix delete mode 100644 changelog.d/11053.bugfix delete mode 100644 changelog.d/11061.bugfix diff --git a/CHANGES.md b/CHANGES.md index 5acc8b537e..0d69830880 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,24 @@ +Synapse 1.45.0rc2 (2021-10-14) +============================== + +Bugfixes +-------- + +- Fix a long-standing bug when using multiple event persister workers where events were not correctly sent down `/sync` due to a race. ([\#11045](https://github.com/matrix-org/synapse/issues/11045)) +- Fix a bug introduced in Synapse 1.45.0rc1 where the user directory would stop updating if it processed an event from a + user not in the `users` table. ([\#11053](https://github.com/matrix-org/synapse/issues/11053)) +- Fix a bug introduced in Synapse v1.44.0 when logging errors during oEmbed processing. ([\#11061](https://github.com/matrix-org/synapse/issues/11061)) + + +Internal Changes +---------------- + +- Add an 'approximate difference' method to `StateFilter`. ([\#10825](https://github.com/matrix-org/synapse/issues/10825)) +- Fix inconsistent behavior of `get_last_client_by_ip` when reporting data that has not been stored in the database yet. ([\#10970](https://github.com/matrix-org/synapse/issues/10970)) +- Fix a bug introduced in Synapse 1.21.0 that causes opentracing and Prometheus metrics for replication requests to be measured incorrectly. ([\#10996](https://github.com/matrix-org/synapse/issues/10996)) +- Ensure that cache config tests do not share state. ([\#11036](https://github.com/matrix-org/synapse/issues/11036)) + + Synapse 1.45.0rc1 (2021-10-12) ============================== diff --git a/changelog.d/10825.misc b/changelog.d/10825.misc deleted file mode 100644 index f9786164d7..0000000000 --- a/changelog.d/10825.misc +++ /dev/null @@ -1 +0,0 @@ -Add an 'approximate difference' method to `StateFilter`. diff --git a/changelog.d/10970.misc b/changelog.d/10970.misc deleted file mode 100644 index bb75ea79a6..0000000000 --- a/changelog.d/10970.misc +++ /dev/null @@ -1 +0,0 @@ -Fix inconsistent behavior of `get_last_client_by_ip` when reporting data that has not been stored in the database yet. diff --git a/changelog.d/10996.misc b/changelog.d/10996.misc deleted file mode 100644 index c830d7ec2c..0000000000 --- a/changelog.d/10996.misc +++ /dev/null @@ -1 +0,0 @@ -Fix a bug introduced in Synapse 1.21.0 that causes opentracing and Prometheus metrics for replication requests to be measured incorrectly. diff --git a/changelog.d/11036.misc b/changelog.d/11036.misc deleted file mode 100644 index aae5ee62b2..0000000000 --- a/changelog.d/11036.misc +++ /dev/null @@ -1 +0,0 @@ -Ensure that cache config tests do not share state. diff --git a/changelog.d/11045.bugfix b/changelog.d/11045.bugfix deleted file mode 100644 index d712dc946a..0000000000 --- a/changelog.d/11045.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a long-standing bug when using multiple event persister workers where events were not correctly sent down `/sync` due to a race. diff --git a/changelog.d/11053.bugfix b/changelog.d/11053.bugfix deleted file mode 100644 index a59cfac931..0000000000 --- a/changelog.d/11053.bugfix +++ /dev/null @@ -1,2 +0,0 @@ -Fix a bug introduced in Synapse 1.45.0rc1 where the user directory would stop updating if it processed an event from a -user not in the `users` table. diff --git a/changelog.d/11061.bugfix b/changelog.d/11061.bugfix deleted file mode 100644 index 26fb643793..0000000000 --- a/changelog.d/11061.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug introduced in Synapse v1.44.0 when logging errors during oEmbed processing. diff --git a/debian/changelog b/debian/changelog index 0d5db739e7..f930befbe7 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.45.0~rc2) stable; urgency=medium + + * New synapse release 1.45.0~rc2. + + -- Synapse Packaging team Thu, 14 Oct 2021 10:58:24 +0100 + matrix-synapse-py3 (1.45.0~rc1) stable; urgency=medium [ Nick @ Beeper ] diff --git a/synapse/__init__.py b/synapse/__init__.py index 6b109ccffa..ab90093e70 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -47,7 +47,7 @@ try: except ImportError: pass -__version__ = "1.45.0rc1" +__version__ = "1.45.0rc2" if bool(os.environ.get("SYNAPSE_TEST_PATCH_LOG_CONTEXTS", False)): # We import here so that we don't have to install a bunch of deps when From 87c3a6dcc08e158d44f36b96bbc9311a9edb99e1 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Thu, 14 Oct 2021 11:03:35 +0100 Subject: [PATCH 2/9] Refer to the bugs mentioned in 1.45.0rc1 note --- CHANGES.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 0d69830880..7ac7e794dc 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,8 @@ Synapse 1.45.0rc2 (2021-10-14) ============================== +**Note:** This release candidate [fixes](https://github.com/matrix-org/synapse/issues/11053) the user directory [bug](https://github.com/matrix-org/synapse/issues/11025) present in 1.45.0rc1. However, the [performance issue](https://github.com/matrix-org/synapse/issues/11049) mentioned in that release is yet to be resolved. + Bugfixes -------- From 4d761d24badb5a05abe9adeb46515553ca007b9d Mon Sep 17 00:00:00 2001 From: David Robertson Date: Thu, 14 Oct 2021 11:18:40 +0100 Subject: [PATCH 3/9] mentioned -> which appeared --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 7ac7e794dc..60f51005e8 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,7 +1,7 @@ Synapse 1.45.0rc2 (2021-10-14) ============================== -**Note:** This release candidate [fixes](https://github.com/matrix-org/synapse/issues/11053) the user directory [bug](https://github.com/matrix-org/synapse/issues/11025) present in 1.45.0rc1. However, the [performance issue](https://github.com/matrix-org/synapse/issues/11049) mentioned in that release is yet to be resolved. +**Note:** This release candidate [fixes](https://github.com/matrix-org/synapse/issues/11053) the user directory [bug](https://github.com/matrix-org/synapse/issues/11025) present in 1.45.0rc1. However, the [performance issue](https://github.com/matrix-org/synapse/issues/11049) which appeared in that release is yet to be resolved. Bugfixes -------- From b1c1a34f4680f89e5de506444155081c380dae97 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Thu, 14 Oct 2021 11:20:02 +0100 Subject: [PATCH 4/9] it appeared in 1.44, not 45rc1 --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 60f51005e8..c8b078a2f4 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,7 +1,7 @@ Synapse 1.45.0rc2 (2021-10-14) ============================== -**Note:** This release candidate [fixes](https://github.com/matrix-org/synapse/issues/11053) the user directory [bug](https://github.com/matrix-org/synapse/issues/11025) present in 1.45.0rc1. However, the [performance issue](https://github.com/matrix-org/synapse/issues/11049) which appeared in that release is yet to be resolved. +**Note:** This release candidate [fixes](https://github.com/matrix-org/synapse/issues/11053) the user directory [bug](https://github.com/matrix-org/synapse/issues/11025) present in 1.45.0rc1. However, the [performance issue](https://github.com/matrix-org/synapse/issues/11049) which appeared in v1.44.0 is yet to be resolved. Bugfixes -------- From 191396f4baf467023b9913f83a1e3a696aa0e3a0 Mon Sep 17 00:00:00 2001 From: Dan Callahan Date: Tue, 19 Oct 2021 11:16:52 +0100 Subject: [PATCH 5/9] Reword changelog regarding a suspected regression (#11117) Signed-off-by: Dan Callahan --- CHANGES.md | 17 ++++++++++++++--- changelog.d/11117.doc | 1 + 2 files changed, 15 insertions(+), 3 deletions(-) create mode 100644 changelog.d/11117.doc diff --git a/CHANGES.md b/CHANGES.md index c8b078a2f4..6a4d5f5c97 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,7 +1,14 @@ Synapse 1.45.0rc2 (2021-10-14) ============================== -**Note:** This release candidate [fixes](https://github.com/matrix-org/synapse/issues/11053) the user directory [bug](https://github.com/matrix-org/synapse/issues/11025) present in 1.45.0rc1. However, the [performance issue](https://github.com/matrix-org/synapse/issues/11049) which appeared in v1.44.0 is yet to be resolved. +This release candidate [fixes](https://github.com/matrix-org/synapse/issues/11053) a user directory [bug](https://github.com/matrix-org/synapse/issues/11025) present in 1.45.0rc1. + +Known Issues +------------ + +- A suspected [performance regression](https://github.com/matrix-org/synapse/issues/11049) which was first reported after the release of v1.44.0 remains unresolved. + + We have not been able to identify a probable cause. Affected users report that setting up a federation sender worker appears to alleviate symptoms of the regression. Bugfixes -------- @@ -24,9 +31,13 @@ Internal Changes Synapse 1.45.0rc1 (2021-10-12) ============================== -**Note:** We are aware of [a performance issue](https://github.com/matrix-org/synapse/issues/11049) introduced in Synapse v1.44.0, as well as [a bug](https://github.com/matrix-org/synapse/issues/11025) with the user directory when using application services. While this release candidate doesn't fix either of those issues, a second release candidate is expected to come out in a few days to address them. +**Note:** Media storage providers module that read from Synapse's configuration need changes as of this version, see the [upgrade notes](https://matrix-org.github.io/synapse/develop/upgrade#upgrading-to-v1450) for more information. -Media storage providers module that read from Synapse's configuration need changes as of this version, see the [upgrade notes](https://matrix-org.github.io/synapse/develop/upgrade#upgrading-to-v1450) for more information. +Known Issues +------------ + +- We are investigating [a performance issue](https://github.com/matrix-org/synapse/issues/11049) which was reported after the release of v1.44.0. +- We are aware of [a bug](https://github.com/matrix-org/synapse/issues/11025) with the user directory when using application services. A second release candidate is expected which will resolve this. Features -------- diff --git a/changelog.d/11117.doc b/changelog.d/11117.doc new file mode 100644 index 0000000000..b4809fd6ea --- /dev/null +++ b/changelog.d/11117.doc @@ -0,0 +1 @@ +Reword changelog to clarify concerns about a suspected performance regression in 1.44. From 8b1185347a3bfa4530500fe274ffc9006ec027ea Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 19 Oct 2021 11:19:55 +0100 Subject: [PATCH 6/9] 1.45.0 --- CHANGES.md | 11 +++++++++++ changelog.d/11117.doc | 1 - debian/changelog | 6 ++++++ synapse/__init__.py | 2 +- 4 files changed, 18 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/11117.doc diff --git a/CHANGES.md b/CHANGES.md index 6a4d5f5c97..7d1b804e8b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,14 @@ +Synapse 1.45.0 (2021-10-19) +=========================== + +No functional changes since Synapse 1.45.0rc2. + +Improved Documentation +---------------------- + +- Reword changelog to clarify concerns about a suspected performance regression in 1.44. ([\#11117](https://github.com/matrix-org/synapse/issues/11117)) + + Synapse 1.45.0rc2 (2021-10-14) ============================== diff --git a/changelog.d/11117.doc b/changelog.d/11117.doc deleted file mode 100644 index b4809fd6ea..0000000000 --- a/changelog.d/11117.doc +++ /dev/null @@ -1 +0,0 @@ -Reword changelog to clarify concerns about a suspected performance regression in 1.44. diff --git a/debian/changelog b/debian/changelog index f930befbe7..5fefb2f2ac 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.45.0) stable; urgency=medium + + * New synapse release 1.45.0. + + -- Synapse Packaging team Tue, 19 Oct 2021 11:18:53 +0100 + matrix-synapse-py3 (1.45.0~rc2) stable; urgency=medium * New synapse release 1.45.0~rc2. diff --git a/synapse/__init__.py b/synapse/__init__.py index ab90093e70..97452f34fe 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -47,7 +47,7 @@ try: except ImportError: pass -__version__ = "1.45.0rc2" +__version__ = "1.45.0" if bool(os.environ.get("SYNAPSE_TEST_PATCH_LOG_CONTEXTS", False)): # We import here so that we don't have to install a bunch of deps when From a21f8c4b41adc323adf4a8ac517ff591856a74fb Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 19 Oct 2021 11:21:21 +0100 Subject: [PATCH 7/9] Duplicate known issues under 1.45 release --- CHANGES.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 7d1b804e8b..e36b1acf8c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -3,6 +3,13 @@ Synapse 1.45.0 (2021-10-19) No functional changes since Synapse 1.45.0rc2. +Known Issues +------------ + +- A suspected [performance regression](https://github.com/matrix-org/synapse/issues/11049) which was first reported after the release of v1.44.0 remains unresolved. + + We have not been able to identify a probable cause. Affected users report that setting up a federation sender worker appears to alleviate symptoms of the regression. + Improved Documentation ---------------------- From 95813ff43cc08f2064049050a67e027751d9b091 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 19 Oct 2021 11:30:16 +0100 Subject: [PATCH 8/9] Be less inconsistent about v1.2.3 versus 1.2.3 --- CHANGES.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index e36b1acf8c..435387d7b0 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,14 +6,14 @@ No functional changes since Synapse 1.45.0rc2. Known Issues ------------ -- A suspected [performance regression](https://github.com/matrix-org/synapse/issues/11049) which was first reported after the release of v1.44.0 remains unresolved. +- A suspected [performance regression](https://github.com/matrix-org/synapse/issues/11049) which was first reported after the release of 1.44.0 remains unresolved. We have not been able to identify a probable cause. Affected users report that setting up a federation sender worker appears to alleviate symptoms of the regression. Improved Documentation ---------------------- -- Reword changelog to clarify concerns about a suspected performance regression in 1.44. ([\#11117](https://github.com/matrix-org/synapse/issues/11117)) +- Reword changelog to clarify concerns about a suspected performance regression in 1.44.0. ([\#11117](https://github.com/matrix-org/synapse/issues/11117)) Synapse 1.45.0rc2 (2021-10-14) @@ -24,7 +24,7 @@ This release candidate [fixes](https://github.com/matrix-org/synapse/issues/1105 Known Issues ------------ -- A suspected [performance regression](https://github.com/matrix-org/synapse/issues/11049) which was first reported after the release of v1.44.0 remains unresolved. +- A suspected [performance regression](https://github.com/matrix-org/synapse/issues/11049) which was first reported after the release of 1.44.0 remains unresolved. We have not been able to identify a probable cause. Affected users report that setting up a federation sender worker appears to alleviate symptoms of the regression. @@ -34,7 +34,7 @@ Bugfixes - Fix a long-standing bug when using multiple event persister workers where events were not correctly sent down `/sync` due to a race. ([\#11045](https://github.com/matrix-org/synapse/issues/11045)) - Fix a bug introduced in Synapse 1.45.0rc1 where the user directory would stop updating if it processed an event from a user not in the `users` table. ([\#11053](https://github.com/matrix-org/synapse/issues/11053)) -- Fix a bug introduced in Synapse v1.44.0 when logging errors during oEmbed processing. ([\#11061](https://github.com/matrix-org/synapse/issues/11061)) +- Fix a bug introduced in Synapse 1.44.0 when logging errors during oEmbed processing. ([\#11061](https://github.com/matrix-org/synapse/issues/11061)) Internal Changes @@ -54,7 +54,7 @@ Synapse 1.45.0rc1 (2021-10-12) Known Issues ------------ -- We are investigating [a performance issue](https://github.com/matrix-org/synapse/issues/11049) which was reported after the release of v1.44.0. +- We are investigating [a performance issue](https://github.com/matrix-org/synapse/issues/11049) which was reported after the release of 1.44.0. - We are aware of [a bug](https://github.com/matrix-org/synapse/issues/11025) with the user directory when using application services. A second release candidate is expected which will resolve this. Features From df95d3aec216f3ee73b162677ca2fca9c30289b4 Mon Sep 17 00:00:00 2001 From: Sean Quah <8349537+squahtx@users.noreply.github.com> Date: Wed, 20 Oct 2021 11:05:29 +0100 Subject: [PATCH 9/9] Revert change to counting of deactivated users towards the monthly active users limit (#11127) Temporarily revert "Add functionality to remove deactivated users from the monthly_active_users table (#10947)". This reverts commit eda8c88b84ee7506379a71ac2a7a88c08b759d43. --- changelog.d/11127.bugfix | 1 + synapse/handlers/deactivate_account.py | 4 -- .../databases/main/monthly_active_users.py | 24 ------------ tests/test_mau.py | 37 ++----------------- 4 files changed, 4 insertions(+), 62 deletions(-) create mode 100644 changelog.d/11127.bugfix diff --git a/changelog.d/11127.bugfix b/changelog.d/11127.bugfix new file mode 100644 index 0000000000..54417a9975 --- /dev/null +++ b/changelog.d/11127.bugfix @@ -0,0 +1 @@ +Revert change to counting of deactivated users towards the monthly active users limit ([\#10947](https://github.com/matrix-org/synapse/issues/10947)). diff --git a/synapse/handlers/deactivate_account.py b/synapse/handlers/deactivate_account.py index e88c3c27ce..bee62cf360 100644 --- a/synapse/handlers/deactivate_account.py +++ b/synapse/handlers/deactivate_account.py @@ -131,10 +131,6 @@ class DeactivateAccountHandler: # delete from user directory await self.user_directory_handler.handle_local_user_deactivated(user_id) - # If the user is present in the monthly active users table - # remove them - await self.store.remove_deactivated_user_from_mau_table(user_id) - # Mark the user as erased, if they asked for that if erase_data: user = UserID.from_string(user_id) diff --git a/synapse/storage/databases/main/monthly_active_users.py b/synapse/storage/databases/main/monthly_active_users.py index ec4d47a560..a14ac03d4b 100644 --- a/synapse/storage/databases/main/monthly_active_users.py +++ b/synapse/storage/databases/main/monthly_active_users.py @@ -354,27 +354,3 @@ class MonthlyActiveUsersStore(MonthlyActiveUsersWorkerStore): await self.upsert_monthly_active_user(user_id) elif now - last_seen_timestamp > LAST_SEEN_GRANULARITY: await self.upsert_monthly_active_user(user_id) - - async def remove_deactivated_user_from_mau_table(self, user_id: str) -> None: - """ - Removes a deactivated user from the monthly active user - table and resets affected caches. - - Args: - user_id(str): the user_id to remove - """ - - rows_deleted = await self.db_pool.simple_delete( - table="monthly_active_users", - keyvalues={"user_id": user_id}, - desc="simple_delete", - ) - - if rows_deleted != 0: - await self.invalidate_cache_and_stream( - "user_last_seen_monthly_active", (user_id,) - ) - await self.invalidate_cache_and_stream("get_monthly_active_count", ()) - await self.invalidate_cache_and_stream( - "get_monthly_active_count_by_service", () - ) diff --git a/tests/test_mau.py b/tests/test_mau.py index c683c8937e..80ab40e255 100644 --- a/tests/test_mau.py +++ b/tests/test_mau.py @@ -13,11 +13,11 @@ # limitations under the License. """Tests REST events for /rooms paths.""" -import synapse.rest.admin + from synapse.api.constants import APP_SERVICE_REGISTRATION_TYPE, LoginType from synapse.api.errors import Codes, HttpResponseException, SynapseError from synapse.appservice import ApplicationService -from synapse.rest.client import login, profile, register, sync +from synapse.rest.client import register, sync from tests import unittest from tests.unittest import override_config @@ -26,13 +26,7 @@ from tests.utils import default_config class TestMauLimit(unittest.HomeserverTestCase): - servlets = [ - register.register_servlets, - sync.register_servlets, - synapse.rest.admin.register_servlets_for_client_rest_resource, - profile.register_servlets, - login.register_servlets, - ] + servlets = [register.register_servlets, sync.register_servlets] def default_config(self): config = default_config("test") @@ -235,31 +229,6 @@ class TestMauLimit(unittest.HomeserverTestCase): self.reactor.advance(100) self.assertEqual(2, self.successResultOf(count)) - def test_deactivated_users_dont_count_towards_mau(self): - user1 = self.register_user("madonna", "password") - self.register_user("prince", "password2") - self.register_user("frodo", "onering", True) - - token1 = self.login("madonna", "password") - token2 = self.login("prince", "password2") - admin_token = self.login("frodo", "onering") - - self.do_sync_for_user(token1) - self.do_sync_for_user(token2) - - # Check that mau count is what we expect - count = self.get_success(self.store.get_monthly_active_count()) - self.assertEqual(count, 2) - - # Deactivate user1 - url = "/_synapse/admin/v1/deactivate/%s" % user1 - channel = self.make_request("POST", url, access_token=admin_token) - self.assertIn("success", channel.json_body["id_server_unbind_result"]) - - # Check that deactivated user is no longer counted - count = self.get_success(self.store.get_monthly_active_count()) - self.assertEqual(count, 1) - def create_user(self, localpart, token=None, appservice=False): request_data = { "username": localpart,