From 8df9941cc2462bc8e99ebd02953c5090f4942463 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 8 Jun 2021 14:09:00 +0100 Subject: [PATCH 1/8] 1.36.0rc1 --- CHANGES.md | 68 +++++++++++++++++++++++++++++++++++++++ changelog.d/10013.misc | 1 - changelog.d/10035.feature | 1 - changelog.d/10040.feature | 1 - changelog.d/10044.feature | 1 - changelog.d/10046.doc | 1 - changelog.d/10048.misc | 1 - changelog.d/10054.misc | 1 - changelog.d/10055.misc | 1 - changelog.d/10057.doc | 1 - changelog.d/10059.misc | 1 - changelog.d/10063.removal | 1 - changelog.d/10069.misc | 1 - changelog.d/10074.misc | 1 - changelog.d/10077.feature | 1 - changelog.d/10078.misc | 1 - changelog.d/10082.bugfix | 1 - changelog.d/10084.feature | 1 - changelog.d/10086.doc | 1 - changelog.d/10089.doc | 1 - changelog.d/10091.misc | 1 - changelog.d/10092.bugfix | 1 - changelog.d/10094.misc | 1 - changelog.d/10102.misc | 1 - changelog.d/10105.misc | 1 - changelog.d/10107.bugfix | 1 - changelog.d/10111.misc | 1 - changelog.d/10112.misc | 1 - changelog.d/10113.feature | 1 - changelog.d/10116.bugfix | 1 - changelog.d/10117.feature | 1 - changelog.d/10118.bugfix | 1 - changelog.d/10124.misc | 1 - changelog.d/10130.doc | 1 - changelog.d/10132.doc | 1 - changelog.d/10133.bugfix | 1 - changelog.d/10135.misc | 1 - changelog.d/10136.feature | 1 - changelog.d/10137.misc | 1 - changelog.d/10141.feature | 1 - changelog.d/9221.doc | 1 - changelog.d/9224.feature | 1 - changelog.d/9906.misc | 1 - changelog.d/9953.feature | 1 - changelog.d/9973.feature | 1 - synapse/__init__.py | 2 +- 46 files changed, 69 insertions(+), 45 deletions(-) delete mode 100644 changelog.d/10013.misc delete mode 100644 changelog.d/10035.feature delete mode 100644 changelog.d/10040.feature delete mode 100644 changelog.d/10044.feature delete mode 100644 changelog.d/10046.doc delete mode 100644 changelog.d/10048.misc delete mode 100644 changelog.d/10054.misc delete mode 100644 changelog.d/10055.misc delete mode 100644 changelog.d/10057.doc delete mode 100644 changelog.d/10059.misc delete mode 100644 changelog.d/10063.removal delete mode 100644 changelog.d/10069.misc delete mode 100644 changelog.d/10074.misc delete mode 100644 changelog.d/10077.feature delete mode 100644 changelog.d/10078.misc delete mode 100644 changelog.d/10082.bugfix delete mode 100644 changelog.d/10084.feature delete mode 100644 changelog.d/10086.doc delete mode 100644 changelog.d/10089.doc delete mode 100644 changelog.d/10091.misc delete mode 100644 changelog.d/10092.bugfix delete mode 100644 changelog.d/10094.misc delete mode 100644 changelog.d/10102.misc delete mode 100644 changelog.d/10105.misc delete mode 100644 changelog.d/10107.bugfix delete mode 100644 changelog.d/10111.misc delete mode 100644 changelog.d/10112.misc delete mode 100644 changelog.d/10113.feature delete mode 100644 changelog.d/10116.bugfix delete mode 100644 changelog.d/10117.feature delete mode 100644 changelog.d/10118.bugfix delete mode 100644 changelog.d/10124.misc delete mode 100644 changelog.d/10130.doc delete mode 100644 changelog.d/10132.doc delete mode 100644 changelog.d/10133.bugfix delete mode 100644 changelog.d/10135.misc delete mode 100644 changelog.d/10136.feature delete mode 100644 changelog.d/10137.misc delete mode 100644 changelog.d/10141.feature delete mode 100644 changelog.d/9221.doc delete mode 100644 changelog.d/9224.feature delete mode 100644 changelog.d/9906.misc delete mode 100644 changelog.d/9953.feature delete mode 100644 changelog.d/9973.feature diff --git a/CHANGES.md b/CHANGES.md index 04d260f8e5..69c876e38e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,71 @@ +Synapse 1.36.0rc1 (2021-06-08) +============================== + +Features +-------- + +- Add new endpoint `/_matrix/client/r0/rooms/{roomId}/aliases` from Client-Server API r0.6.1 (previously [MSC2432](https://github.com/matrix-org/matrix-doc/pull/2432)). ([\#9224](https://github.com/matrix-org/synapse/issues/9224)) +- Improve performance of incoming federation transactions in large rooms. ([\#9953](https://github.com/matrix-org/synapse/issues/9953), [\#9973](https://github.com/matrix-org/synapse/issues/9973)) +- Rewrite logic around verifying JSON object and fetching server keys to be more performant and use less memory. ([\#10035](https://github.com/matrix-org/synapse/issues/10035)) +- Add an admin API for unprotecting local media from quarantine. Contributed by @dklimpel. ([\#10040](https://github.com/matrix-org/synapse/issues/10040)) +- Add new admin APIs to remove media by media ID from quarantine. Contributed by @dkimpel. ([\#10044](https://github.com/matrix-org/synapse/issues/10044)) +- Make reason and score parameters optional for reporting content. Implements [MSC2414](https://github.com/matrix-org/matrix-doc/pull/2414). Contributed by Callum Brown. ([\#10077](https://github.com/matrix-org/synapse/issues/10077)) +- Add support for routing more requests to workers. ([\#10084](https://github.com/matrix-org/synapse/issues/10084)) +- Report OpenTracing spans for database activity. ([\#10113](https://github.com/matrix-org/synapse/issues/10113), [\#10136](https://github.com/matrix-org/synapse/issues/10136), [\#10141](https://github.com/matrix-org/synapse/issues/10141)) +- Significantly reduce memory usage of joining large remote rooms. ([\#10117](https://github.com/matrix-org/synapse/issues/10117)) + + +Bugfixes +-------- + +- Fixed a bug causing replication requests to fail when receiving a lot of events via federation. ([\#10082](https://github.com/matrix-org/synapse/issues/10082)) +- Fix a bug in the `force_tracing_for_users` option introduced in Synapse v1.35 which meant that the OpenTracing spans produced were missing most tags. ([\#10092](https://github.com/matrix-org/synapse/issues/10092)) +- Fixed a bug that could cause Synapse to stop notifying application services. Contributed by Willem Mulder. ([\#10107](https://github.com/matrix-org/synapse/issues/10107)) +- Fix bug where the server would attempt to fetch the same history in the room from a remote server multiple times in parallel. ([\#10116](https://github.com/matrix-org/synapse/issues/10116)) +- Fix a bug introduced in Synapse 1.33.0 which caused replication requests to fail when receiving a lot of very large events via federation. ([\#10118](https://github.com/matrix-org/synapse/issues/10118)) +- Fix bug when using workers where pagination requests failed if a remote server returned zero events from `/backfill`. Introduced in 1.35.0. ([\#10133](https://github.com/matrix-org/synapse/issues/10133)) + + +Improved Documentation +---------------------- + +- Clarify security note regarding hosting Synapse on the same domain as other web applications. ([\#9221](https://github.com/matrix-org/synapse/issues/9221)) +- Update CAPTCHA documentation to mention turning off the verify origin feature. Contributed by @aaronraimist. ([\#10046](https://github.com/matrix-org/synapse/issues/10046)) +- Tweak wording of database recommendation in `INSTALL.md`. Contributed by @aaronraimist. ([\#10057](https://github.com/matrix-org/synapse/issues/10057)) +- Add initial infrastructure for rendering Synapse documentation with mdbook. ([\#10086](https://github.com/matrix-org/synapse/issues/10086)) +- Convert the remaining Admin API documentation files to markdown. ([\#10089](https://github.com/matrix-org/synapse/issues/10089)) +- Make a link in docs use HTTPS. Contributed by @RhnSharma. ([\#10130](https://github.com/matrix-org/synapse/issues/10130)) +- Fix broken link in Docker docs. ([\#10132](https://github.com/matrix-org/synapse/issues/10132)) + + +Deprecations and Removals +------------------------- + +- Remove the experimental `spaces_enabled` flag. The spaces features are always available now. ([\#10063](https://github.com/matrix-org/synapse/issues/10063)) + + +Internal Changes +---------------- + +- Tell CircleCI to build Docker images from `main` branch. ([\#9906](https://github.com/matrix-org/synapse/issues/9906)) +- Simplify naming convention for release branches to only include the major and minor version numbers. ([\#10013](https://github.com/matrix-org/synapse/issues/10013)) +- Add `parse_strings_from_args` for parsing an array from query parameters. ([\#10048](https://github.com/matrix-org/synapse/issues/10048), [\#10137](https://github.com/matrix-org/synapse/issues/10137)) +- Remove some dead code regarding TLS certificate handling. ([\#10054](https://github.com/matrix-org/synapse/issues/10054)) +- Remove redundant, unmaintained `convert_server_keys` script. ([\#10055](https://github.com/matrix-org/synapse/issues/10055)) +- Improve the error message printed by synctl when synapse fails to start. ([\#10059](https://github.com/matrix-org/synapse/issues/10059)) +- Fix GitHub Actions lint for newsfragments. ([\#10069](https://github.com/matrix-org/synapse/issues/10069)) +- Update opentracing to inject the right context into the carrier. ([\#10074](https://github.com/matrix-org/synapse/issues/10074)) +- Fix up `BatchingQueue` implementation. ([\#10078](https://github.com/matrix-org/synapse/issues/10078)) +- Log method and path when dropping request due to size limit. ([\#10091](https://github.com/matrix-org/synapse/issues/10091)) +- In Github Actions workflows, summarize the Sytest results in an easy-to-read format. ([\#10094](https://github.com/matrix-org/synapse/issues/10094)) +- Make `/sync` do fewer state resolutions. ([\#10102](https://github.com/matrix-org/synapse/issues/10102)) +- Add missing type hints to the admin API servlets. ([\#10105](https://github.com/matrix-org/synapse/issues/10105)) +- Improve opentracing annotations for `Notifier`. ([\#10111](https://github.com/matrix-org/synapse/issues/10111)) +- Enable Prometheus metrics for the jaeger client library. ([\#10112](https://github.com/matrix-org/synapse/issues/10112)) +- Work to improve the responsiveness of `/sync` requests. ([\#10124](https://github.com/matrix-org/synapse/issues/10124)) +- OpenTracing: use a consistent name for background processes. ([\#10135](https://github.com/matrix-org/synapse/issues/10135)) + + Synapse 1.35.1 (2021-06-03) =========================== diff --git a/changelog.d/10013.misc b/changelog.d/10013.misc deleted file mode 100644 index 9d164d9ce2..0000000000 --- a/changelog.d/10013.misc +++ /dev/null @@ -1 +0,0 @@ -Simplify naming convention for release branches to only include the major and minor version numbers. diff --git a/changelog.d/10035.feature b/changelog.d/10035.feature deleted file mode 100644 index 68052b5a7e..0000000000 --- a/changelog.d/10035.feature +++ /dev/null @@ -1 +0,0 @@ -Rewrite logic around verifying JSON object and fetching server keys to be more performant and use less memory. diff --git a/changelog.d/10040.feature b/changelog.d/10040.feature deleted file mode 100644 index ec78a30f00..0000000000 --- a/changelog.d/10040.feature +++ /dev/null @@ -1 +0,0 @@ -Add an admin API for unprotecting local media from quarantine. Contributed by @dklimpel. diff --git a/changelog.d/10044.feature b/changelog.d/10044.feature deleted file mode 100644 index 70c0a3851e..0000000000 --- a/changelog.d/10044.feature +++ /dev/null @@ -1 +0,0 @@ -Add new admin APIs to remove media by media ID from quarantine. Contributed by @dkimpel. diff --git a/changelog.d/10046.doc b/changelog.d/10046.doc deleted file mode 100644 index 995960163b..0000000000 --- a/changelog.d/10046.doc +++ /dev/null @@ -1 +0,0 @@ -Update CAPTCHA documentation to mention turning off the verify origin feature. Contributed by @aaronraimist. diff --git a/changelog.d/10048.misc b/changelog.d/10048.misc deleted file mode 100644 index a901f8431e..0000000000 --- a/changelog.d/10048.misc +++ /dev/null @@ -1 +0,0 @@ -Add `parse_strings_from_args` for parsing an array from query parameters. diff --git a/changelog.d/10054.misc b/changelog.d/10054.misc deleted file mode 100644 index cebe39ce54..0000000000 --- a/changelog.d/10054.misc +++ /dev/null @@ -1 +0,0 @@ -Remove some dead code regarding TLS certificate handling. diff --git a/changelog.d/10055.misc b/changelog.d/10055.misc deleted file mode 100644 index da84a2dde8..0000000000 --- a/changelog.d/10055.misc +++ /dev/null @@ -1 +0,0 @@ -Remove redundant, unmaintained `convert_server_keys` script. diff --git a/changelog.d/10057.doc b/changelog.d/10057.doc deleted file mode 100644 index 35437cb017..0000000000 --- a/changelog.d/10057.doc +++ /dev/null @@ -1 +0,0 @@ -Tweak wording of database recommendation in `INSTALL.md`. Contributed by @aaronraimist. \ No newline at end of file diff --git a/changelog.d/10059.misc b/changelog.d/10059.misc deleted file mode 100644 index ca6e0e8a5a..0000000000 --- a/changelog.d/10059.misc +++ /dev/null @@ -1 +0,0 @@ -Improve the error message printed by synctl when synapse fails to start. diff --git a/changelog.d/10063.removal b/changelog.d/10063.removal deleted file mode 100644 index 0f8889b6b4..0000000000 --- a/changelog.d/10063.removal +++ /dev/null @@ -1 +0,0 @@ -Remove the experimental `spaces_enabled` flag. The spaces features are always available now. diff --git a/changelog.d/10069.misc b/changelog.d/10069.misc deleted file mode 100644 index a8d2629e9b..0000000000 --- a/changelog.d/10069.misc +++ /dev/null @@ -1 +0,0 @@ -Fix GitHub Actions lint for newsfragments. diff --git a/changelog.d/10074.misc b/changelog.d/10074.misc deleted file mode 100644 index 8dbe2cd2bc..0000000000 --- a/changelog.d/10074.misc +++ /dev/null @@ -1 +0,0 @@ -Update opentracing to inject the right context into the carrier. diff --git a/changelog.d/10077.feature b/changelog.d/10077.feature deleted file mode 100644 index 808feb2215..0000000000 --- a/changelog.d/10077.feature +++ /dev/null @@ -1 +0,0 @@ -Make reason and score parameters optional for reporting content. Implements [MSC2414](https://github.com/matrix-org/matrix-doc/pull/2414). Contributed by Callum Brown. diff --git a/changelog.d/10078.misc b/changelog.d/10078.misc deleted file mode 100644 index a4b089d0fd..0000000000 --- a/changelog.d/10078.misc +++ /dev/null @@ -1 +0,0 @@ -Fix up `BatchingQueue` implementation. diff --git a/changelog.d/10082.bugfix b/changelog.d/10082.bugfix deleted file mode 100644 index b4f8bcc4fa..0000000000 --- a/changelog.d/10082.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fixed a bug causing replication requests to fail when receiving a lot of events via federation. diff --git a/changelog.d/10084.feature b/changelog.d/10084.feature deleted file mode 100644 index 602cb6ff51..0000000000 --- a/changelog.d/10084.feature +++ /dev/null @@ -1 +0,0 @@ -Add support for routing more requests to workers. diff --git a/changelog.d/10086.doc b/changelog.d/10086.doc deleted file mode 100644 index 2200579012..0000000000 --- a/changelog.d/10086.doc +++ /dev/null @@ -1 +0,0 @@ -Add initial infrastructure for rendering Synapse documentation with mdbook. diff --git a/changelog.d/10089.doc b/changelog.d/10089.doc deleted file mode 100644 index d9e93773ab..0000000000 --- a/changelog.d/10089.doc +++ /dev/null @@ -1 +0,0 @@ -Convert the remaining Admin API documentation files to markdown. diff --git a/changelog.d/10091.misc b/changelog.d/10091.misc deleted file mode 100644 index dbe310fd17..0000000000 --- a/changelog.d/10091.misc +++ /dev/null @@ -1 +0,0 @@ -Log method and path when dropping request due to size limit. diff --git a/changelog.d/10092.bugfix b/changelog.d/10092.bugfix deleted file mode 100644 index 09b2aba7ff..0000000000 --- a/changelog.d/10092.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug in the `force_tracing_for_users` option introduced in Synapse v1.35 which meant that the OpenTracing spans produced were missing most tags. diff --git a/changelog.d/10094.misc b/changelog.d/10094.misc deleted file mode 100644 index 01efe14f74..0000000000 --- a/changelog.d/10094.misc +++ /dev/null @@ -1 +0,0 @@ -In Github Actions workflows, summarize the Sytest results in an easy-to-read format. diff --git a/changelog.d/10102.misc b/changelog.d/10102.misc deleted file mode 100644 index 87672ee295..0000000000 --- a/changelog.d/10102.misc +++ /dev/null @@ -1 +0,0 @@ -Make `/sync` do fewer state resolutions. diff --git a/changelog.d/10105.misc b/changelog.d/10105.misc deleted file mode 100644 index 244a893d3e..0000000000 --- a/changelog.d/10105.misc +++ /dev/null @@ -1 +0,0 @@ -Add missing type hints to the admin API servlets. \ No newline at end of file diff --git a/changelog.d/10107.bugfix b/changelog.d/10107.bugfix deleted file mode 100644 index 80030efab2..0000000000 --- a/changelog.d/10107.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fixed a bug that could cause Synapse to stop notifying application services. Contributed by Willem Mulder. diff --git a/changelog.d/10111.misc b/changelog.d/10111.misc deleted file mode 100644 index 42e42b69ab..0000000000 --- a/changelog.d/10111.misc +++ /dev/null @@ -1 +0,0 @@ -Improve opentracing annotations for `Notifier`. diff --git a/changelog.d/10112.misc b/changelog.d/10112.misc deleted file mode 100644 index 40af09760c..0000000000 --- a/changelog.d/10112.misc +++ /dev/null @@ -1 +0,0 @@ -Enable Prometheus metrics for the jaeger client library. diff --git a/changelog.d/10113.feature b/changelog.d/10113.feature deleted file mode 100644 index 2658ab8918..0000000000 --- a/changelog.d/10113.feature +++ /dev/null @@ -1 +0,0 @@ -Report OpenTracing spans for database activity. diff --git a/changelog.d/10116.bugfix b/changelog.d/10116.bugfix deleted file mode 100644 index 90ef707559..0000000000 --- a/changelog.d/10116.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix bug where the server would attempt to fetch the same history in the room from a remote server multiple times in parallel. diff --git a/changelog.d/10117.feature b/changelog.d/10117.feature deleted file mode 100644 index e137e142c6..0000000000 --- a/changelog.d/10117.feature +++ /dev/null @@ -1 +0,0 @@ -Significantly reduce memory usage of joining large remote rooms. diff --git a/changelog.d/10118.bugfix b/changelog.d/10118.bugfix deleted file mode 100644 index db62b50e0b..0000000000 --- a/changelog.d/10118.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug introduced in Synapse 1.33.0 which caused replication requests to fail when receiving a lot of very large events via federation. diff --git a/changelog.d/10124.misc b/changelog.d/10124.misc deleted file mode 100644 index c06593238d..0000000000 --- a/changelog.d/10124.misc +++ /dev/null @@ -1 +0,0 @@ -Work to improve the responsiveness of `/sync` requests. diff --git a/changelog.d/10130.doc b/changelog.d/10130.doc deleted file mode 100644 index 42ed1f3eac..0000000000 --- a/changelog.d/10130.doc +++ /dev/null @@ -1 +0,0 @@ -Make a link in docs use HTTPS. Contributed by @RhnSharma. diff --git a/changelog.d/10132.doc b/changelog.d/10132.doc deleted file mode 100644 index 70f538f077..0000000000 --- a/changelog.d/10132.doc +++ /dev/null @@ -1 +0,0 @@ -Fix broken link in Docker docs. diff --git a/changelog.d/10133.bugfix b/changelog.d/10133.bugfix deleted file mode 100644 index a62c15b260..0000000000 --- a/changelog.d/10133.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix bug when using workers where pagination requests failed if a remote server returned zero events from `/backfill`. Introduced in 1.35.0. diff --git a/changelog.d/10135.misc b/changelog.d/10135.misc deleted file mode 100644 index 17819cbbcc..0000000000 --- a/changelog.d/10135.misc +++ /dev/null @@ -1 +0,0 @@ -OpenTracing: use a consistent name for background processes. diff --git a/changelog.d/10136.feature b/changelog.d/10136.feature deleted file mode 100644 index 2658ab8918..0000000000 --- a/changelog.d/10136.feature +++ /dev/null @@ -1 +0,0 @@ -Report OpenTracing spans for database activity. diff --git a/changelog.d/10137.misc b/changelog.d/10137.misc deleted file mode 100644 index a901f8431e..0000000000 --- a/changelog.d/10137.misc +++ /dev/null @@ -1 +0,0 @@ -Add `parse_strings_from_args` for parsing an array from query parameters. diff --git a/changelog.d/10141.feature b/changelog.d/10141.feature deleted file mode 100644 index 2658ab8918..0000000000 --- a/changelog.d/10141.feature +++ /dev/null @@ -1 +0,0 @@ -Report OpenTracing spans for database activity. diff --git a/changelog.d/9221.doc b/changelog.d/9221.doc deleted file mode 100644 index 9b3476064b..0000000000 --- a/changelog.d/9221.doc +++ /dev/null @@ -1 +0,0 @@ -Clarify security note regarding hosting Synapse on the same domain as other web applications. diff --git a/changelog.d/9224.feature b/changelog.d/9224.feature deleted file mode 100644 index 76519c23e2..0000000000 --- a/changelog.d/9224.feature +++ /dev/null @@ -1 +0,0 @@ -Add new endpoint `/_matrix/client/r0/rooms/{roomId}/aliases` from Client-Server API r0.6.1 (previously [MSC2432](https://github.com/matrix-org/matrix-doc/pull/2432)). diff --git a/changelog.d/9906.misc b/changelog.d/9906.misc deleted file mode 100644 index 667d51a4c0..0000000000 --- a/changelog.d/9906.misc +++ /dev/null @@ -1 +0,0 @@ -Tell CircleCI to build Docker images from `main` branch. diff --git a/changelog.d/9953.feature b/changelog.d/9953.feature deleted file mode 100644 index 6b3d1adc70..0000000000 --- a/changelog.d/9953.feature +++ /dev/null @@ -1 +0,0 @@ -Improve performance of incoming federation transactions in large rooms. diff --git a/changelog.d/9973.feature b/changelog.d/9973.feature deleted file mode 100644 index 6b3d1adc70..0000000000 --- a/changelog.d/9973.feature +++ /dev/null @@ -1 +0,0 @@ -Improve performance of incoming federation transactions in large rooms. diff --git a/synapse/__init__.py b/synapse/__init__.py index 445e8a5cad..58261d04ef 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -47,7 +47,7 @@ try: except ImportError: pass -__version__ = "1.35.1" +__version__ = "1.36.0rc1" if bool(os.environ.get("SYNAPSE_TEST_PATCH_LOG_CONTEXTS", False)): # We import here so that we don't have to install a bunch of deps when From 684df9b21d3e7d66c919970c705f28e45275f88f Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 8 Jun 2021 14:11:16 +0100 Subject: [PATCH 2/8] fix typo in changelog --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 69c876e38e..f9aaecc6bd 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -8,7 +8,7 @@ Features - Improve performance of incoming federation transactions in large rooms. ([\#9953](https://github.com/matrix-org/synapse/issues/9953), [\#9973](https://github.com/matrix-org/synapse/issues/9973)) - Rewrite logic around verifying JSON object and fetching server keys to be more performant and use less memory. ([\#10035](https://github.com/matrix-org/synapse/issues/10035)) - Add an admin API for unprotecting local media from quarantine. Contributed by @dklimpel. ([\#10040](https://github.com/matrix-org/synapse/issues/10040)) -- Add new admin APIs to remove media by media ID from quarantine. Contributed by @dkimpel. ([\#10044](https://github.com/matrix-org/synapse/issues/10044)) +- Add new admin APIs to remove media by media ID from quarantine. Contributed by @dklimpel. ([\#10044](https://github.com/matrix-org/synapse/issues/10044)) - Make reason and score parameters optional for reporting content. Implements [MSC2414](https://github.com/matrix-org/matrix-doc/pull/2414). Contributed by Callum Brown. ([\#10077](https://github.com/matrix-org/synapse/issues/10077)) - Add support for routing more requests to workers. ([\#10084](https://github.com/matrix-org/synapse/issues/10084)) - Report OpenTracing spans for database activity. ([\#10113](https://github.com/matrix-org/synapse/issues/10113), [\#10136](https://github.com/matrix-org/synapse/issues/10136), [\#10141](https://github.com/matrix-org/synapse/issues/10141)) From e0ddd82f2ce58f9dd0038cb95047c316295b1b0d Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 8 Jun 2021 14:21:22 +0100 Subject: [PATCH 3/8] Make changelog lines consistent --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index f9aaecc6bd..48e9b55c8a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -7,7 +7,7 @@ Features - Add new endpoint `/_matrix/client/r0/rooms/{roomId}/aliases` from Client-Server API r0.6.1 (previously [MSC2432](https://github.com/matrix-org/matrix-doc/pull/2432)). ([\#9224](https://github.com/matrix-org/synapse/issues/9224)) - Improve performance of incoming federation transactions in large rooms. ([\#9953](https://github.com/matrix-org/synapse/issues/9953), [\#9973](https://github.com/matrix-org/synapse/issues/9973)) - Rewrite logic around verifying JSON object and fetching server keys to be more performant and use less memory. ([\#10035](https://github.com/matrix-org/synapse/issues/10035)) -- Add an admin API for unprotecting local media from quarantine. Contributed by @dklimpel. ([\#10040](https://github.com/matrix-org/synapse/issues/10040)) +- Add new admin APIs for unprotecting local media from quarantine. Contributed by @dklimpel. ([\#10040](https://github.com/matrix-org/synapse/issues/10040)) - Add new admin APIs to remove media by media ID from quarantine. Contributed by @dklimpel. ([\#10044](https://github.com/matrix-org/synapse/issues/10044)) - Make reason and score parameters optional for reporting content. Implements [MSC2414](https://github.com/matrix-org/matrix-doc/pull/2414). Contributed by Callum Brown. ([\#10077](https://github.com/matrix-org/synapse/issues/10077)) - Add support for routing more requests to workers. ([\#10084](https://github.com/matrix-org/synapse/issues/10084)) From c955f22e2c88676944124a4a3c80112b35231035 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 11 Jun 2021 10:27:12 +0100 Subject: [PATCH 4/8] Fix bug when running presence off master (#10149) Hopefully fixes #10027. --- changelog.d/10149.bugfix | 1 + synapse/storage/databases/main/presence.py | 2 +- synapse/storage/util/id_generators.py | 15 +++++++++++++++ 3 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 changelog.d/10149.bugfix diff --git a/changelog.d/10149.bugfix b/changelog.d/10149.bugfix new file mode 100644 index 0000000000..cb2d2eedb3 --- /dev/null +++ b/changelog.d/10149.bugfix @@ -0,0 +1 @@ +Fix a bug which caused presence updates to stop working some time after restart, when using a presence writer worker. diff --git a/synapse/storage/databases/main/presence.py b/synapse/storage/databases/main/presence.py index 6a2baa7841..1388771c40 100644 --- a/synapse/storage/databases/main/presence.py +++ b/synapse/storage/databases/main/presence.py @@ -50,7 +50,7 @@ class PresenceStore(SQLBaseStore): instance_name=self._instance_name, tables=[("presence_stream", "instance_name", "stream_id")], sequence_name="presence_stream_sequence", - writers=hs.config.worker.writers.to_device, + writers=hs.config.worker.writers.presence, ) else: self._presence_id_gen = StreamIdGenerator( diff --git a/synapse/storage/util/id_generators.py b/synapse/storage/util/id_generators.py index b1bd3a52d9..f1e62f9e85 100644 --- a/synapse/storage/util/id_generators.py +++ b/synapse/storage/util/id_generators.py @@ -397,6 +397,11 @@ class MultiWriterIdGenerator: # ... persist event ... """ + # If we have a list of instances that are allowed to write to this + # stream, make sure we're in it. + if self._writers and self._instance_name not in self._writers: + raise Exception("Tried to allocate stream ID on non-writer") + return _MultiWriterCtxManager(self) def get_next_mult(self, n: int): @@ -406,6 +411,11 @@ class MultiWriterIdGenerator: # ... persist events ... """ + # If we have a list of instances that are allowed to write to this + # stream, make sure we're in it. + if self._writers and self._instance_name not in self._writers: + raise Exception("Tried to allocate stream ID on non-writer") + return _MultiWriterCtxManager(self, n) def get_next_txn(self, txn: LoggingTransaction): @@ -416,6 +426,11 @@ class MultiWriterIdGenerator: # ... persist event ... """ + # If we have a list of instances that are allowed to write to this + # stream, make sure we're in it. + if self._writers and self._instance_name not in self._writers: + raise Exception("Tried to allocate stream ID on non-writer") + next_id = self._load_next_id_txn(txn) with self._lock: From 5e0b4719ea6650596470f2d3bff91a19096067b8 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 11 Jun 2021 13:08:30 +0100 Subject: [PATCH 5/8] Fix sending presence over federation when using workers (#10163) When using a federation sender we'd send out all local presence updates over federation even when they shouldn't be. Fixes #10153. --- changelog.d/10163.bugfix | 1 + synapse/handlers/presence.py | 25 +++++++++++++++++++------ 2 files changed, 20 insertions(+), 6 deletions(-) create mode 100644 changelog.d/10163.bugfix diff --git a/changelog.d/10163.bugfix b/changelog.d/10163.bugfix new file mode 100644 index 0000000000..7ccde66743 --- /dev/null +++ b/changelog.d/10163.bugfix @@ -0,0 +1 @@ +Fix a bug when using federation sender worker where it would send out more presence updates than necessary, leading to high resource usage. Broke in v1.33.0. diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index f5a049d754..79508580ac 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -495,9 +495,6 @@ class WorkerPresenceHandler(BasePresenceHandler): users=users_to_states.keys(), ) - # If this is a federation sender, notify about presence updates. - await self.maybe_send_presence_to_interested_destinations(states) - async def process_replication_rows( self, stream_name: str, instance_name: str, token: int, rows: list ): @@ -519,11 +516,27 @@ class WorkerPresenceHandler(BasePresenceHandler): for row in rows ] - for state in states: - self.user_to_current_state[state.user_id] = state + # The list of states to notify sync streams and remote servers about. + # This is calculated by comparing the old and new states for each user + # using `should_notify(..)`. + # + # Note that this is necessary as the presence writer will periodically + # flush presence state changes that should not be notified about to the + # DB, and so will be sent over the replication stream. + state_to_notify = [] + + for new_state in states: + old_state = self.user_to_current_state.get(new_state.user_id) + self.user_to_current_state[new_state.user_id] = new_state + + if not old_state or should_notify(old_state, new_state): + state_to_notify.append(new_state) stream_id = token - await self.notify_from_replication(states, stream_id) + await self.notify_from_replication(state_to_notify, stream_id) + + # If this is a federation sender, notify about presence updates. + await self.maybe_send_presence_to_interested_destinations(state_to_notify) def get_currently_syncing_users_for_replication(self) -> Iterable[str]: return [ From cdd985c64facb15b36fdc3bf479d25d6572f29a7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 11 Jun 2021 15:19:42 +0100 Subject: [PATCH 6/8] Only send a presence state to a destination once (#10165) It turns out that we were sending the same presence state to a remote potentially multiple times. --- changelog.d/10165.bugfix | 1 + synapse/handlers/presence.py | 25 ++++++++++++------------- 2 files changed, 13 insertions(+), 13 deletions(-) create mode 100644 changelog.d/10165.bugfix diff --git a/changelog.d/10165.bugfix b/changelog.d/10165.bugfix new file mode 100644 index 0000000000..8b1eeff352 --- /dev/null +++ b/changelog.d/10165.bugfix @@ -0,0 +1 @@ +Fix a bug where Synapse could send the same presence update to a remote twice. diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 79508580ac..44ed7a0712 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -299,14 +299,14 @@ class BasePresenceHandler(abc.ABC): if not states: return - hosts_and_states = await get_interested_remotes( + hosts_to_states = await get_interested_remotes( self.store, self.presence_router, states, ) - for destinations, states in hosts_and_states: - self._federation.send_presence_to_destinations(states, destinations) + for destination, host_states in hosts_to_states.items(): + self._federation.send_presence_to_destinations(host_states, [destination]) async def send_full_presence_to_users(self, user_ids: Collection[str]): """ @@ -842,15 +842,15 @@ class PresenceHandler(BasePresenceHandler): if to_federation_ping: federation_presence_out_counter.inc(len(to_federation_ping)) - hosts_and_states = await get_interested_remotes( + hosts_to_states = await get_interested_remotes( self.store, self.presence_router, list(to_federation_ping.values()), ) - for destinations, states in hosts_and_states: + for destination, states in hosts_to_states.items(): self._federation_queue.send_presence_to_destinations( - states, destinations + states, [destination] ) async def _handle_timeouts(self) -> None: @@ -1975,7 +1975,7 @@ async def get_interested_remotes( store: DataStore, presence_router: PresenceRouter, states: List[UserPresenceState], -) -> List[Tuple[Collection[str], List[UserPresenceState]]]: +) -> Dict[str, Set[UserPresenceState]]: """Given a list of presence states figure out which remote servers should be sent which. @@ -1987,11 +1987,9 @@ async def get_interested_remotes( states: A list of incoming user presence updates. Returns: - A list of 2-tuples of destinations and states, where for - each tuple the list of UserPresenceState should be sent to each - destination + A map from destinations to presence states to send to that destination. """ - hosts_and_states = [] # type: List[Tuple[Collection[str], List[UserPresenceState]]] + hosts_and_states: Dict[str, Set[UserPresenceState]] = {} # First we look up the rooms each user is in (as well as any explicit # subscriptions), then for each distinct room we look up the remote @@ -2003,11 +2001,12 @@ async def get_interested_remotes( for room_id, states in room_ids_to_states.items(): user_ids = await store.get_users_in_room(room_id) hosts = {get_domain_from_id(user_id) for user_id in user_ids} - hosts_and_states.append((hosts, states)) + for host in hosts: + hosts_and_states.setdefault(host, set()).update(states) for user_id, states in users_to_states.items(): host = get_domain_from_id(user_id) - hosts_and_states.append(([host], states)) + hosts_and_states.setdefault(host, set()).update(states) return hosts_and_states From fb10a73e85ff4a5c090226d046b6b7ede7e57d6e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 11 Jun 2021 15:21:34 +0100 Subject: [PATCH 7/8] 1.36.0rc2 --- CHANGES.md | 11 +++++++++++ changelog.d/10149.bugfix | 1 - changelog.d/10163.bugfix | 1 - changelog.d/10165.bugfix | 1 - synapse/__init__.py | 2 +- 5 files changed, 12 insertions(+), 4 deletions(-) delete mode 100644 changelog.d/10149.bugfix delete mode 100644 changelog.d/10163.bugfix delete mode 100644 changelog.d/10165.bugfix diff --git a/CHANGES.md b/CHANGES.md index 48e9b55c8a..cafb79124d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,14 @@ +Synapse 1.36.0rc2 (2021-06-11) +============================== + +Bugfixes +-------- + +- Fix a bug which caused presence updates to stop working some time after restart, when using a presence writer worker. ([\#10149](https://github.com/matrix-org/synapse/issues/10149)) +- Fix a bug when using federation sender worker where it would send out more presence updates than necessary, leading to high resource usage. Broke in v1.33.0. ([\#10163](https://github.com/matrix-org/synapse/issues/10163)) +- Fix a bug where Synapse could send the same presence update to a remote twice. ([\#10165](https://github.com/matrix-org/synapse/issues/10165)) + + Synapse 1.36.0rc1 (2021-06-08) ============================== diff --git a/changelog.d/10149.bugfix b/changelog.d/10149.bugfix deleted file mode 100644 index cb2d2eedb3..0000000000 --- a/changelog.d/10149.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug which caused presence updates to stop working some time after restart, when using a presence writer worker. diff --git a/changelog.d/10163.bugfix b/changelog.d/10163.bugfix deleted file mode 100644 index 7ccde66743..0000000000 --- a/changelog.d/10163.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug when using federation sender worker where it would send out more presence updates than necessary, leading to high resource usage. Broke in v1.33.0. diff --git a/changelog.d/10165.bugfix b/changelog.d/10165.bugfix deleted file mode 100644 index 8b1eeff352..0000000000 --- a/changelog.d/10165.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug where Synapse could send the same presence update to a remote twice. diff --git a/synapse/__init__.py b/synapse/__init__.py index 58261d04ef..407ba14a76 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -47,7 +47,7 @@ try: except ImportError: pass -__version__ = "1.36.0rc1" +__version__ = "1.36.0rc2" if bool(os.environ.get("SYNAPSE_TEST_PATCH_LOG_CONTEXTS", False)): # We import here so that we don't have to install a bunch of deps when From cbf350db63f74b9eb3922a8ebe0284f71e248a3c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 11 Jun 2021 15:30:42 +0100 Subject: [PATCH 8/8] Fixup changelog --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index cafb79124d..aeec4fa5fa 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,7 +4,7 @@ Synapse 1.36.0rc2 (2021-06-11) Bugfixes -------- -- Fix a bug which caused presence updates to stop working some time after restart, when using a presence writer worker. ([\#10149](https://github.com/matrix-org/synapse/issues/10149)) +- Fix a bug which caused presence updates to stop working some time after a restart, when using a presence writer worker. Broke in v1.33.0. ([\#10149](https://github.com/matrix-org/synapse/issues/10149)) - Fix a bug when using federation sender worker where it would send out more presence updates than necessary, leading to high resource usage. Broke in v1.33.0. ([\#10163](https://github.com/matrix-org/synapse/issues/10163)) - Fix a bug where Synapse could send the same presence update to a remote twice. ([\#10165](https://github.com/matrix-org/synapse/issues/10165))