From 641f43ba81d4c155cf2821a7be6b499aed66e1ec Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 12 Apr 2022 13:39:35 +0100 Subject: [PATCH 01/60] 1.57.0rc1 --- CHANGES.md | 116 ++++++++++++++++++++++++++++++++++++++ changelog.d/11881.feature | 1 - changelog.d/12040.feature | 1 - changelog.d/12115.bugfix | 1 - changelog.d/12165.misc | 1 - changelog.d/12191.misc | 1 - changelog.d/12193.misc | 1 - changelog.d/12209.misc | 1 - changelog.d/12235.bugfix | 1 - changelog.d/12251.feature | 1 - changelog.d/12252.feature | 1 - changelog.d/12267.misc | 1 - changelog.d/12271.doc | 1 - changelog.d/12280.misc | 1 - changelog.d/12293.removal | 1 - changelog.d/12295.misc | 1 - changelog.d/12302.feature | 1 - changelog.d/12310.feature | 1 - changelog.d/12315.doc | 1 - changelog.d/12316.misc | 1 - changelog.d/12317.misc | 1 - changelog.d/12321.misc | 1 - changelog.d/12326.misc | 1 - changelog.d/12327.feature | 1 - changelog.d/12329.bugfix | 1 - changelog.d/12330.misc | 1 - changelog.d/12331.doc | 1 - changelog.d/12332.misc | 1 - changelog.d/12333.bugfix | 1 - changelog.d/12334.misc | 1 - changelog.d/12335.misc | 1 - changelog.d/12336.misc | 1 - changelog.d/12338.bugfix | 1 - changelog.d/12339.doc | 1 - changelog.d/12341.feature | 1 - changelog.d/12345.doc | 1 - changelog.d/12346.misc | 1 - changelog.d/12347.misc | 1 - changelog.d/12348.misc | 1 - changelog.d/12349.misc | 1 - changelog.d/12350.bugfix | 1 - changelog.d/12351.misc | 1 - changelog.d/12353.misc | 1 - changelog.d/12354.misc | 1 - changelog.d/12355.misc | 1 - changelog.d/12357.misc | 1 - changelog.d/12358.misc | 1 - changelog.d/12364.bugfix | 1 - changelog.d/12366.misc | 1 - changelog.d/12367.feature | 1 - changelog.d/12369.doc | 1 - changelog.d/12370.feature | 1 - changelog.d/12371.misc | 1 - changelog.d/12372.feature | 1 - changelog.d/12376.misc | 1 - changelog.d/12379.misc | 1 - changelog.d/12380.misc | 1 - changelog.d/12381.misc | 1 - changelog.d/12384.misc | 1 - changelog.d/12385.docker | 1 - changelog.d/12389.misc | 1 - changelog.d/12390.bugfix | 1 - changelog.d/12391.feature | 1 - changelog.d/12392.doc | 1 - changelog.d/12398.misc | 1 - changelog.d/12400.misc | 1 - changelog.d/12402.misc | 1 - changelog.d/12403.feature | 1 - changelog.d/12404.misc | 1 - changelog.d/12405.misc | 1 - changelog.d/12407.misc | 1 - changelog.d/12408.misc | 1 - changelog.d/12409.misc | 1 - changelog.d/12410.bugfix | 1 - changelog.d/12412.misc | 1 - changelog.d/12415.misc | 1 - changelog.d/12418.misc | 1 - changelog.d/12420.misc | 1 - changelog.d/12421.misc | 1 - changelog.d/12422.misc | 1 - changelog.d/12423.misc | 1 - changelog.d/12435.misc | 1 - changelog.d/12436.feature | 2 - changelog.d/12439.misc | 1 - changelog.d/12444.misc | 1 - debian/changelog | 6 ++ synapse/__init__.py | 2 +- 87 files changed, 123 insertions(+), 86 deletions(-) delete mode 100644 changelog.d/11881.feature delete mode 100644 changelog.d/12040.feature delete mode 100644 changelog.d/12115.bugfix delete mode 100644 changelog.d/12165.misc delete mode 100644 changelog.d/12191.misc delete mode 100644 changelog.d/12193.misc delete mode 100644 changelog.d/12209.misc delete mode 100644 changelog.d/12235.bugfix delete mode 100644 changelog.d/12251.feature delete mode 100644 changelog.d/12252.feature delete mode 100644 changelog.d/12267.misc delete mode 100644 changelog.d/12271.doc delete mode 100644 changelog.d/12280.misc delete mode 100644 changelog.d/12293.removal delete mode 100644 changelog.d/12295.misc delete mode 100644 changelog.d/12302.feature delete mode 100644 changelog.d/12310.feature delete mode 100644 changelog.d/12315.doc delete mode 100644 changelog.d/12316.misc delete mode 100644 changelog.d/12317.misc delete mode 100644 changelog.d/12321.misc delete mode 100644 changelog.d/12326.misc delete mode 100644 changelog.d/12327.feature delete mode 100644 changelog.d/12329.bugfix delete mode 100644 changelog.d/12330.misc delete mode 100644 changelog.d/12331.doc delete mode 100644 changelog.d/12332.misc delete mode 100644 changelog.d/12333.bugfix delete mode 100644 changelog.d/12334.misc delete mode 100644 changelog.d/12335.misc delete mode 100644 changelog.d/12336.misc delete mode 100644 changelog.d/12338.bugfix delete mode 100644 changelog.d/12339.doc delete mode 100644 changelog.d/12341.feature delete mode 100644 changelog.d/12345.doc delete mode 100644 changelog.d/12346.misc delete mode 100644 changelog.d/12347.misc delete mode 100644 changelog.d/12348.misc delete mode 100644 changelog.d/12349.misc delete mode 100644 changelog.d/12350.bugfix delete mode 100644 changelog.d/12351.misc delete mode 100644 changelog.d/12353.misc delete mode 100644 changelog.d/12354.misc delete mode 100644 changelog.d/12355.misc delete mode 100644 changelog.d/12357.misc delete mode 100644 changelog.d/12358.misc delete mode 100644 changelog.d/12364.bugfix delete mode 100644 changelog.d/12366.misc delete mode 100644 changelog.d/12367.feature delete mode 100644 changelog.d/12369.doc delete mode 100644 changelog.d/12370.feature delete mode 100644 changelog.d/12371.misc delete mode 100644 changelog.d/12372.feature delete mode 100644 changelog.d/12376.misc delete mode 100644 changelog.d/12379.misc delete mode 100644 changelog.d/12380.misc delete mode 100644 changelog.d/12381.misc delete mode 100644 changelog.d/12384.misc delete mode 100644 changelog.d/12385.docker delete mode 100644 changelog.d/12389.misc delete mode 100644 changelog.d/12390.bugfix delete mode 100644 changelog.d/12391.feature delete mode 100644 changelog.d/12392.doc delete mode 100644 changelog.d/12398.misc delete mode 100644 changelog.d/12400.misc delete mode 100644 changelog.d/12402.misc delete mode 100644 changelog.d/12403.feature delete mode 100644 changelog.d/12404.misc delete mode 100644 changelog.d/12405.misc delete mode 100644 changelog.d/12407.misc delete mode 100644 changelog.d/12408.misc delete mode 100644 changelog.d/12409.misc delete mode 100644 changelog.d/12410.bugfix delete mode 100644 changelog.d/12412.misc delete mode 100644 changelog.d/12415.misc delete mode 100644 changelog.d/12418.misc delete mode 100644 changelog.d/12420.misc delete mode 100644 changelog.d/12421.misc delete mode 100644 changelog.d/12422.misc delete mode 100644 changelog.d/12423.misc delete mode 100644 changelog.d/12435.misc delete mode 100644 changelog.d/12436.feature delete mode 100644 changelog.d/12439.misc delete mode 100644 changelog.d/12444.misc diff --git a/CHANGES.md b/CHANGES.md index 518781f931..114b1e94ea 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,119 @@ +Synapse 1.57.0rc1 (2022-04-12) +============================== + +This version includes a [change](https://github.com/matrix-org/synapse/pull/12209) +to the way transaction IDs are managed for application services. If your +deployment uses a dedicated worker for application service traffic, **it must +be stopped** when the database is upgraded (which normally happens when the +main process is upgraded), to ensure the change is made safely without any risk +of reusing transaction IDs. + +See the [upgrade notes](https://github.com/matrix-org/synapse/blob/develop/docs/upgrade.md#upgrading-to-v1570) for more details. + +Features +-------- + +- Send device list changes to application services as specified by [MSC3202](https://github.com/matrix-org/matrix-spec-proposals/pull/3202), using unstable prefixes. The `msc3202_transaction_extensions` experimental homeserver config option must be enabled and `org.matrix.msc3202: true` must be present in the application service registration file for device list changes to be sent. The "left" field is currently always empty. ([\#11881](https://github.com/matrix-org/synapse/issues/11881)) +- Optimise fetching large quantities of missing room state over federation. ([\#12040](https://github.com/matrix-org/synapse/issues/12040)) +- Offload the `update_client_ip` background job from the main process to the background worker, when using Redis-based replication. ([\#12251](https://github.com/matrix-org/synapse/issues/12251)) +- Move `update_client_ip` background job from the main process to the background worker. ([\#12252](https://github.com/matrix-org/synapse/issues/12252)) +- Add a module callback to react to new 3PID (email address, phone number) associations. ([\#12302](https://github.com/matrix-org/synapse/issues/12302)) +- Add a configuration option to remove a specific set of rooms from sync responses. ([\#12310](https://github.com/matrix-org/synapse/issues/12310)) +- Add a module callback to react to account data changes. ([\#12327](https://github.com/matrix-org/synapse/issues/12327)) +- Allow setting user admin status using the module API. Contributed by Famedly. ([\#12341](https://github.com/matrix-org/synapse/issues/12341)) +- Reduce overhead of restarting synchrotrons. ([\#12367](https://github.com/matrix-org/synapse/issues/12367), [\#12372](https://github.com/matrix-org/synapse/issues/12372)) +- Update `/messages` to use historic pagination tokens if no `from` query parameter is given. ([\#12370](https://github.com/matrix-org/synapse/issues/12370)) +- Add a module API for reading and writing global account data. ([\#12391](https://github.com/matrix-org/synapse/issues/12391)) +- Support the stable `v1` endpoint for `/relations`, per [MSC2675](https://github.com/matrix-org/matrix-doc/pull/2675). ([\#12403](https://github.com/matrix-org/synapse/issues/12403)) +- Include bundled aggregations in search results + ([MSC3666](https://github.com/matrix-org/matrix-spec-proposals/pull/3666)). ([\#12436](https://github.com/matrix-org/synapse/issues/12436)) + + +Bugfixes +-------- + +- Fix a long-standing bug that updating the server notices user profile (display name/avatar URL) in the configuration would not be applied to pre-existing rooms. Contributed by Jorge Florian. ([\#12115](https://github.com/matrix-org/synapse/issues/12115)) +- Fix a long-standing bug where events from ignored users were still considered for bundled aggregations. ([\#12235](https://github.com/matrix-org/synapse/issues/12235), [\#12338](https://github.com/matrix-org/synapse/issues/12338)) +- Fix non-member state events not resolving for historical events when used in [MSC2716](https://github.com/matrix-org/matrix-spec-proposals/pull/2716) `/batch_send` `state_events_at_start`. ([\#12329](https://github.com/matrix-org/synapse/issues/12329)) +- Fix a long-standing bug affecting URL previews that would generate a 500 response instead of a 403 if the previewed URL includes a port that isn't allowed by the relevant blacklist. ([\#12333](https://github.com/matrix-org/synapse/issues/12333)) +- Default to `private` room visibility rather than `public` when a client does not specify one, according to spec. ([\#12350](https://github.com/matrix-org/synapse/issues/12350)) +- Fix a spec compliance issue where requests to the `/publicRooms` federation API would specify `limit` as a string. ([\#12364](https://github.com/matrix-org/synapse/issues/12364), [\#12410](https://github.com/matrix-org/synapse/issues/12410)) +- Fix a bug introduced in Synapse 1.49.0 which caused the `synapse_event_persisted_position` metric to have invalid values. ([\#12390](https://github.com/matrix-org/synapse/issues/12390)) + + +Updates to the Docker image +--------------------------- + +- Bundle locked versions of dependencies into the Docker image. ([\#12385](https://github.com/matrix-org/synapse/issues/12385)) + + +Improved Documentation +---------------------- + +- Clarify documentation for running SyTest against Synapse, including use of Postgres and worker mode. ([\#12271](https://github.com/matrix-org/synapse/issues/12271)) +- Document the behaviour of `LoggingTransaction.call_after` and `LoggingTransaction.call_on_exception` methods when transactions are retried. ([\#12315](https://github.com/matrix-org/synapse/issues/12315)) +- Update dead links in `check-newsfragment.sh` to point to the correct documentation URL. ([\#12331](https://github.com/matrix-org/synapse/issues/12331)) +- Upgrade the version of `mdbook` in CI to 0.4.17. ([\#12339](https://github.com/matrix-org/synapse/issues/12339)) +- Updates to the Room DAG concepts development document to clarify that we mark events as outliers because we don't have any state for them. ([\#12345](https://github.com/matrix-org/synapse/issues/12345)) +- Update the link to Redis pub/sub documentation in the workers documentation.. ([\#12369](https://github.com/matrix-org/synapse/issues/12369)) +- Remove documentation for converting a legacy structured logging configuration to the new format. ([\#12392](https://github.com/matrix-org/synapse/issues/12392)) + + +Deprecations and Removals +------------------------- + +- Remove the unused and unstable `/aggregations` endpoint which was removed from [MSC2675](https://github.com/matrix-org/matrix-doc/pull/2675). ([\#12293](https://github.com/matrix-org/synapse/issues/12293)) + + +Internal Changes +---------------- + +- Remove lingering unstable references to MSC2403 (knocking). ([\#12165](https://github.com/matrix-org/synapse/issues/12165)) +- Avoid trying to calculate the state at outlier events. ([\#12191](https://github.com/matrix-org/synapse/issues/12191), [\#12316](https://github.com/matrix-org/synapse/issues/12316), [\#12330](https://github.com/matrix-org/synapse/issues/12330), [\#12332](https://github.com/matrix-org/synapse/issues/12332), [\#12409](https://github.com/matrix-org/synapse/issues/12409)) +- Omit sending "offline" presence updates to application services after they are initially configured. ([\#12193](https://github.com/matrix-org/synapse/issues/12193)) +- Switch to using a sequence to generate AS transaction IDs. Contributed by Nick Beeper. If running synapse with a dedicated appservice worker, this MUST be stopped before upgrading the main process and database. ([\#12209](https://github.com/matrix-org/synapse/issues/12209)) +- Add missing type hints for storage. ([\#12267](https://github.com/matrix-org/synapse/issues/12267)) +- Add missing type definitions for scripts in docker folder. Contributed by Jorge Florian. ([\#12280](https://github.com/matrix-org/synapse/issues/12280)) +- Move [MSC2654](https://github.com/matrix-org/matrix-doc/pull/2654) support behind an experimental configuration flag. ([\#12295](https://github.com/matrix-org/synapse/issues/12295)) +- Update docstrings to explain how to decipher live and historic pagination tokens. ([\#12317](https://github.com/matrix-org/synapse/issues/12317)) +- Add ground work for speeding up device list updates for users in large numbers of rooms. ([\#12321](https://github.com/matrix-org/synapse/issues/12321)) +- Fix typechecker problems exposed by signedjson 1.1.2. ([\#12326](https://github.com/matrix-org/synapse/issues/12326)) +- Remove the `tox` packaging job: it will be redundant once #11537 lands. ([\#12334](https://github.com/matrix-org/synapse/issues/12334)) +- Ignore `.envrc` for `direnv` users. ([\#12335](https://github.com/matrix-org/synapse/issues/12335)) +- Remove the (broadly unused, dev-only) dockerfile for pg tests. ([\#12336](https://github.com/matrix-org/synapse/issues/12336)) +- Remove redundant `get_success` calls in test code. ([\#12346](https://github.com/matrix-org/synapse/issues/12346)) +- Add type annotations for `tests/unittest.py`. ([\#12347](https://github.com/matrix-org/synapse/issues/12347)) +- Move single-use methods out of `TestCase`. ([\#12348](https://github.com/matrix-org/synapse/issues/12348)) +- Remove broken and unused development scripts. ([\#12349](https://github.com/matrix-org/synapse/issues/12349), [\#12351](https://github.com/matrix-org/synapse/issues/12351), [\#12355](https://github.com/matrix-org/synapse/issues/12355)) +- Convert `Linearizer` tests from `inlineCallbacks` to async. ([\#12353](https://github.com/matrix-org/synapse/issues/12353)) +- Update docstrings for `ReadWriteLock` tests. ([\#12354](https://github.com/matrix-org/synapse/issues/12354)) +- Refactor `Linearizer`, convert methods to async and use an async context manager. ([\#12357](https://github.com/matrix-org/synapse/issues/12357)) +- Fix a long-standing bug where `Linearizer`s could get stuck if a cancellation were to happen at the wrong time. ([\#12358](https://github.com/matrix-org/synapse/issues/12358)) +- Make `StreamToken.from_string` and `RoomStreamToken.parse` propagate cancellations instead of replacing them with `SynapseError`s. ([\#12366](https://github.com/matrix-org/synapse/issues/12366)) +- Add type hints to tests files. ([\#12371](https://github.com/matrix-org/synapse/issues/12371)) +- Allow specifying the Postgres database's port when running unit tests with Postgres. ([\#12376](https://github.com/matrix-org/synapse/issues/12376)) +- Remove temporary pin of signedjson<=1.1.1 that was added in Synapse 1.56.0. ([\#12379](https://github.com/matrix-org/synapse/issues/12379)) +- Add opentracing spans to calls to external cache. ([\#12380](https://github.com/matrix-org/synapse/issues/12380)) +- Lay groundwork for using `poetry` to manage Synapse's dependencies. ([\#12381](https://github.com/matrix-org/synapse/issues/12381)) +- Make missing `importlib_metadata` dependency explicit. ([\#12384](https://github.com/matrix-org/synapse/issues/12384), [\#12400](https://github.com/matrix-org/synapse/issues/12400)) +- Update type annotations for compatiblity with prometheus_client 0.14. ([\#12389](https://github.com/matrix-org/synapse/issues/12389)) +- Remove support for the unstable identifiers specified in [MSC3288](https://github.com/matrix-org/matrix-doc/pull/3288). ([\#12398](https://github.com/matrix-org/synapse/issues/12398)) +- Add missing type hints to configuration classes. ([\#12402](https://github.com/matrix-org/synapse/issues/12402)) +- Add files used to build the Docker image used for complement testing into the Synapse repository. ([\#12404](https://github.com/matrix-org/synapse/issues/12404)) +- Fix up healthcheck generation for workers docker image. ([\#12405](https://github.com/matrix-org/synapse/issues/12405)) +- Run the olddeps CI job using Poetry. ([\#12407](https://github.com/matrix-org/synapse/issues/12407)) +- Do not include groups in the sync response when disabled. ([\#12408](https://github.com/matrix-org/synapse/issues/12408)) +- Run the CI portdb script in the locked poetry environment. ([\#12412](https://github.com/matrix-org/synapse/issues/12412)) +- Improve type hints related to HTTP query parameters. ([\#12415](https://github.com/matrix-org/synapse/issues/12415)) +- Run the CI export-data script in the locked poetry environment. ([\#12418](https://github.com/matrix-org/synapse/issues/12418)) +- Stop maintaining a list of lint targets. ([\#12420](https://github.com/matrix-org/synapse/issues/12420)) +- Make `synapse._scripts` pass type checks. ([\#12421](https://github.com/matrix-org/synapse/issues/12421), [\#12422](https://github.com/matrix-org/synapse/issues/12422)) +- Add some type hints to datastore. ([\#12423](https://github.com/matrix-org/synapse/issues/12423)) +- Enable certificate checking during complement tests. ([\#12435](https://github.com/matrix-org/synapse/issues/12435)) +- Bundle locked versions of dependencies into the Docker image. ([\#12439](https://github.com/matrix-org/synapse/issues/12439)) +- Explicitly specify the `tls` extra for Twisted dependency. ([\#12444](https://github.com/matrix-org/synapse/issues/12444)) + + Synapse 1.56.0 (2022-04-05) =========================== diff --git a/changelog.d/11881.feature b/changelog.d/11881.feature deleted file mode 100644 index 392294ffc3..0000000000 --- a/changelog.d/11881.feature +++ /dev/null @@ -1 +0,0 @@ -Send device list changes to application services as specified by [MSC3202](https://github.com/matrix-org/matrix-spec-proposals/pull/3202), using unstable prefixes. The `msc3202_transaction_extensions` experimental homeserver config option must be enabled and `org.matrix.msc3202: true` must be present in the application service registration file for device list changes to be sent. The "left" field is currently always empty. \ No newline at end of file diff --git a/changelog.d/12040.feature b/changelog.d/12040.feature deleted file mode 100644 index 0a88c6e80c..0000000000 --- a/changelog.d/12040.feature +++ /dev/null @@ -1 +0,0 @@ -Optimise fetching large quantities of missing room state over federation. diff --git a/changelog.d/12115.bugfix b/changelog.d/12115.bugfix deleted file mode 100644 index eff842b2a5..0000000000 --- a/changelog.d/12115.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a long-standing bug that updating the server notices user profile (display name/avatar URL) in the configuration would not be applied to pre-existing rooms. Contributed by Jorge Florian. diff --git a/changelog.d/12165.misc b/changelog.d/12165.misc deleted file mode 100644 index 4b80b0562e..0000000000 --- a/changelog.d/12165.misc +++ /dev/null @@ -1 +0,0 @@ -Remove lingering unstable references to MSC2403 (knocking). diff --git a/changelog.d/12191.misc b/changelog.d/12191.misc deleted file mode 100644 index 9f333e718a..0000000000 --- a/changelog.d/12191.misc +++ /dev/null @@ -1 +0,0 @@ -Avoid trying to calculate the state at outlier events. diff --git a/changelog.d/12193.misc b/changelog.d/12193.misc deleted file mode 100644 index a721254d22..0000000000 --- a/changelog.d/12193.misc +++ /dev/null @@ -1 +0,0 @@ -Omit sending "offline" presence updates to application services after they are initially configured. \ No newline at end of file diff --git a/changelog.d/12209.misc b/changelog.d/12209.misc deleted file mode 100644 index d145b5eb04..0000000000 --- a/changelog.d/12209.misc +++ /dev/null @@ -1 +0,0 @@ -Switch to using a sequence to generate AS transaction IDs. Contributed by Nick Beeper. If running synapse with a dedicated appservice worker, this MUST be stopped before upgrading the main process and database. diff --git a/changelog.d/12235.bugfix b/changelog.d/12235.bugfix deleted file mode 100644 index b5d2bede67..0000000000 --- a/changelog.d/12235.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a long-standing bug where events from ignored users were still considered for bundled aggregations. diff --git a/changelog.d/12251.feature b/changelog.d/12251.feature deleted file mode 100644 index ba9ede03c6..0000000000 --- a/changelog.d/12251.feature +++ /dev/null @@ -1 +0,0 @@ -Offload the `update_client_ip` background job from the main process to the background worker, when using Redis-based replication. diff --git a/changelog.d/12252.feature b/changelog.d/12252.feature deleted file mode 100644 index 82b9e82f86..0000000000 --- a/changelog.d/12252.feature +++ /dev/null @@ -1 +0,0 @@ -Move `update_client_ip` background job from the main process to the background worker. \ No newline at end of file diff --git a/changelog.d/12267.misc b/changelog.d/12267.misc deleted file mode 100644 index e43844d44a..0000000000 --- a/changelog.d/12267.misc +++ /dev/null @@ -1 +0,0 @@ -Add missing type hints for storage. diff --git a/changelog.d/12271.doc b/changelog.d/12271.doc deleted file mode 100644 index d9696fc5d5..0000000000 --- a/changelog.d/12271.doc +++ /dev/null @@ -1 +0,0 @@ -Clarify documentation for running SyTest against Synapse, including use of Postgres and worker mode. \ No newline at end of file diff --git a/changelog.d/12280.misc b/changelog.d/12280.misc deleted file mode 100644 index bdc00b821a..0000000000 --- a/changelog.d/12280.misc +++ /dev/null @@ -1 +0,0 @@ -Add missing type definitions for scripts in docker folder. Contributed by Jorge Florian. diff --git a/changelog.d/12293.removal b/changelog.d/12293.removal deleted file mode 100644 index 25214a4b49..0000000000 --- a/changelog.d/12293.removal +++ /dev/null @@ -1 +0,0 @@ -Remove the unused and unstable `/aggregations` endpoint which was removed from [MSC2675](https://github.com/matrix-org/matrix-doc/pull/2675). diff --git a/changelog.d/12295.misc b/changelog.d/12295.misc deleted file mode 100644 index 9c34e16909..0000000000 --- a/changelog.d/12295.misc +++ /dev/null @@ -1 +0,0 @@ -Move [MSC2654](https://github.com/matrix-org/matrix-doc/pull/2654) support behind an experimental configuration flag. diff --git a/changelog.d/12302.feature b/changelog.d/12302.feature deleted file mode 100644 index 603fa2d23a..0000000000 --- a/changelog.d/12302.feature +++ /dev/null @@ -1 +0,0 @@ -Add a module callback to react to new 3PID (email address, phone number) associations. diff --git a/changelog.d/12310.feature b/changelog.d/12310.feature deleted file mode 100644 index f3fbb298f7..0000000000 --- a/changelog.d/12310.feature +++ /dev/null @@ -1 +0,0 @@ -Add a configuration option to remove a specific set of rooms from sync responses. diff --git a/changelog.d/12315.doc b/changelog.d/12315.doc deleted file mode 100644 index ed72f55cba..0000000000 --- a/changelog.d/12315.doc +++ /dev/null @@ -1 +0,0 @@ -Document the behaviour of `LoggingTransaction.call_after` and `LoggingTransaction.call_on_exception` methods when transactions are retried. diff --git a/changelog.d/12316.misc b/changelog.d/12316.misc deleted file mode 100644 index 9f333e718a..0000000000 --- a/changelog.d/12316.misc +++ /dev/null @@ -1 +0,0 @@ -Avoid trying to calculate the state at outlier events. diff --git a/changelog.d/12317.misc b/changelog.d/12317.misc deleted file mode 100644 index 1dfee496d8..0000000000 --- a/changelog.d/12317.misc +++ /dev/null @@ -1 +0,0 @@ -Update docstrings to explain how to decipher live and historic pagination tokens. diff --git a/changelog.d/12321.misc b/changelog.d/12321.misc deleted file mode 100644 index 200e7c44fe..0000000000 --- a/changelog.d/12321.misc +++ /dev/null @@ -1 +0,0 @@ -Add ground work for speeding up device list updates for users in large numbers of rooms. diff --git a/changelog.d/12326.misc b/changelog.d/12326.misc deleted file mode 100644 index 2d2a00e571..0000000000 --- a/changelog.d/12326.misc +++ /dev/null @@ -1 +0,0 @@ -Fix typechecker problems exposed by signedjson 1.1.2. diff --git a/changelog.d/12327.feature b/changelog.d/12327.feature deleted file mode 100644 index 4fe294f1b1..0000000000 --- a/changelog.d/12327.feature +++ /dev/null @@ -1 +0,0 @@ -Add a module callback to react to account data changes. diff --git a/changelog.d/12329.bugfix b/changelog.d/12329.bugfix deleted file mode 100644 index aef4117343..0000000000 --- a/changelog.d/12329.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix non-member state events not resolving for historical events when used in [MSC2716](https://github.com/matrix-org/matrix-spec-proposals/pull/2716) `/batch_send` `state_events_at_start`. diff --git a/changelog.d/12330.misc b/changelog.d/12330.misc deleted file mode 100644 index 9f333e718a..0000000000 --- a/changelog.d/12330.misc +++ /dev/null @@ -1 +0,0 @@ -Avoid trying to calculate the state at outlier events. diff --git a/changelog.d/12331.doc b/changelog.d/12331.doc deleted file mode 100644 index ec0ca3ea95..0000000000 --- a/changelog.d/12331.doc +++ /dev/null @@ -1 +0,0 @@ -Update dead links in `check-newsfragment.sh` to point to the correct documentation URL. diff --git a/changelog.d/12332.misc b/changelog.d/12332.misc deleted file mode 100644 index 9f333e718a..0000000000 --- a/changelog.d/12332.misc +++ /dev/null @@ -1 +0,0 @@ -Avoid trying to calculate the state at outlier events. diff --git a/changelog.d/12333.bugfix b/changelog.d/12333.bugfix deleted file mode 100644 index 2c073a77d5..0000000000 --- a/changelog.d/12333.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a long-standing bug affecting URL previews that would generate a 500 response instead of a 403 if the previewed URL includes a port that isn't allowed by the relevant blacklist. diff --git a/changelog.d/12334.misc b/changelog.d/12334.misc deleted file mode 100644 index 10a57e23b3..0000000000 --- a/changelog.d/12334.misc +++ /dev/null @@ -1 +0,0 @@ -Remove the `tox` packaging job: it will be redundant once #11537 lands. diff --git a/changelog.d/12335.misc b/changelog.d/12335.misc deleted file mode 100644 index 39ea361159..0000000000 --- a/changelog.d/12335.misc +++ /dev/null @@ -1 +0,0 @@ -Ignore `.envrc` for `direnv` users. diff --git a/changelog.d/12336.misc b/changelog.d/12336.misc deleted file mode 100644 index 0aecd543f9..0000000000 --- a/changelog.d/12336.misc +++ /dev/null @@ -1 +0,0 @@ -Remove the (broadly unused, dev-only) dockerfile for pg tests. diff --git a/changelog.d/12338.bugfix b/changelog.d/12338.bugfix deleted file mode 100644 index b5d2bede67..0000000000 --- a/changelog.d/12338.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a long-standing bug where events from ignored users were still considered for bundled aggregations. diff --git a/changelog.d/12339.doc b/changelog.d/12339.doc deleted file mode 100644 index cdbd55bb34..0000000000 --- a/changelog.d/12339.doc +++ /dev/null @@ -1 +0,0 @@ -Upgrade the version of `mdbook` in CI to 0.4.17. diff --git a/changelog.d/12341.feature b/changelog.d/12341.feature deleted file mode 100644 index ebb96ee486..0000000000 --- a/changelog.d/12341.feature +++ /dev/null @@ -1 +0,0 @@ -Allow setting user admin status using the module API. Contributed by Famedly. diff --git a/changelog.d/12345.doc b/changelog.d/12345.doc deleted file mode 100644 index 0dcf836e68..0000000000 --- a/changelog.d/12345.doc +++ /dev/null @@ -1 +0,0 @@ -Updates to the Room DAG concepts development document to clarify that we mark events as outliers because we don't have any state for them. diff --git a/changelog.d/12346.misc b/changelog.d/12346.misc deleted file mode 100644 index 6561b3be82..0000000000 --- a/changelog.d/12346.misc +++ /dev/null @@ -1 +0,0 @@ -Remove redundant `get_success` calls in test code. diff --git a/changelog.d/12347.misc b/changelog.d/12347.misc deleted file mode 100644 index 1f6f584e6d..0000000000 --- a/changelog.d/12347.misc +++ /dev/null @@ -1 +0,0 @@ -Add type annotations for `tests/unittest.py`. diff --git a/changelog.d/12348.misc b/changelog.d/12348.misc deleted file mode 100644 index 2bfeadb7f8..0000000000 --- a/changelog.d/12348.misc +++ /dev/null @@ -1 +0,0 @@ -Move single-use methods out of `TestCase`. diff --git a/changelog.d/12349.misc b/changelog.d/12349.misc deleted file mode 100644 index b2d83a7f78..0000000000 --- a/changelog.d/12349.misc +++ /dev/null @@ -1 +0,0 @@ -Remove broken and unused development scripts. diff --git a/changelog.d/12350.bugfix b/changelog.d/12350.bugfix deleted file mode 100644 index 9cbdc28038..0000000000 --- a/changelog.d/12350.bugfix +++ /dev/null @@ -1 +0,0 @@ -Default to `private` room visibility rather than `public` when a client does not specify one, according to spec. \ No newline at end of file diff --git a/changelog.d/12351.misc b/changelog.d/12351.misc deleted file mode 100644 index b1dd1e2696..0000000000 --- a/changelog.d/12351.misc +++ /dev/null @@ -1 +0,0 @@ -Remove broken and unused development scripts. \ No newline at end of file diff --git a/changelog.d/12353.misc b/changelog.d/12353.misc deleted file mode 100644 index 1d681fb0e3..0000000000 --- a/changelog.d/12353.misc +++ /dev/null @@ -1 +0,0 @@ -Convert `Linearizer` tests from `inlineCallbacks` to async. diff --git a/changelog.d/12354.misc b/changelog.d/12354.misc deleted file mode 100644 index e3b8950faa..0000000000 --- a/changelog.d/12354.misc +++ /dev/null @@ -1 +0,0 @@ -Update docstrings for `ReadWriteLock` tests. diff --git a/changelog.d/12355.misc b/changelog.d/12355.misc deleted file mode 100644 index b1dd1e2696..0000000000 --- a/changelog.d/12355.misc +++ /dev/null @@ -1 +0,0 @@ -Remove broken and unused development scripts. \ No newline at end of file diff --git a/changelog.d/12357.misc b/changelog.d/12357.misc deleted file mode 100644 index d571ae034c..0000000000 --- a/changelog.d/12357.misc +++ /dev/null @@ -1 +0,0 @@ -Refactor `Linearizer`, convert methods to async and use an async context manager. diff --git a/changelog.d/12358.misc b/changelog.d/12358.misc deleted file mode 100644 index fcacbcba5c..0000000000 --- a/changelog.d/12358.misc +++ /dev/null @@ -1 +0,0 @@ -Fix a long-standing bug where `Linearizer`s could get stuck if a cancellation were to happen at the wrong time. diff --git a/changelog.d/12364.bugfix b/changelog.d/12364.bugfix deleted file mode 100644 index d042425d20..0000000000 --- a/changelog.d/12364.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a spec compliance issue where requests to the `/publicRooms` federation API would specify `limit` as a string. \ No newline at end of file diff --git a/changelog.d/12366.misc b/changelog.d/12366.misc deleted file mode 100644 index 33d8e6c712..0000000000 --- a/changelog.d/12366.misc +++ /dev/null @@ -1 +0,0 @@ -Make `StreamToken.from_string` and `RoomStreamToken.parse` propagate cancellations instead of replacing them with `SynapseError`s. diff --git a/changelog.d/12367.feature b/changelog.d/12367.feature deleted file mode 100644 index 34bb60e966..0000000000 --- a/changelog.d/12367.feature +++ /dev/null @@ -1 +0,0 @@ -Reduce overhead of restarting synchrotrons. diff --git a/changelog.d/12369.doc b/changelog.d/12369.doc deleted file mode 100644 index c34271b2ce..0000000000 --- a/changelog.d/12369.doc +++ /dev/null @@ -1 +0,0 @@ -Update the link to Redis pub/sub documentation in the workers documentation.. \ No newline at end of file diff --git a/changelog.d/12370.feature b/changelog.d/12370.feature deleted file mode 100644 index 2c695ddaa0..0000000000 --- a/changelog.d/12370.feature +++ /dev/null @@ -1 +0,0 @@ -Update `/messages` to use historic pagination tokens if no `from` query parameter is given. diff --git a/changelog.d/12371.misc b/changelog.d/12371.misc deleted file mode 100644 index c5b6356799..0000000000 --- a/changelog.d/12371.misc +++ /dev/null @@ -1 +0,0 @@ -Add type hints to tests files. \ No newline at end of file diff --git a/changelog.d/12372.feature b/changelog.d/12372.feature deleted file mode 100644 index 34bb60e966..0000000000 --- a/changelog.d/12372.feature +++ /dev/null @@ -1 +0,0 @@ -Reduce overhead of restarting synchrotrons. diff --git a/changelog.d/12376.misc b/changelog.d/12376.misc deleted file mode 100644 index 191f84bbd4..0000000000 --- a/changelog.d/12376.misc +++ /dev/null @@ -1 +0,0 @@ -Allow specifying the Postgres database's port when running unit tests with Postgres. \ No newline at end of file diff --git a/changelog.d/12379.misc b/changelog.d/12379.misc deleted file mode 100644 index b55778577d..0000000000 --- a/changelog.d/12379.misc +++ /dev/null @@ -1 +0,0 @@ -Remove temporary pin of signedjson<=1.1.1 that was added in Synapse 1.56.0. diff --git a/changelog.d/12380.misc b/changelog.d/12380.misc deleted file mode 100644 index 178a00321a..0000000000 --- a/changelog.d/12380.misc +++ /dev/null @@ -1 +0,0 @@ -Add opentracing spans to calls to external cache. diff --git a/changelog.d/12381.misc b/changelog.d/12381.misc deleted file mode 100644 index a1d8d60ebb..0000000000 --- a/changelog.d/12381.misc +++ /dev/null @@ -1 +0,0 @@ -Lay groundwork for using `poetry` to manage Synapse's dependencies. \ No newline at end of file diff --git a/changelog.d/12384.misc b/changelog.d/12384.misc deleted file mode 100644 index 8a60f32876..0000000000 --- a/changelog.d/12384.misc +++ /dev/null @@ -1 +0,0 @@ -Make missing `importlib_metadata` dependency explicit. \ No newline at end of file diff --git a/changelog.d/12385.docker b/changelog.d/12385.docker deleted file mode 100644 index abe2127ea0..0000000000 --- a/changelog.d/12385.docker +++ /dev/null @@ -1 +0,0 @@ -Bundle locked versions of dependencies into the Docker image. \ No newline at end of file diff --git a/changelog.d/12389.misc b/changelog.d/12389.misc deleted file mode 100644 index 00e3a5d758..0000000000 --- a/changelog.d/12389.misc +++ /dev/null @@ -1 +0,0 @@ -Update type annotations for compatiblity with prometheus_client 0.14. diff --git a/changelog.d/12390.bugfix b/changelog.d/12390.bugfix deleted file mode 100644 index 159c6b3c0f..0000000000 --- a/changelog.d/12390.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug introduced in Synapse 1.49.0 which caused the `synapse_event_persisted_position` metric to have invalid values. diff --git a/changelog.d/12391.feature b/changelog.d/12391.feature deleted file mode 100644 index 9a064ec8be..0000000000 --- a/changelog.d/12391.feature +++ /dev/null @@ -1 +0,0 @@ -Add a module API for reading and writing global account data. diff --git a/changelog.d/12392.doc b/changelog.d/12392.doc deleted file mode 100644 index 6a2f522bf6..0000000000 --- a/changelog.d/12392.doc +++ /dev/null @@ -1 +0,0 @@ -Remove documentation for converting a legacy structured logging configuration to the new format. diff --git a/changelog.d/12398.misc b/changelog.d/12398.misc deleted file mode 100644 index f94054bdaa..0000000000 --- a/changelog.d/12398.misc +++ /dev/null @@ -1 +0,0 @@ -Remove support for the unstable identifiers specified in [MSC3288](https://github.com/matrix-org/matrix-doc/pull/3288). diff --git a/changelog.d/12400.misc b/changelog.d/12400.misc deleted file mode 100644 index 8a60f32876..0000000000 --- a/changelog.d/12400.misc +++ /dev/null @@ -1 +0,0 @@ -Make missing `importlib_metadata` dependency explicit. \ No newline at end of file diff --git a/changelog.d/12402.misc b/changelog.d/12402.misc deleted file mode 100644 index 51ed8ad44e..0000000000 --- a/changelog.d/12402.misc +++ /dev/null @@ -1 +0,0 @@ -Add missing type hints to configuration classes. diff --git a/changelog.d/12403.feature b/changelog.d/12403.feature deleted file mode 100644 index 5b55e86ecb..0000000000 --- a/changelog.d/12403.feature +++ /dev/null @@ -1 +0,0 @@ -Support the stable `v1` endpoint for `/relations`, per [MSC2675](https://github.com/matrix-org/matrix-doc/pull/2675). diff --git a/changelog.d/12404.misc b/changelog.d/12404.misc deleted file mode 100644 index 00100fdfef..0000000000 --- a/changelog.d/12404.misc +++ /dev/null @@ -1 +0,0 @@ -Add files used to build the Docker image used for complement testing into the Synapse repository. diff --git a/changelog.d/12405.misc b/changelog.d/12405.misc deleted file mode 100644 index 697b36c2c9..0000000000 --- a/changelog.d/12405.misc +++ /dev/null @@ -1 +0,0 @@ -Fix up healthcheck generation for workers docker image. diff --git a/changelog.d/12407.misc b/changelog.d/12407.misc deleted file mode 100644 index 896da9b74d..0000000000 --- a/changelog.d/12407.misc +++ /dev/null @@ -1 +0,0 @@ -Run the olddeps CI job using Poetry. \ No newline at end of file diff --git a/changelog.d/12408.misc b/changelog.d/12408.misc deleted file mode 100644 index e09a6b8423..0000000000 --- a/changelog.d/12408.misc +++ /dev/null @@ -1 +0,0 @@ -Do not include groups in the sync response when disabled. diff --git a/changelog.d/12409.misc b/changelog.d/12409.misc deleted file mode 100644 index 9f333e718a..0000000000 --- a/changelog.d/12409.misc +++ /dev/null @@ -1 +0,0 @@ -Avoid trying to calculate the state at outlier events. diff --git a/changelog.d/12410.bugfix b/changelog.d/12410.bugfix deleted file mode 100644 index d042425d20..0000000000 --- a/changelog.d/12410.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a spec compliance issue where requests to the `/publicRooms` federation API would specify `limit` as a string. \ No newline at end of file diff --git a/changelog.d/12412.misc b/changelog.d/12412.misc deleted file mode 100644 index 03c90c556a..0000000000 --- a/changelog.d/12412.misc +++ /dev/null @@ -1 +0,0 @@ -Run the CI portdb script in the locked poetry environment. \ No newline at end of file diff --git a/changelog.d/12415.misc b/changelog.d/12415.misc deleted file mode 100644 index 87a5bae572..0000000000 --- a/changelog.d/12415.misc +++ /dev/null @@ -1 +0,0 @@ -Improve type hints related to HTTP query parameters. \ No newline at end of file diff --git a/changelog.d/12418.misc b/changelog.d/12418.misc deleted file mode 100644 index d4b333ce81..0000000000 --- a/changelog.d/12418.misc +++ /dev/null @@ -1 +0,0 @@ -Run the CI export-data script in the locked poetry environment. diff --git a/changelog.d/12420.misc b/changelog.d/12420.misc deleted file mode 100644 index e1965b0713..0000000000 --- a/changelog.d/12420.misc +++ /dev/null @@ -1 +0,0 @@ -Stop maintaining a list of lint targets. diff --git a/changelog.d/12421.misc b/changelog.d/12421.misc deleted file mode 100644 index 3a7cbc34e7..0000000000 --- a/changelog.d/12421.misc +++ /dev/null @@ -1 +0,0 @@ -Make `synapse._scripts` pass type checks. diff --git a/changelog.d/12422.misc b/changelog.d/12422.misc deleted file mode 100644 index 3a7cbc34e7..0000000000 --- a/changelog.d/12422.misc +++ /dev/null @@ -1 +0,0 @@ -Make `synapse._scripts` pass type checks. diff --git a/changelog.d/12423.misc b/changelog.d/12423.misc deleted file mode 100644 index e793d08e5e..0000000000 --- a/changelog.d/12423.misc +++ /dev/null @@ -1 +0,0 @@ -Add some type hints to datastore. \ No newline at end of file diff --git a/changelog.d/12435.misc b/changelog.d/12435.misc deleted file mode 100644 index 3a68452573..0000000000 --- a/changelog.d/12435.misc +++ /dev/null @@ -1 +0,0 @@ -Enable certificate checking during complement tests. diff --git a/changelog.d/12436.feature b/changelog.d/12436.feature deleted file mode 100644 index 883ac15574..0000000000 --- a/changelog.d/12436.feature +++ /dev/null @@ -1,2 +0,0 @@ -Include bundled aggregations in search results -([MSC3666](https://github.com/matrix-org/matrix-spec-proposals/pull/3666)). diff --git a/changelog.d/12439.misc b/changelog.d/12439.misc deleted file mode 100644 index 43bc64ed55..0000000000 --- a/changelog.d/12439.misc +++ /dev/null @@ -1 +0,0 @@ -Bundle locked versions of dependencies into the Docker image. diff --git a/changelog.d/12444.misc b/changelog.d/12444.misc deleted file mode 100644 index 4643a155a9..0000000000 --- a/changelog.d/12444.misc +++ /dev/null @@ -1 +0,0 @@ -Explicitly specify the `tls` extra for Twisted dependency. diff --git a/debian/changelog b/debian/changelog index 903d98af02..67215aa080 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.57.0~rc1) stable; urgency=medium + + * New synapse release 1.57.0~rc1. + + -- Synapse Packaging team Tue, 12 Apr 2022 13:36:25 +0100 + matrix-synapse-py3 (1.56.0) stable; urgency=medium * New synapse release 1.56.0. diff --git a/synapse/__init__.py b/synapse/__init__.py index 2e6510537f..7cdd921366 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -68,7 +68,7 @@ try: except ImportError: pass -__version__ = "1.56.0" +__version__ = "1.57.0rc1" if bool(os.environ.get("SYNAPSE_TEST_PATCH_LOG_CONTEXTS", False)): # We import here so that we don't have to install a bunch of deps when From 3810730ba5e020b4d84edf87887fb65847be0a21 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 12 Apr 2022 13:41:58 +0100 Subject: [PATCH 02/60] Don't line wrap --- CHANGES.md | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 114b1e94ea..39f71fdee0 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,12 +1,7 @@ Synapse 1.57.0rc1 (2022-04-12) ============================== -This version includes a [change](https://github.com/matrix-org/synapse/pull/12209) -to the way transaction IDs are managed for application services. If your -deployment uses a dedicated worker for application service traffic, **it must -be stopped** when the database is upgraded (which normally happens when the -main process is upgraded), to ensure the change is made safely without any risk -of reusing transaction IDs. +This version includes a [change](https://github.com/matrix-org/synapse/pull/12209) to the way transaction IDs are managed for application services. If your deployment uses a dedicated worker for application service traffic, **it must be stopped** when the database is upgraded (which normally happens when the main process is upgraded), to ensure the change is made safely without any risk of reusing transaction IDs. See the [upgrade notes](https://github.com/matrix-org/synapse/blob/develop/docs/upgrade.md#upgrading-to-v1570) for more details. @@ -25,7 +20,7 @@ Features - Update `/messages` to use historic pagination tokens if no `from` query parameter is given. ([\#12370](https://github.com/matrix-org/synapse/issues/12370)) - Add a module API for reading and writing global account data. ([\#12391](https://github.com/matrix-org/synapse/issues/12391)) - Support the stable `v1` endpoint for `/relations`, per [MSC2675](https://github.com/matrix-org/matrix-doc/pull/2675). ([\#12403](https://github.com/matrix-org/synapse/issues/12403)) -- Include bundled aggregations in search results +- Include bundled aggregations in search results ([MSC3666](https://github.com/matrix-org/matrix-spec-proposals/pull/3666)). ([\#12436](https://github.com/matrix-org/synapse/issues/12436)) From 86cf6a3a17376dd83a9222f77521d1619b528d66 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Tue, 12 Apr 2022 08:42:03 -0400 Subject: [PATCH 03/60] Remove references to unstable identifiers from MSC3440. (#12382) Removes references to unstable thread relation, unstable identifiers for filtering parameters, and the experimental config flag. --- changelog.d/12382.removal | 1 + synapse/api/constants.py | 2 - synapse/api/filtering.py | 12 ---- synapse/config/experimental.py | 3 - synapse/events/utils.py | 6 -- synapse/handlers/message.py | 5 +- synapse/rest/client/versions.py | 1 - synapse/server.py | 2 +- synapse/storage/databases/main/events.py | 5 +- synapse/storage/databases/main/relations.py | 78 +++++---------------- tests/api/test_filtering.py | 4 +- 11 files changed, 21 insertions(+), 98 deletions(-) create mode 100644 changelog.d/12382.removal diff --git a/changelog.d/12382.removal b/changelog.d/12382.removal new file mode 100644 index 0000000000..eb91186340 --- /dev/null +++ b/changelog.d/12382.removal @@ -0,0 +1 @@ +Remove unstable identifiers from [MSC3440](https://github.com/matrix-org/matrix-doc/pull/3440). diff --git a/synapse/api/constants.py b/synapse/api/constants.py index 92907415e6..0172eb60b8 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -179,8 +179,6 @@ class RelationTypes: REPLACE: Final = "m.replace" REFERENCE: Final = "m.reference" THREAD: Final = "m.thread" - # TODO Remove this in Synapse >= v1.57.0. - UNSTABLE_THREAD: Final = "io.element.thread" class LimitBlockingTypes: diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index 27e97d6f37..4a808e33fe 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -89,9 +89,7 @@ ROOM_EVENT_FILTER_SCHEMA = { "org.matrix.not_labels": {"type": "array", "items": {"type": "string"}}, # MSC3440, filtering by event relations. "related_by_senders": {"type": "array", "items": {"type": "string"}}, - "io.element.relation_senders": {"type": "array", "items": {"type": "string"}}, "related_by_rel_types": {"type": "array", "items": {"type": "string"}}, - "io.element.relation_types": {"type": "array", "items": {"type": "string"}}, }, } @@ -323,16 +321,6 @@ class Filter: self.related_by_senders = self.filter_json.get("related_by_senders", None) self.related_by_rel_types = self.filter_json.get("related_by_rel_types", None) - # Fallback to the unstable prefix if the stable version is not given. - if hs.config.experimental.msc3440_enabled: - self.related_by_senders = self.related_by_senders or self.filter_json.get( - "io.element.relation_senders", None - ) - self.related_by_rel_types = ( - self.related_by_rel_types - or self.filter_json.get("io.element.relation_types", None) - ) - def filters_all_types(self) -> bool: return "*" in self.not_types diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py index 447476fbfa..0772dce411 100644 --- a/synapse/config/experimental.py +++ b/synapse/config/experimental.py @@ -26,9 +26,6 @@ class ExperimentalConfig(Config): def read_config(self, config: JsonDict, **kwargs: Any) -> None: experimental = config.get("experimental_features") or {} - # MSC3440 (thread relation) - self.msc3440_enabled: bool = experimental.get("msc3440_enabled", False) - # MSC3026 (busy presence state) self.msc3026_enabled: bool = experimental.get("msc3026_enabled", False) diff --git a/synapse/events/utils.py b/synapse/events/utils.py index 918e87ed9c..43c3241fb0 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -39,7 +39,6 @@ from . import EventBase if TYPE_CHECKING: from synapse.handlers.relations import BundledAggregations - from synapse.server import HomeServer # Split strings on "." but not "\." This uses a negative lookbehind assertion for '\' @@ -396,9 +395,6 @@ class EventClientSerializer: clients. """ - def __init__(self, hs: "HomeServer"): - self._msc3440_enabled = hs.config.experimental.msc3440_enabled - def serialize_event( self, event: Union[JsonDict, EventBase], @@ -525,8 +521,6 @@ class EventClientSerializer: "current_user_participated": thread.current_user_participated, } serialized_aggregations[RelationTypes.THREAD] = thread_summary - if self._msc3440_enabled: - serialized_aggregations[RelationTypes.UNSTABLE_THREAD] = thread_summary # Include the bundled aggregations in the event. if serialized_aggregations: diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 7db6905c61..47a63005a9 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -1102,10 +1102,7 @@ class EventCreationHandler: raise SynapseError(400, "Can't send same reaction twice") # Don't attempt to start a thread if the parent event is a relation. - elif ( - relation_type == RelationTypes.THREAD - or relation_type == RelationTypes.UNSTABLE_THREAD - ): + elif relation_type == RelationTypes.THREAD: if await self.store.event_includes_relation(relates_to): raise SynapseError( 400, "Cannot start threads from an event with a relation" diff --git a/synapse/rest/client/versions.py b/synapse/rest/client/versions.py index 9a65aa4843..7ee6b5505b 100644 --- a/synapse/rest/client/versions.py +++ b/synapse/rest/client/versions.py @@ -100,7 +100,6 @@ class VersionsRestServlet(RestServlet): # Adds support for jump to date endpoints (/timestamp_to_event) as per MSC3030 "org.matrix.msc3030": self.config.experimental.msc3030_enabled, # Adds support for thread relations, per MSC3440. - "org.matrix.msc3440": self.config.experimental.msc3440_enabled, "org.matrix.msc3440.stable": True, # TODO: remove when "v1.3" is added above }, }, diff --git a/synapse/server.py b/synapse/server.py index 380369db92..37c72bd83a 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -758,7 +758,7 @@ class HomeServer(metaclass=abc.ABCMeta): @cache_in_self def get_event_client_serializer(self) -> EventClientSerializer: - return EventClientSerializer(self) + return EventClientSerializer() @cache_in_self def get_password_policy_handler(self) -> PasswordPolicyHandler: diff --git a/synapse/storage/databases/main/events.py b/synapse/storage/databases/main/events.py index 3fcd5f5b99..e3be537cee 100644 --- a/synapse/storage/databases/main/events.py +++ b/synapse/storage/databases/main/events.py @@ -1819,10 +1819,7 @@ class PersistEventsStore: if rel_type == RelationTypes.REPLACE: txn.call_after(self.store.get_applicable_edit.invalidate, (parent_id,)) - if ( - rel_type == RelationTypes.THREAD - or rel_type == RelationTypes.UNSTABLE_THREAD - ): + if rel_type == RelationTypes.THREAD: txn.call_after(self.store.get_thread_summary.invalidate, (parent_id,)) # It should be safe to only invalidate the cache if the user has not # previously participated in the thread, but that's difficult (and diff --git a/synapse/storage/databases/main/relations.py b/synapse/storage/databases/main/relations.py index 407158ceee..a5c31f6787 100644 --- a/synapse/storage/databases/main/relations.py +++ b/synapse/storage/databases/main/relations.py @@ -14,7 +14,6 @@ import logging from typing import ( - TYPE_CHECKING, Collection, Dict, FrozenSet, @@ -32,20 +31,12 @@ import attr from synapse.api.constants import RelationTypes from synapse.events import EventBase from synapse.storage._base import SQLBaseStore -from synapse.storage.database import ( - DatabasePool, - LoggingDatabaseConnection, - LoggingTransaction, - make_in_list_sql_clause, -) +from synapse.storage.database import LoggingTransaction, make_in_list_sql_clause from synapse.storage.databases.main.stream import generate_pagination_where_clause from synapse.storage.engines import PostgresEngine from synapse.types import JsonDict, RoomStreamToken, StreamToken from synapse.util.caches.descriptors import cached, cachedList -if TYPE_CHECKING: - from synapse.server import HomeServer - logger = logging.getLogger(__name__) @@ -63,16 +54,6 @@ class _RelatedEvent: class RelationsWorkerStore(SQLBaseStore): - def __init__( - self, - database: DatabasePool, - db_conn: LoggingDatabaseConnection, - hs: "HomeServer", - ): - super().__init__(database, db_conn, hs) - - self._msc3440_enabled = hs.config.experimental.msc3440_enabled - @cached(uncached_args=("event",), tree=True) async def get_relations_for_event( self, @@ -497,7 +478,7 @@ class RelationsWorkerStore(SQLBaseStore): AND parent.room_id = child.room_id WHERE %s - AND %s + AND relation_type = ? ORDER BY parent.event_id, child.topological_ordering DESC, child.stream_ordering DESC """ else: @@ -512,22 +493,16 @@ class RelationsWorkerStore(SQLBaseStore): AND parent.room_id = child.room_id WHERE %s - AND %s + AND relation_type = ? ORDER BY child.topological_ordering DESC, child.stream_ordering DESC """ clause, args = make_in_list_sql_clause( txn.database_engine, "relates_to_id", event_ids ) + args.append(RelationTypes.THREAD) - if self._msc3440_enabled: - relations_clause = "(relation_type = ? OR relation_type = ?)" - args.extend((RelationTypes.THREAD, RelationTypes.UNSTABLE_THREAD)) - else: - relations_clause = "relation_type = ?" - args.append(RelationTypes.THREAD) - - txn.execute(sql % (clause, relations_clause), args) + txn.execute(sql % (clause,), args) latest_event_ids = {} for parent_event_id, child_event_id in txn: # Only consider the latest threaded reply (by topological ordering). @@ -547,7 +522,7 @@ class RelationsWorkerStore(SQLBaseStore): AND parent.room_id = child.room_id WHERE %s - AND %s + AND relation_type = ? GROUP BY parent.event_id """ @@ -556,15 +531,9 @@ class RelationsWorkerStore(SQLBaseStore): clause, args = make_in_list_sql_clause( txn.database_engine, "relates_to_id", latest_event_ids.keys() ) + args.append(RelationTypes.THREAD) - if self._msc3440_enabled: - relations_clause = "(relation_type = ? OR relation_type = ?)" - args.extend((RelationTypes.THREAD, RelationTypes.UNSTABLE_THREAD)) - else: - relations_clause = "relation_type = ?" - args.append(RelationTypes.THREAD) - - txn.execute(sql % (clause, relations_clause), args) + txn.execute(sql % (clause,), args) counts = dict(cast(List[Tuple[str, int]], txn.fetchall())) return counts, latest_event_ids @@ -622,7 +591,7 @@ class RelationsWorkerStore(SQLBaseStore): parent.event_id = relates_to_id AND parent.room_id = child.room_id WHERE - %s + relation_type = ? AND %s AND %s GROUP BY parent.event_id, child.sender @@ -638,16 +607,9 @@ class RelationsWorkerStore(SQLBaseStore): txn.database_engine, "relates_to_id", event_ids ) - if self._msc3440_enabled: - relations_clause = "(relation_type = ? OR relation_type = ?)" - relations_args = [RelationTypes.THREAD, RelationTypes.UNSTABLE_THREAD] - else: - relations_clause = "relation_type = ?" - relations_args = [RelationTypes.THREAD] - txn.execute( - sql % (users_sql, events_clause, relations_clause), - users_args + events_args + relations_args, + sql % (users_sql, events_clause), + [RelationTypes.THREAD] + users_args + events_args, ) return {(row[0], row[1]): row[2] for row in txn} @@ -677,7 +639,7 @@ class RelationsWorkerStore(SQLBaseStore): user participated in that event's thread, otherwise false. """ - def _get_thread_summary_txn(txn: LoggingTransaction) -> Set[str]: + def _get_threads_participated_txn(txn: LoggingTransaction) -> Set[str]: # Fetch whether the requester has participated or not. sql = """ SELECT DISTINCT relates_to_id @@ -688,28 +650,20 @@ class RelationsWorkerStore(SQLBaseStore): AND parent.room_id = child.room_id WHERE %s - AND %s + AND relation_type = ? AND child.sender = ? """ clause, args = make_in_list_sql_clause( txn.database_engine, "relates_to_id", event_ids ) + args.extend([RelationTypes.THREAD, user_id]) - if self._msc3440_enabled: - relations_clause = "(relation_type = ? OR relation_type = ?)" - args.extend((RelationTypes.THREAD, RelationTypes.UNSTABLE_THREAD)) - else: - relations_clause = "relation_type = ?" - args.append(RelationTypes.THREAD) - - args.append(user_id) - - txn.execute(sql % (clause, relations_clause), args) + txn.execute(sql % (clause,), args) return {row[0] for row in txn.fetchall()} participated_threads = await self.db_pool.runInteraction( - "get_thread_summary", _get_thread_summary_txn + "get_threads_participated", _get_threads_participated_txn ) return {event_id: event_id in participated_threads for event_id in event_ids} diff --git a/tests/api/test_filtering.py b/tests/api/test_filtering.py index 8c3354ce3c..985d6e397d 100644 --- a/tests/api/test_filtering.py +++ b/tests/api/test_filtering.py @@ -481,9 +481,7 @@ class FilteringTestCase(unittest.HomeserverTestCase): # events). This is a bit cheeky, but tests the logic of _check_event_relations. # Filter for a particular sender. - definition = { - "io.element.relation_senders": ["@foo:bar"], - } + definition = {"related_by_senders": ["@foo:bar"]} async def events_have_relations(*args, **kwargs): return ["$with_relation"] From e31d06f6f0923d001736f5e6e08616293bd4b120 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 12 Apr 2022 13:54:36 +0100 Subject: [PATCH 04/60] Split changelogs by year --- CHANGES.md | 5754 +---------------- docs/changelogs/CHANGES-2019.md | 1039 +++ docs/changelogs/CHANGES-2020.md | 2145 ++++++ docs/changelogs/CHANGES-2021.md | 2573 ++++++++ .../changelogs/CHANGES-pre-1.0.md | 0 docs/changelogs/README.md | 1 + 6 files changed, 5759 insertions(+), 5753 deletions(-) create mode 100644 docs/changelogs/CHANGES-2019.md create mode 100644 docs/changelogs/CHANGES-2020.md create mode 100644 docs/changelogs/CHANGES-2021.md rename CHANGES-pre-1.0.md => docs/changelogs/CHANGES-pre-1.0.md (100%) create mode 100644 docs/changelogs/README.md diff --git a/CHANGES.md b/CHANGES.md index 39f71fdee0..0db95bad84 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -824,5756 +824,4 @@ Internal Changes - Work around Mjolnir compatibility issue by adding an import for `glob_to_regex` in `synapse.util`, where it moved from. ([\#11696](https://github.com/matrix-org/synapse/issues/11696)) -Synapse 1.49.2 (2021-12-21) -=========================== - -This release fixes a regression introduced in Synapse 1.49.0 which could cause `/sync` requests to take significantly longer. This would particularly affect "initial" syncs for users participating in a large number of rooms, and in extreme cases, could make it impossible for such users to log in on a new client. - -**Note:** in line with our [deprecation policy](https://matrix-org.github.io/synapse/latest/deprecation_policy.html) for platform dependencies, this will be the last release to support Python 3.6 and PostgreSQL 9.6, both of which have now reached upstream end-of-life. Synapse will require Python 3.7+ and PostgreSQL 10+. - -**Note:** We will also stop producing packages for Ubuntu 18.04 (Bionic Beaver) after this release, as it uses Python 3.6. - -Bugfixes --------- - -- Fix a performance regression in `/sync` handling, introduced in 1.49.0. ([\#11583](https://github.com/matrix-org/synapse/issues/11583)) - -Internal Changes ----------------- - -- Work around a build problem on Debian Buster. ([\#11625](https://github.com/matrix-org/synapse/issues/11625)) - - -Synapse 1.49.1 (2021-12-21) -=========================== - -Not released due to problems building the debian packages. - - -Synapse 1.49.0 (2021-12-14) -=========================== - -No significant changes since version 1.49.0rc1. - - -Support for Ubuntu 21.04 ends next month on the 20th of January ---------------------------------------------------------------- - -For users of Ubuntu 21.04 (Hirsute Hippo), please be aware that [upstream support for this version of Ubuntu will end next month][Ubuntu2104EOL]. -We will stop producing packages for Ubuntu 21.04 after upstream support ends. - -[Ubuntu2104EOL]: https://lists.ubuntu.com/archives/ubuntu-announce/2021-December/000275.html - - -The wiki has been migrated to the documentation website -------------------------------------------------------- - -We've decided to move the existing, somewhat stagnant pages from the GitHub wiki -to the [documentation website](https://matrix-org.github.io/synapse/latest/). - -This was done for two reasons. The first was to ensure that changes are checked by -multiple authors before being committed (everyone makes mistakes!) and the second -was visibility of the documentation. Not everyone knows that Synapse has some very -useful information hidden away in its GitHub wiki pages. Bringing them to the -documentation website should help with visibility, as well as keep all Synapse documentation -in one, easily-searchable location. - -Note that contributions to the documentation website happen through [GitHub pull -requests](https://github.com/matrix-org/synapse/pulls). Please visit [#synapse-dev:matrix.org](https://matrix.to/#/#synapse-dev:matrix.org) -if you need help with the process! - - -Synapse 1.49.0rc1 (2021-12-07) -============================== - -Features --------- - -- Add [MSC3030](https://github.com/matrix-org/matrix-doc/pull/3030) experimental client and federation API endpoints to get the closest event to a given timestamp. ([\#9445](https://github.com/matrix-org/synapse/issues/9445)) -- Include bundled relation aggregations during a limited `/sync` request and `/relations` request, per [MSC2675](https://github.com/matrix-org/matrix-doc/pull/2675). ([\#11284](https://github.com/matrix-org/synapse/issues/11284), [\#11478](https://github.com/matrix-org/synapse/issues/11478)) -- Add plugin support for controlling database background updates. ([\#11306](https://github.com/matrix-org/synapse/issues/11306), [\#11475](https://github.com/matrix-org/synapse/issues/11475), [\#11479](https://github.com/matrix-org/synapse/issues/11479)) -- Support the stable API endpoints for [MSC2946](https://github.com/matrix-org/matrix-doc/pull/2946): the room `/hierarchy` endpoint. ([\#11329](https://github.com/matrix-org/synapse/issues/11329)) -- Add admin API to get some information about federation status with remote servers. ([\#11407](https://github.com/matrix-org/synapse/issues/11407)) -- Support expiry of refresh tokens and expiry of the overall session when refresh tokens are in use. ([\#11425](https://github.com/matrix-org/synapse/issues/11425)) -- Stabilise support for [MSC2918](https://github.com/matrix-org/matrix-doc/blob/main/proposals/2918-refreshtokens.md#msc2918-refresh-tokens) refresh tokens as they have now been merged into the Matrix specification. ([\#11435](https://github.com/matrix-org/synapse/issues/11435), [\#11522](https://github.com/matrix-org/synapse/issues/11522)) -- Update [MSC2918 refresh token](https://github.com/matrix-org/matrix-doc/blob/main/proposals/2918-refreshtokens.md#msc2918-refresh-tokens) support to confirm with the latest revision: accept the `refresh_tokens` parameter in the request body rather than in the URL parameters. ([\#11430](https://github.com/matrix-org/synapse/issues/11430)) -- Support configuring the lifetime of non-refreshable access tokens separately to refreshable access tokens. ([\#11445](https://github.com/matrix-org/synapse/issues/11445)) -- Expose `synapse_homeserver` and `synapse_worker` commands as entry points to run Synapse's main process and worker processes, respectively. Contributed by @Ma27. ([\#11449](https://github.com/matrix-org/synapse/issues/11449)) -- `synctl stop` will now wait for Synapse to exit before returning. ([\#11459](https://github.com/matrix-org/synapse/issues/11459), [\#11490](https://github.com/matrix-org/synapse/issues/11490)) -- Extend the "delete room" admin api to work correctly on rooms which have previously been partially deleted. ([\#11523](https://github.com/matrix-org/synapse/issues/11523)) -- Add support for the `/_matrix/client/v3/login/sso/redirect/{idpId}` API from Matrix v1.1. This endpoint was overlooked when support for v3 endpoints was added in Synapse 1.48.0rc1. ([\#11451](https://github.com/matrix-org/synapse/issues/11451)) - - -Bugfixes --------- - -- Fix using [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) batch sending in combination with event persistence workers. Contributed by @tulir at Beeper. ([\#11220](https://github.com/matrix-org/synapse/issues/11220)) -- Fix a long-standing bug where all requests that read events from the database could get stuck as a result of losing the database connection, properly this time. Also fix a race condition introduced in the previous insufficient fix in Synapse 1.47.0. ([\#11376](https://github.com/matrix-org/synapse/issues/11376)) -- The `/send_join` response now includes the stable `event` field instead of the unstable field from [MSC3083](https://github.com/matrix-org/matrix-doc/pull/3083). ([\#11413](https://github.com/matrix-org/synapse/issues/11413)) -- Fix a bug introduced in Synapse 1.47.0 where `send_join` could fail due to an outdated `ijson` version. ([\#11439](https://github.com/matrix-org/synapse/issues/11439), [\#11441](https://github.com/matrix-org/synapse/issues/11441), [\#11460](https://github.com/matrix-org/synapse/issues/11460)) -- Fix a bug introduced in Synapse 1.36.0 which could cause problems fetching event-signing keys from trusted key servers. ([\#11440](https://github.com/matrix-org/synapse/issues/11440)) -- Fix a bug introduced in Synapse 1.47.1 where the media repository would fail to work if the media store path contained any symbolic links. ([\#11446](https://github.com/matrix-org/synapse/issues/11446)) -- Fix an `LruCache` corruption bug, introduced in Synapse 1.38.0, that would cause certain requests to fail until the next Synapse restart. ([\#11454](https://github.com/matrix-org/synapse/issues/11454)) -- Fix a long-standing bug where invites from ignored users were included in incremental syncs. ([\#11511](https://github.com/matrix-org/synapse/issues/11511)) -- Fix a regression in Synapse 1.48.0 where presence workers would not clear their presence updates over replication on shutdown. ([\#11518](https://github.com/matrix-org/synapse/issues/11518)) -- Fix a regression in Synapse 1.48.0 where the module API's `looping_background_call` method would spam errors to the logs when given a non-async function. ([\#11524](https://github.com/matrix-org/synapse/issues/11524)) - - -Updates to the Docker image ---------------------------- - -- Update `Dockerfile-workers` to healthcheck all workers in the container. ([\#11429](https://github.com/matrix-org/synapse/issues/11429)) - - -Improved Documentation ----------------------- - -- Update the media repository documentation. ([\#11415](https://github.com/matrix-org/synapse/issues/11415)) -- Update section about backward extremities in the room DAG concepts doc to correct the misconception about backward extremities indicating whether we have fetched an events' `prev_events`. ([\#11469](https://github.com/matrix-org/synapse/issues/11469)) - - -Internal Changes ----------------- - -- Add `Final` annotation to string constants in `synapse.api.constants` so that they get typed as `Literal`s. ([\#11356](https://github.com/matrix-org/synapse/issues/11356)) -- Add a check to ensure that users cannot start the Synapse master process when `worker_app` is set. ([\#11416](https://github.com/matrix-org/synapse/issues/11416)) -- Add a note about postgres memory management and hugepages to postgres doc. ([\#11467](https://github.com/matrix-org/synapse/issues/11467)) -- Add missing type hints to `synapse.config` module. ([\#11465](https://github.com/matrix-org/synapse/issues/11465)) -- Add missing type hints to `synapse.federation`. ([\#11483](https://github.com/matrix-org/synapse/issues/11483)) -- Add type annotations to `tests.storage.test_appservice`. ([\#11488](https://github.com/matrix-org/synapse/issues/11488), [\#11492](https://github.com/matrix-org/synapse/issues/11492)) -- Add type annotations to some of the configuration surrounding refresh tokens. ([\#11428](https://github.com/matrix-org/synapse/issues/11428)) -- Add type hints to `synapse/tests/rest/admin`. ([\#11501](https://github.com/matrix-org/synapse/issues/11501)) -- Add type hints to storage classes. ([\#11411](https://github.com/matrix-org/synapse/issues/11411)) -- Add wiki pages to documentation website. ([\#11402](https://github.com/matrix-org/synapse/issues/11402)) -- Clean up `tests.storage.test_main` to remove use of legacy code. ([\#11493](https://github.com/matrix-org/synapse/issues/11493)) -- Clean up `tests.test_visibility` to remove legacy code. ([\#11495](https://github.com/matrix-org/synapse/issues/11495)) -- Convert status codes to `HTTPStatus` in `synapse.rest.admin`. ([\#11452](https://github.com/matrix-org/synapse/issues/11452), [\#11455](https://github.com/matrix-org/synapse/issues/11455)) -- Extend the `scripts-dev/sign_json` script to support signing events. ([\#11486](https://github.com/matrix-org/synapse/issues/11486)) -- Improve internal types in push code. ([\#11409](https://github.com/matrix-org/synapse/issues/11409)) -- Improve type annotations in `synapse.module_api`. ([\#11029](https://github.com/matrix-org/synapse/issues/11029)) -- Improve type hints for `LruCache`. ([\#11453](https://github.com/matrix-org/synapse/issues/11453)) -- Preparation for database schema simplifications: disambiguate queries on `state_key`. ([\#11497](https://github.com/matrix-org/synapse/issues/11497)) -- Refactor `backfilled` into specific behavior function arguments (`_persist_events_and_state_updates` and downstream calls). ([\#11417](https://github.com/matrix-org/synapse/issues/11417)) -- Refactor `get_version_string` to fix-up types and duplicated code. ([\#11468](https://github.com/matrix-org/synapse/issues/11468)) -- Refactor various parts of the `/sync` handler. ([\#11494](https://github.com/matrix-org/synapse/issues/11494), [\#11515](https://github.com/matrix-org/synapse/issues/11515)) -- Remove unnecessary `json.dumps` from `tests.rest.admin`. ([\#11461](https://github.com/matrix-org/synapse/issues/11461)) -- Save the OpenID Connect session ID on login. ([\#11482](https://github.com/matrix-org/synapse/issues/11482)) -- Update and clean up recently ported documentation pages. ([\#11466](https://github.com/matrix-org/synapse/issues/11466)) - - -Synapse 1.48.0 (2021-11-30) -=========================== - -This release removes support for the long-deprecated `trust_identity_server_for_password_resets` configuration flag. - -This release also fixes some performance issues with some background database updates introduced in Synapse 1.47.0. - -No significant changes since 1.48.0rc1. - -Synapse 1.48.0rc1 (2021-11-25) -============================== - -Features --------- - -- Experimental support for the thread relation defined in [MSC3440](https://github.com/matrix-org/matrix-doc/pull/3440). ([\#11161](https://github.com/matrix-org/synapse/issues/11161)) -- Support filtering by relation senders & types per [MSC3440](https://github.com/matrix-org/matrix-doc/pull/3440). ([\#11236](https://github.com/matrix-org/synapse/issues/11236)) -- Add support for the `/_matrix/client/v3` and `/_matrix/media/v3` APIs from Matrix v1.1. ([\#11318](https://github.com/matrix-org/synapse/issues/11318), [\#11371](https://github.com/matrix-org/synapse/issues/11371)) -- Support the stable version of [MSC2778](https://github.com/matrix-org/matrix-doc/pull/2778): the `m.login.application_service` login type. Contributed by @tulir. ([\#11335](https://github.com/matrix-org/synapse/issues/11335)) -- Add a new version of delete room admin API `DELETE /_synapse/admin/v2/rooms/` to run it in the background. Contributed by @dklimpel. ([\#11223](https://github.com/matrix-org/synapse/issues/11223)) -- Allow the admin [Delete Room API](https://matrix-org.github.io/synapse/latest/admin_api/rooms.html#delete-room-api) to block a room without the need to join it. ([\#11228](https://github.com/matrix-org/synapse/issues/11228)) -- Add an admin API to un-shadow-ban a user. ([\#11347](https://github.com/matrix-org/synapse/issues/11347)) -- Add an admin API to run background database schema updates. ([\#11352](https://github.com/matrix-org/synapse/issues/11352)) -- Add an admin API for blocking a room. ([\#11324](https://github.com/matrix-org/synapse/issues/11324)) -- Update the JWT login type to support custom a `sub` claim. ([\#11361](https://github.com/matrix-org/synapse/issues/11361)) -- Store and allow querying of arbitrary event relations. ([\#11391](https://github.com/matrix-org/synapse/issues/11391)) - - -Bugfixes --------- - -- Fix a long-standing bug wherein display names or avatar URLs containing null bytes cause an internal server error when stored in the DB. ([\#11230](https://github.com/matrix-org/synapse/issues/11230)) -- Prevent [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) historical state events from being pushed to an application service via `/transactions`. ([\#11265](https://github.com/matrix-org/synapse/issues/11265)) -- Fix a long-standing bug where uploading extremely thin images (e.g. 1000x1) would fail. Contributed by @Neeeflix. ([\#11288](https://github.com/matrix-org/synapse/issues/11288)) -- Fix a bug, introduced in Synapse 1.46.0, which caused the `check_3pid_auth` and `on_logged_out` callbacks in legacy password authentication provider modules to not be registered. Modules using the generic module interface were not affected. ([\#11340](https://github.com/matrix-org/synapse/issues/11340)) -- Fix a bug introduced in 1.41.0 where space hierarchy responses would be incorrectly reused if multiple users were to make the same request at the same time. ([\#11355](https://github.com/matrix-org/synapse/issues/11355)) -- Fix a bug introduced in 1.45.0 where the `read_templates` method of the module API would error. ([\#11377](https://github.com/matrix-org/synapse/issues/11377)) -- Fix an issue introduced in 1.47.0 which prevented servers re-joining rooms they had previously left, if their signing keys were replaced. ([\#11379](https://github.com/matrix-org/synapse/issues/11379)) -- Fix a bug introduced in 1.13.0 where creating and publishing a room could cause errors if `room_list_publication_rules` is configured. ([\#11392](https://github.com/matrix-org/synapse/issues/11392)) -- Improve performance of various background database updates. ([\#11421](https://github.com/matrix-org/synapse/issues/11421), [\#11422](https://github.com/matrix-org/synapse/issues/11422)) - - -Improved Documentation ----------------------- - -- Suggest users of the Debian packages add configuration to `/etc/matrix-synapse/conf.d/` to prevent, upon upgrade, being asked to choose between their configuration and the maintainer's. ([\#11281](https://github.com/matrix-org/synapse/issues/11281)) -- Fix typos in the documentation for the `username_available` admin API. Contributed by Stanislav Motylkov. ([\#11286](https://github.com/matrix-org/synapse/issues/11286)) -- Add Single Sign-On, SAML and CAS pages to the documentation. ([\#11298](https://github.com/matrix-org/synapse/issues/11298)) -- Change the word 'Home server' as one word 'homeserver' in documentation. ([\#11320](https://github.com/matrix-org/synapse/issues/11320)) -- Fix missing quotes for wildcard domains in `federation_certificate_verification_whitelist`. ([\#11381](https://github.com/matrix-org/synapse/issues/11381)) - - -Deprecations and Removals -------------------------- - -- Remove deprecated `trust_identity_server_for_password_resets` configuration flag. ([\#11333](https://github.com/matrix-org/synapse/issues/11333), [\#11395](https://github.com/matrix-org/synapse/issues/11395)) - - -Internal Changes ----------------- - -- Add type annotations to `synapse.metrics`. ([\#10847](https://github.com/matrix-org/synapse/issues/10847)) -- Split out federated PDU retrieval function into a non-cached version. ([\#11242](https://github.com/matrix-org/synapse/issues/11242)) -- Clean up code relating to to-device messages and sending ephemeral events to application services. ([\#11247](https://github.com/matrix-org/synapse/issues/11247)) -- Fix a small typo in the error response when a relation type other than 'm.annotation' is passed to `GET /rooms/{room_id}/aggregations/{event_id}`. ([\#11278](https://github.com/matrix-org/synapse/issues/11278)) -- Drop unused database tables `room_stats_historical` and `user_stats_historical`. ([\#11280](https://github.com/matrix-org/synapse/issues/11280)) -- Require all files in synapse/ and tests/ to pass mypy unless specifically excluded. ([\#11282](https://github.com/matrix-org/synapse/issues/11282), [\#11285](https://github.com/matrix-org/synapse/issues/11285), [\#11359](https://github.com/matrix-org/synapse/issues/11359)) -- Add missing type hints to `synapse.app`. ([\#11287](https://github.com/matrix-org/synapse/issues/11287)) -- Remove unused parameters on `FederationEventHandler._check_event_auth`. ([\#11292](https://github.com/matrix-org/synapse/issues/11292)) -- Add type hints to `synapse._scripts`. ([\#11297](https://github.com/matrix-org/synapse/issues/11297)) -- Fix an issue which prevented the `remove_deleted_devices_from_device_inbox` background database schema update from running when updating from a recent Synapse version. ([\#11303](https://github.com/matrix-org/synapse/issues/11303)) -- Add type hints to storage classes. ([\#11307](https://github.com/matrix-org/synapse/issues/11307), [\#11310](https://github.com/matrix-org/synapse/issues/11310), [\#11311](https://github.com/matrix-org/synapse/issues/11311), [\#11312](https://github.com/matrix-org/synapse/issues/11312), [\#11313](https://github.com/matrix-org/synapse/issues/11313), [\#11314](https://github.com/matrix-org/synapse/issues/11314), [\#11316](https://github.com/matrix-org/synapse/issues/11316), [\#11322](https://github.com/matrix-org/synapse/issues/11322), [\#11332](https://github.com/matrix-org/synapse/issues/11332), [\#11339](https://github.com/matrix-org/synapse/issues/11339), [\#11342](https://github.com/matrix-org/synapse/issues/11342)) -- Add type hints to `synapse.util`. ([\#11321](https://github.com/matrix-org/synapse/issues/11321), [\#11328](https://github.com/matrix-org/synapse/issues/11328)) -- Improve type annotations in Synapse's test suite. ([\#11323](https://github.com/matrix-org/synapse/issues/11323), [\#11330](https://github.com/matrix-org/synapse/issues/11330)) -- Test that room alias deletion works as intended. ([\#11327](https://github.com/matrix-org/synapse/issues/11327)) -- Add type annotations for some methods and properties in the module API. ([\#11341](https://github.com/matrix-org/synapse/issues/11341)) -- Fix running `scripts-dev/complement.sh`, which was broken in v1.47.0rc1. ([\#11368](https://github.com/matrix-org/synapse/issues/11368)) -- Rename internal functions for token generation to better reflect what they do. ([\#11369](https://github.com/matrix-org/synapse/issues/11369), [\#11370](https://github.com/matrix-org/synapse/issues/11370)) -- Add type hints to configuration classes. ([\#11377](https://github.com/matrix-org/synapse/issues/11377)) -- Publish a `develop` image to Docker Hub. ([\#11380](https://github.com/matrix-org/synapse/issues/11380)) -- Keep fallback key marked as used if it's re-uploaded. ([\#11382](https://github.com/matrix-org/synapse/issues/11382)) -- Use `auto_attribs` on the `attrs` class `RefreshTokenLookupResult`. ([\#11386](https://github.com/matrix-org/synapse/issues/11386)) -- Rename unstable `access_token_lifetime` configuration option to `refreshable_access_token_lifetime` to make it clear it only concerns refreshable access tokens. ([\#11388](https://github.com/matrix-org/synapse/issues/11388)) -- Do not run the broken MSC2716 tests when running `scripts-dev/complement.sh`. ([\#11389](https://github.com/matrix-org/synapse/issues/11389)) -- Remove dead code from supporting ACME. ([\#11393](https://github.com/matrix-org/synapse/issues/11393)) -- Refactor including the bundled relations when serializing an event. ([\#11408](https://github.com/matrix-org/synapse/issues/11408)) - - -Synapse 1.47.1 (2021-11-23) -=========================== - -This release fixes a security issue in the media store, affecting all prior releases of Synapse. Server administrators are encouraged to update Synapse as soon as possible. We are not aware of these vulnerabilities being exploited in the wild. - -Server administrators who are unable to update Synapse may use the workarounds described in the linked GitHub Security Advisory below. - -Security advisory ------------------ - -The following issue is fixed in 1.47.1. - -- **[GHSA-3hfw-x7gx-437c](https://github.com/matrix-org/synapse/security/advisories/GHSA-3hfw-x7gx-437c) / [CVE-2021-41281](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-41281): Path traversal when downloading remote media.** - - Synapse instances with the media repository enabled can be tricked into downloading a file from a remote server into an arbitrary directory, potentially outside the media store directory. - - The last two directories and file name of the path are chosen randomly by Synapse and cannot be controlled by an attacker, which limits the impact. - - Homeservers with the media repository disabled are unaffected. Homeservers configured with a federation whitelist are also unaffected. - - Fixed by [91f2bd090](https://github.com/matrix-org/synapse/commit/91f2bd090). - - -Synapse 1.47.0 (2021-11-17) -=========================== - -No significant changes since 1.47.0rc3. - - -Synapse 1.47.0rc3 (2021-11-16) -============================== - -Bugfixes --------- - -- Fix a bug introduced in 1.47.0rc1 which caused worker processes to not halt startup in the presence of outstanding database migrations. ([\#11346](https://github.com/matrix-org/synapse/issues/11346)) -- Fix a bug introduced in 1.47.0rc1 which prevented the 'remove deleted devices from `device_inbox` column' background process from running when updating from a recent Synapse version. ([\#11303](https://github.com/matrix-org/synapse/issues/11303), [\#11353](https://github.com/matrix-org/synapse/issues/11353)) - - -Synapse 1.47.0rc2 (2021-11-10) -============================== - -This fixes an issue with publishing the Debian packages for 1.47.0rc1. -It is otherwise identical to 1.47.0rc1. - - -Synapse 1.47.0rc1 (2021-11-09) -============================== - -Deprecations and Removals -------------------------- - -- The `user_may_create_room_with_invites` module callback is now deprecated. Please refer to the [upgrade notes](https://matrix-org.github.io/synapse/develop/upgrade#upgrading-to-v1470) for more information. ([\#11206](https://github.com/matrix-org/synapse/issues/11206)) -- Remove deprecated admin API to delete rooms (`POST /_synapse/admin/v1/rooms//delete`). ([\#11213](https://github.com/matrix-org/synapse/issues/11213)) - - -Features --------- - -- Advertise support for Client-Server API r0.6.1. ([\#11097](https://github.com/matrix-org/synapse/issues/11097)) -- Add search by room ID and room alias to the List Room admin API. ([\#11099](https://github.com/matrix-org/synapse/issues/11099)) -- Add an `on_new_event` third-party rules callback to allow Synapse modules to act after an event has been sent into a room. ([\#11126](https://github.com/matrix-org/synapse/issues/11126)) -- Add a module API method to update a user's membership in a room. ([\#11147](https://github.com/matrix-org/synapse/issues/11147)) -- Add metrics for thread pool usage. ([\#11178](https://github.com/matrix-org/synapse/issues/11178)) -- Support the stable room type field for [MSC3288](https://github.com/matrix-org/matrix-doc/pull/3288). ([\#11187](https://github.com/matrix-org/synapse/issues/11187)) -- Add a module API method to retrieve the current state of a room. ([\#11204](https://github.com/matrix-org/synapse/issues/11204)) -- Calculate a default value for `public_baseurl` based on `server_name`. ([\#11210](https://github.com/matrix-org/synapse/issues/11210)) -- Add support for serving `/.well-known/matrix/server` files, to redirect federation traffic to port 443. ([\#11211](https://github.com/matrix-org/synapse/issues/11211)) -- Add admin APIs to pause, start and check the status of background updates. ([\#11263](https://github.com/matrix-org/synapse/issues/11263)) - - -Bugfixes --------- - -- Fix a long-standing bug which allowed hidden devices to receive to-device messages, resulting in unnecessary database bloat. ([\#10097](https://github.com/matrix-org/synapse/issues/10097)) -- Fix a long-standing bug where messages in the `device_inbox` table for deleted devices would persist indefinitely. Contributed by @dklimpel and @JohannesKleine. ([\#10969](https://github.com/matrix-org/synapse/issues/10969), [\#11212](https://github.com/matrix-org/synapse/issues/11212)) -- Do not accept events if a third-party rule `check_event_allowed` callback raises an exception. ([\#11033](https://github.com/matrix-org/synapse/issues/11033)) -- Fix long-standing bug where verification requests could fail in certain cases if a federation whitelist was in place but did not include your own homeserver. ([\#11129](https://github.com/matrix-org/synapse/issues/11129)) -- Allow an empty list of `state_events_at_start` to be sent when using the [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) `/batch_send` endpoint and the author of the historical messages is already part of the current room state at the given `?prev_event_id`. ([\#11188](https://github.com/matrix-org/synapse/issues/11188)) -- Fix a bug introduced in Synapse 1.45.0 which prevented the `synapse_review_recent_signups` script from running. Contributed by @samuel-p. ([\#11191](https://github.com/matrix-org/synapse/issues/11191)) -- Delete `to_device` messages for hidden devices that will never be read, reducing database size. ([\#11199](https://github.com/matrix-org/synapse/issues/11199)) -- Fix a long-standing bug wherein a missing `Content-Type` header when downloading remote media would cause Synapse to throw an error. ([\#11200](https://github.com/matrix-org/synapse/issues/11200)) -- Fix a long-standing bug which could result in serialization errors and potentially duplicate transaction data when sending ephemeral events to application services. Contributed by @Fizzadar at Beeper. ([\#11207](https://github.com/matrix-org/synapse/issues/11207)) -- Fix a bug introduced in Synapse 1.35.0 which made it impossible to join rooms that return a `send_join` response containing floats. ([\#11217](https://github.com/matrix-org/synapse/issues/11217)) -- Fix long-standing bug where cross signing keys were not included in the response to `/r0/keys/query` the first time a remote user was queried. ([\#11234](https://github.com/matrix-org/synapse/issues/11234)) -- Fix a long-standing bug where all requests that read events from the database could get stuck as a result of losing the database connection. ([\#11240](https://github.com/matrix-org/synapse/issues/11240)) -- Fix a bug preventing Synapse from being rolled back to an earlier version when using workers. ([\#11255](https://github.com/matrix-org/synapse/issues/11255), [\#11276](https://github.com/matrix-org/synapse/issues/11276)) -- Fix a bug introduced in Synapse 1.37.1 which caused a remote event being processed by a worker to not get processed on restart if the worker was killed. ([\#11262](https://github.com/matrix-org/synapse/issues/11262)) -- Only allow old Element/Riot Android clients to send read receipts without a request body. All other clients must include a request body as required by the specification. Contributed by @rogersheu. ([\#11157](https://github.com/matrix-org/synapse/issues/11157)) - - -Updates to the Docker image ---------------------------- - -- Avoid changing user ID when started as a non-root user, and no explicit `UID` is set. ([\#11209](https://github.com/matrix-org/synapse/issues/11209)) - - -Improved Documentation ----------------------- - -- Improve example HAProxy config in the docs to properly handle HTTP `Host` headers with port information. This is required for federation over port 443 to work correctly. ([\#11128](https://github.com/matrix-org/synapse/issues/11128)) -- Add documentation for using Authentik as an OpenID Connect Identity Provider. Contributed by @samip5. ([\#11151](https://github.com/matrix-org/synapse/issues/11151)) -- Clarify lack of support for Windows. ([\#11198](https://github.com/matrix-org/synapse/issues/11198)) -- Improve code formatting and fix a few typos in docs. Contributed by @sumnerevans at Beeper. ([\#11221](https://github.com/matrix-org/synapse/issues/11221)) -- Add documentation for using LemonLDAP as an OpenID Connect Identity Provider. Contributed by @l00ptr. ([\#11257](https://github.com/matrix-org/synapse/issues/11257)) - - -Internal Changes ----------------- - -- Add type annotations for the `log_function` decorator. ([\#10943](https://github.com/matrix-org/synapse/issues/10943)) -- Add type hints to `synapse.events`. ([\#11098](https://github.com/matrix-org/synapse/issues/11098)) -- Remove and document unnecessary `RoomStreamToken` checks in application service ephemeral event code. ([\#11137](https://github.com/matrix-org/synapse/issues/11137)) -- Add type hints so that `synapse.http` passes `mypy` checks. ([\#11164](https://github.com/matrix-org/synapse/issues/11164)) -- Update scripts to pass Shellcheck lints. ([\#11166](https://github.com/matrix-org/synapse/issues/11166)) -- Add knock information in admin export. Contributed by Rafael Gonçalves. ([\#11171](https://github.com/matrix-org/synapse/issues/11171)) -- Add tests to check that `ClientIpStore.get_last_client_ip_by_device` and `get_user_ip_and_agents` combine database and in-memory data correctly. ([\#11179](https://github.com/matrix-org/synapse/issues/11179)) -- Refactor `Filter` to check different fields depending on the data type. ([\#11194](https://github.com/matrix-org/synapse/issues/11194)) -- Improve type hints for the relations datastore. ([\#11205](https://github.com/matrix-org/synapse/issues/11205)) -- Replace outdated links in the pull request checklist with links to the rendered documentation. ([\#11225](https://github.com/matrix-org/synapse/issues/11225)) -- Fix a bug in unit test `test_block_room_and_not_purge`. ([\#11226](https://github.com/matrix-org/synapse/issues/11226)) -- In `ObservableDeferred`, run observers in the order they were registered. ([\#11229](https://github.com/matrix-org/synapse/issues/11229)) -- Minor speed up to start up times and getting updates for groups by adding missing index to `local_group_updates.stream_id`. ([\#11231](https://github.com/matrix-org/synapse/issues/11231)) -- Add `twine` and `towncrier` as dev dependencies, as they're used by the release script. ([\#11233](https://github.com/matrix-org/synapse/issues/11233)) -- Allow `stream_writers.typing` config to be a list of one worker. ([\#11237](https://github.com/matrix-org/synapse/issues/11237)) -- Remove debugging statement in tests. ([\#11239](https://github.com/matrix-org/synapse/issues/11239)) -- Fix [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) historical messages backfilling in random order on remote homeservers. ([\#11244](https://github.com/matrix-org/synapse/issues/11244)) -- Add an additional test for the `cachedList` method decorator. ([\#11246](https://github.com/matrix-org/synapse/issues/11246)) -- Make minor correction to the type of `auth_checkers` callbacks. ([\#11253](https://github.com/matrix-org/synapse/issues/11253)) -- Clean up trivial aspects of the Debian package build tooling. ([\#11269](https://github.com/matrix-org/synapse/issues/11269), [\#11273](https://github.com/matrix-org/synapse/issues/11273)) -- Blacklist new SyTest that checks that key uploads are valid pending the validation being implemented in Synapse. ([\#11270](https://github.com/matrix-org/synapse/issues/11270)) - - -Synapse 1.46.0 (2021-11-02) -=========================== - -The cause of the [performance regression affecting Synapse 1.44](https://github.com/matrix-org/synapse/issues/11049) has been identified and fixed. ([\#11177](https://github.com/matrix-org/synapse/issues/11177)) - -Bugfixes --------- - -- Fix a bug introduced in v1.46.0rc1 where URL previews of some XML documents would fail. ([\#11196](https://github.com/matrix-org/synapse/issues/11196)) - - -Synapse 1.46.0rc1 (2021-10-27) -============================== - -Features --------- - -- Add support for Ubuntu 21.10 "Impish Indri". ([\#11024](https://github.com/matrix-org/synapse/issues/11024)) -- Port the Password Auth Providers module interface to the new generic interface. ([\#10548](https://github.com/matrix-org/synapse/issues/10548), [\#11180](https://github.com/matrix-org/synapse/issues/11180)) -- Experimental support for the thread relation defined in [MSC3440](https://github.com/matrix-org/matrix-doc/pull/3440). ([\#11088](https://github.com/matrix-org/synapse/issues/11088), [\#11181](https://github.com/matrix-org/synapse/issues/11181), [\#11192](https://github.com/matrix-org/synapse/issues/11192)) -- Users admin API can now also modify user type in addition to allowing it to be set on user creation. ([\#11174](https://github.com/matrix-org/synapse/issues/11174)) - - -Bugfixes --------- - -- Newly-created public rooms are now only assigned an alias if the room's creation has not been blocked by permission settings. Contributed by @AndrewFerr. ([\#10930](https://github.com/matrix-org/synapse/issues/10930)) -- Fix a long-standing bug which meant that events received over federation were sometimes incorrectly accepted into the room state. ([\#11001](https://github.com/matrix-org/synapse/issues/11001), [\#11009](https://github.com/matrix-org/synapse/issues/11009), [\#11012](https://github.com/matrix-org/synapse/issues/11012)) -- Fix 500 error on `/messages` when the server accumulates more than 5 backwards extremities at a given depth for a room. ([\#11027](https://github.com/matrix-org/synapse/issues/11027)) -- Fix a bug where setting a user's `external_id` via the admin API returns 500 and deletes user's existing external mappings if that external ID is already mapped. ([\#11051](https://github.com/matrix-org/synapse/issues/11051)) -- Fix a long-standing bug where users excluded from the user directory were added into the directory if they belonged to a room which became public or private. ([\#11075](https://github.com/matrix-org/synapse/issues/11075)) -- Fix a long-standing bug when attempting to preview URLs which are in the `windows-1252` character encoding. ([\#11077](https://github.com/matrix-org/synapse/issues/11077), [\#11089](https://github.com/matrix-org/synapse/issues/11089)) -- Fix broken export-data admin command and add test script checking the command to CI. ([\#11078](https://github.com/matrix-org/synapse/issues/11078)) -- Show an error when timestamp in seconds is provided to the `/purge_media_cache` Admin API. ([\#11101](https://github.com/matrix-org/synapse/issues/11101)) -- Fix local users who left all their rooms being removed from the user directory, even if the `search_all_users` config option was enabled. ([\#11103](https://github.com/matrix-org/synapse/issues/11103)) -- Fix a bug which caused the module API's `get_user_ip_and_agents` function to always fail on workers. `get_user_ip_and_agents` was introduced in 1.44.0 and did not function correctly on worker processes at the time. ([\#11112](https://github.com/matrix-org/synapse/issues/11112)) -- Identity server connection is no longer ignoring `ip_range_whitelist`. ([\#11120](https://github.com/matrix-org/synapse/issues/11120)) -- Fix a bug introduced in Synapse 1.45.0 breaking the configuration file parsing script. ([\#11145](https://github.com/matrix-org/synapse/issues/11145)) -- Fix a performance regression introduced in 1.44.0 which could cause client requests to time out when making large numbers of outbound requests. ([\#11177](https://github.com/matrix-org/synapse/issues/11177), [\#11190](https://github.com/matrix-org/synapse/issues/11190)) -- Resolve and share `state_groups` for all [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) historical events in batch. ([\#10975](https://github.com/matrix-org/synapse/issues/10975)) - - -Improved Documentation ----------------------- - -- Fix broken links relating to module API deprecation in the upgrade notes. ([\#11069](https://github.com/matrix-org/synapse/issues/11069)) -- Add more information about what happens when a user is deactivated. ([\#11083](https://github.com/matrix-org/synapse/issues/11083)) -- Clarify the the sample log config can be copied from the documentation without issue. ([\#11092](https://github.com/matrix-org/synapse/issues/11092)) -- Update the admin API documentation with an updated list of the characters allowed in registration tokens. ([\#11093](https://github.com/matrix-org/synapse/issues/11093)) -- Document Synapse's behaviour when dealing with multiple modules registering the same callbacks and/or handlers for the same HTTP endpoints. ([\#11096](https://github.com/matrix-org/synapse/issues/11096)) -- Fix instances of `[example]{.title-ref}` in the upgrade documentation as a result of prior RST to Markdown conversion. ([\#11118](https://github.com/matrix-org/synapse/issues/11118)) -- Document the version of Synapse each module callback was introduced in. ([\#11132](https://github.com/matrix-org/synapse/issues/11132)) -- Document the version of Synapse that introduced each module API method. ([\#11183](https://github.com/matrix-org/synapse/issues/11183)) - - -Internal Changes ----------------- -- Fix spurious warnings about losing the logging context on the `ReplicationCommandHandler` when losing the replication connection. ([\#10984](https://github.com/matrix-org/synapse/issues/10984)) -- Include rejected status when we log events. ([\#11008](https://github.com/matrix-org/synapse/issues/11008)) -- Add some extra logging to the event persistence code. ([\#11014](https://github.com/matrix-org/synapse/issues/11014)) -- Rearrange the internal workings of the incremental user directory updates. ([\#11035](https://github.com/matrix-org/synapse/issues/11035)) -- Fix a long-standing bug where users excluded from the directory could still be added to the `users_who_share_private_rooms` table after a regular user joins a private room. ([\#11143](https://github.com/matrix-org/synapse/issues/11143)) -- Add and improve type hints. ([\#10972](https://github.com/matrix-org/synapse/issues/10972), [\#11055](https://github.com/matrix-org/synapse/issues/11055), [\#11066](https://github.com/matrix-org/synapse/issues/11066), [\#11076](https://github.com/matrix-org/synapse/issues/11076), [\#11095](https://github.com/matrix-org/synapse/issues/11095), [\#11109](https://github.com/matrix-org/synapse/issues/11109), [\#11121](https://github.com/matrix-org/synapse/issues/11121), [\#11146](https://github.com/matrix-org/synapse/issues/11146)) -- Mark the Synapse package as containing type annotations and fix export declarations so that Synapse pluggable modules may be type checked against Synapse. ([\#11054](https://github.com/matrix-org/synapse/issues/11054)) -- Remove dead code from `MediaFilePaths`. ([\#11056](https://github.com/matrix-org/synapse/issues/11056)) -- Be more lenient when parsing oEmbed response versions. ([\#11065](https://github.com/matrix-org/synapse/issues/11065)) -- Create a separate module for the retention configuration. ([\#11070](https://github.com/matrix-org/synapse/issues/11070)) -- Clean up some of the federation event authentication code for clarity. ([\#11115](https://github.com/matrix-org/synapse/issues/11115), [\#11116](https://github.com/matrix-org/synapse/issues/11116), [\#11122](https://github.com/matrix-org/synapse/issues/11122)) -- Add docstrings and comments to the application service ephemeral event sending code. ([\#11138](https://github.com/matrix-org/synapse/issues/11138)) -- Update the `sign_json` script to support inline configuration of the signing key. ([\#11139](https://github.com/matrix-org/synapse/issues/11139)) -- Fix broken link in the docker image README. ([\#11144](https://github.com/matrix-org/synapse/issues/11144)) -- Always dump logs from unit tests during CI runs. ([\#11068](https://github.com/matrix-org/synapse/issues/11068)) -- Add tests for `MediaFilePaths` class. ([\#11057](https://github.com/matrix-org/synapse/issues/11057)) -- Simplify the user admin API tests. ([\#11048](https://github.com/matrix-org/synapse/issues/11048)) -- Add a test for the workaround introduced in [\#11042](https://github.com/matrix-org/synapse/pull/11042) concerning the behaviour of third-party rule modules and `SynapseError`s. ([\#11071](https://github.com/matrix-org/synapse/issues/11071)) - - -Synapse 1.45.1 (2021-10-20) -=========================== - -Bugfixes --------- - -- Revert change to counting of deactivated users towards the monthly active users limit, introduced in 1.45.0rc1. ([\#11127](https://github.com/matrix-org/synapse/issues/11127)) - - -Synapse 1.45.0 (2021-10-19) -=========================== - -No functional changes since Synapse 1.45.0rc2. - -Known Issues ------------- - -- A suspected [performance regression](https://github.com/matrix-org/synapse/issues/11049) which was first reported after the release of 1.44.0 remains unresolved. - - We have not been able to identify a probable cause. Affected users report that setting up a federation sender worker appears to alleviate symptoms of the regression. - -Improved Documentation ----------------------- - -- Reword changelog to clarify concerns about a suspected performance regression in 1.44.0. ([\#11117](https://github.com/matrix-org/synapse/issues/11117)) - - -Synapse 1.45.0rc2 (2021-10-14) -============================== - -This release candidate [fixes](https://github.com/matrix-org/synapse/issues/11053) a user directory [bug](https://github.com/matrix-org/synapse/issues/11025) present in 1.45.0rc1. - -Known Issues ------------- - -- A suspected [performance regression](https://github.com/matrix-org/synapse/issues/11049) which was first reported after the release of 1.44.0 remains unresolved. - - We have not been able to identify a probable cause. Affected users report that setting up a federation sender worker appears to alleviate symptoms of the regression. - -Bugfixes --------- - -- Fix a long-standing bug when using multiple event persister workers where events were not correctly sent down `/sync` due to a race. ([\#11045](https://github.com/matrix-org/synapse/issues/11045)) -- Fix a bug introduced in Synapse 1.45.0rc1 where the user directory would stop updating if it processed an event from a - user not in the `users` table. ([\#11053](https://github.com/matrix-org/synapse/issues/11053)) -- Fix a bug introduced in Synapse 1.44.0 when logging errors during oEmbed processing. ([\#11061](https://github.com/matrix-org/synapse/issues/11061)) - - -Internal Changes ----------------- - -- Add an 'approximate difference' method to `StateFilter`. ([\#10825](https://github.com/matrix-org/synapse/issues/10825)) -- Fix inconsistent behavior of `get_last_client_by_ip` when reporting data that has not been stored in the database yet. ([\#10970](https://github.com/matrix-org/synapse/issues/10970)) -- Fix a bug introduced in Synapse 1.21.0 that causes opentracing and Prometheus metrics for replication requests to be measured incorrectly. ([\#10996](https://github.com/matrix-org/synapse/issues/10996)) -- Ensure that cache config tests do not share state. ([\#11036](https://github.com/matrix-org/synapse/issues/11036)) - - -Synapse 1.45.0rc1 (2021-10-12) -============================== - -**Note:** Media storage providers module that read from Synapse's configuration need changes as of this version, see the [upgrade notes](https://matrix-org.github.io/synapse/develop/upgrade#upgrading-to-v1450) for more information. - -Known Issues ------------- - -- We are investigating [a performance issue](https://github.com/matrix-org/synapse/issues/11049) which was reported after the release of 1.44.0. -- We are aware of [a bug](https://github.com/matrix-org/synapse/issues/11025) with the user directory when using application services. A second release candidate is expected which will resolve this. - -Features --------- - -- Add [MSC3069](https://github.com/matrix-org/matrix-doc/pull/3069) support to `/account/whoami`. ([\#9655](https://github.com/matrix-org/synapse/issues/9655)) -- Support autodiscovery of oEmbed previews. ([\#10822](https://github.com/matrix-org/synapse/issues/10822)) -- Add a `user_may_send_3pid_invite` spam checker callback for modules to allow or deny 3PID invites. ([\#10894](https://github.com/matrix-org/synapse/issues/10894)) -- Add a spam checker callback to allow or deny room joins. ([\#10910](https://github.com/matrix-org/synapse/issues/10910)) -- Include an `update_synapse_database` script in the distribution. Contributed by @Fizzadar at Beeper. ([\#10954](https://github.com/matrix-org/synapse/issues/10954)) -- Include exception information in JSON logging output. Contributed by @Fizzadar at Beeper. ([\#11028](https://github.com/matrix-org/synapse/issues/11028)) - - -Bugfixes --------- - -- Fix a minor bug in the response to `/_matrix/client/r0/voip/turnServer`. Contributed by @lukaslihotzki. ([\#10922](https://github.com/matrix-org/synapse/issues/10922)) -- Fix a bug where empty `yyyy-mm-dd/` directories would be left behind in the media store's `url_cache_thumbnails/` directory. ([\#10924](https://github.com/matrix-org/synapse/issues/10924)) -- Fix a bug introduced in Synapse v1.40.0 where the signature checks for room version 8 and 9 could be applied to earlier room versions in some situations. ([\#10927](https://github.com/matrix-org/synapse/issues/10927)) -- Fix a long-standing bug wherein deactivated users still count towards the monthly active users limit. ([\#10947](https://github.com/matrix-org/synapse/issues/10947)) -- Fix a long-standing bug which meant that events received over federation were sometimes incorrectly accepted into the room state. ([\#10956](https://github.com/matrix-org/synapse/issues/10956)) -- Fix a long-standing bug where rebuilding the user directory wouldn't exclude support and deactivated users. ([\#10960](https://github.com/matrix-org/synapse/issues/10960)) -- Fix [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) `/batch_send` endpoint rejecting subsequent batches with unknown batch ID error in existing room versions from the room creator. ([\#10962](https://github.com/matrix-org/synapse/issues/10962)) -- Fix a bug that could leak local users' per-room nicknames and avatars when the user directory is rebuilt. ([\#10981](https://github.com/matrix-org/synapse/issues/10981)) -- Fix a long-standing bug where the remainder of a batch of user directory changes would be silently dropped if the server left a room early in the batch. ([\#10982](https://github.com/matrix-org/synapse/issues/10982)) -- Correct a bugfix introduced in Synapse v1.44.0 that would catch the wrong error if a connection is lost before a response could be written to it. ([\#10995](https://github.com/matrix-org/synapse/issues/10995)) -- Fix a long-standing bug where local users' per-room nicknames/avatars were visible to anyone who could see you in the user directory. ([\#11002](https://github.com/matrix-org/synapse/issues/11002)) -- Fix a long-standing bug where a user's per-room nickname/avatar would overwrite their profile in the user directory when a room was made public. ([\#11003](https://github.com/matrix-org/synapse/issues/11003)) -- Work around a regression, introduced in Synapse v1.39.0, that caused `SynapseError`s raised by the experimental third-party rules module callback `check_event_allowed` to be ignored. ([\#11042](https://github.com/matrix-org/synapse/issues/11042)) -- Fix a bug in [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) insertion events in rooms that could cause cross-talk/conflicts between batches. ([\#10877](https://github.com/matrix-org/synapse/issues/10877)) - - -Improved Documentation ----------------------- - -- Change wording ("reference homeserver") in Synapse repository documentation. Contributed by @maxkratz. ([\#10971](https://github.com/matrix-org/synapse/issues/10971)) -- Fix a dead URL in development documentation (SAML) and change wording from "Riot" to "Element". Contributed by @maxkratz. ([\#10973](https://github.com/matrix-org/synapse/issues/10973)) -- Add additional content to the Welcome and Overview page of the documentation. ([\#10990](https://github.com/matrix-org/synapse/issues/10990)) -- Update links to MSCs in documentation. Contributed by @dklimpel. ([\#10991](https://github.com/matrix-org/synapse/issues/10991)) - - -Internal Changes ----------------- - -- Improve type hinting in `synapse.util`. ([\#10888](https://github.com/matrix-org/synapse/issues/10888)) -- Add further type hints to `synapse.storage.util`. ([\#10892](https://github.com/matrix-org/synapse/issues/10892)) -- Fix type hints to be compatible with an upcoming change to Twisted. ([\#10895](https://github.com/matrix-org/synapse/issues/10895)) -- Update utility code to handle C implementations of frozendict. ([\#10902](https://github.com/matrix-org/synapse/issues/10902)) -- Drop old functionality which maintained database compatibility with Synapse versions before v1.31. ([\#10903](https://github.com/matrix-org/synapse/issues/10903)) -- Clean-up configuration helper classes for the `ServerConfig` class. ([\#10915](https://github.com/matrix-org/synapse/issues/10915)) -- Use direct references to config flags. ([\#10916](https://github.com/matrix-org/synapse/issues/10916), [\#10959](https://github.com/matrix-org/synapse/issues/10959), [\#10985](https://github.com/matrix-org/synapse/issues/10985)) -- Clean up some of the federation event authentication code for clarity. ([\#10926](https://github.com/matrix-org/synapse/issues/10926), [\#10940](https://github.com/matrix-org/synapse/issues/10940), [\#10986](https://github.com/matrix-org/synapse/issues/10986), [\#10987](https://github.com/matrix-org/synapse/issues/10987), [\#10988](https://github.com/matrix-org/synapse/issues/10988), [\#11010](https://github.com/matrix-org/synapse/issues/11010), [\#11011](https://github.com/matrix-org/synapse/issues/11011)) -- Refactor various parts of the codebase to use `RoomVersion` objects instead of room version identifier strings. ([\#10934](https://github.com/matrix-org/synapse/issues/10934)) -- Refactor user directory tests in preparation for upcoming changes. ([\#10935](https://github.com/matrix-org/synapse/issues/10935)) -- Include the event id in the logcontext when handling PDUs received over federation. ([\#10936](https://github.com/matrix-org/synapse/issues/10936)) -- Fix logged errors in unit tests. ([\#10939](https://github.com/matrix-org/synapse/issues/10939)) -- Fix a broken test to ensure that consent configuration works during registration. ([\#10945](https://github.com/matrix-org/synapse/issues/10945)) -- Add type hints to filtering classes. ([\#10958](https://github.com/matrix-org/synapse/issues/10958)) -- Add type-hint to `HomeserverTestcase.setup_test_homeserver`. ([\#10961](https://github.com/matrix-org/synapse/issues/10961)) -- Fix the test utility function `create_room_as` so that `is_public=True` will explicitly set the `visibility` parameter of room creation requests to `public`. Contributed by @AndrewFerr. ([\#10963](https://github.com/matrix-org/synapse/issues/10963)) -- Make the release script more robust and transparent. ([\#10966](https://github.com/matrix-org/synapse/issues/10966)) -- Refactor [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) `/batch_send` mega function into smaller handler functions. ([\#10974](https://github.com/matrix-org/synapse/issues/10974)) -- Log stack traces when a missing opentracing span is detected. ([\#10983](https://github.com/matrix-org/synapse/issues/10983)) -- Update GHA config to run tests against Python 3.10 and PostgreSQL 14. ([\#10992](https://github.com/matrix-org/synapse/issues/10992)) -- Fix a long-standing bug where `ReadWriteLock`s could drop logging contexts on exit. ([\#10993](https://github.com/matrix-org/synapse/issues/10993)) -- Add a `CODEOWNERS` file to automatically request reviews from the `@matrix-org/synapse-core` team on new pull requests. ([\#10994](https://github.com/matrix-org/synapse/issues/10994)) -- Add further type hints to `synapse.state`. ([\#11004](https://github.com/matrix-org/synapse/issues/11004)) -- Remove the deprecated `BaseHandler` object. ([\#11005](https://github.com/matrix-org/synapse/issues/11005)) -- Bump mypy version for CI to 0.910, and pull in new type stubs for dependencies. ([\#11006](https://github.com/matrix-org/synapse/issues/11006)) -- Fix CI to run the unit tests without optional deps. ([\#11017](https://github.com/matrix-org/synapse/issues/11017)) -- Ensure that cache config tests do not share state. ([\#11019](https://github.com/matrix-org/synapse/issues/11019)) -- Add additional type hints to `synapse.server_notices`. ([\#11021](https://github.com/matrix-org/synapse/issues/11021)) -- Add additional type hints for `synapse.push`. ([\#11023](https://github.com/matrix-org/synapse/issues/11023)) -- When installing the optional developer dependencies, also include the dependencies needed for type-checking and unit testing. ([\#11034](https://github.com/matrix-org/synapse/issues/11034)) -- Remove unnecessary list comprehension from `synapse_port_db` to satisfy code style requirements. ([\#11043](https://github.com/matrix-org/synapse/issues/11043)) - - -Synapse 1.44.0 (2021-10-05) -=========================== - -No significant changes since 1.44.0rc3. - - -Synapse 1.44.0rc3 (2021-10-04) -============================== - -Bugfixes --------- - -- Fix a bug introduced in Synapse v1.40.0 where changing a user's display name or avatar in a restricted room would cause an authentication error. ([\#10933](https://github.com/matrix-org/synapse/issues/10933)) -- Fix `/admin/whois/{user_id}` endpoint, which was broken in v1.44.0rc1. ([\#10968](https://github.com/matrix-org/synapse/issues/10968)) - - -Synapse 1.44.0rc2 (2021-09-30) -============================== - -Bugfixes --------- - -- Fix a bug introduced in v1.44.0rc1 which caused the experimental [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) `/batch_send` endpoint to return a 500 error. ([\#10938](https://github.com/matrix-org/synapse/issues/10938)) -- Fix a bug introduced in v1.44.0rc1 which prevented sending presence events to application services. ([\#10944](https://github.com/matrix-org/synapse/issues/10944)) - - -Improved Documentation ----------------------- - -- Minor updates to the installation instructions. ([\#10919](https://github.com/matrix-org/synapse/issues/10919)) - - -Synapse 1.44.0rc1 (2021-09-29) -============================== - -Features --------- - -- Only allow the [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) `/batch_send?chunk_id=xxx` endpoint to connect to an already existing insertion event. ([\#10776](https://github.com/matrix-org/synapse/issues/10776)) -- Improve oEmbed URL previews by processing the author name, photo, and video information. ([\#10814](https://github.com/matrix-org/synapse/issues/10814), [\#10819](https://github.com/matrix-org/synapse/issues/10819)) -- Speed up responding with large JSON objects to requests. ([\#10868](https://github.com/matrix-org/synapse/issues/10868), [\#10905](https://github.com/matrix-org/synapse/issues/10905)) -- Add a `user_may_create_room_with_invites` spam checker callback to allow modules to allow or deny a room creation request based on the invites and/or 3PID invites it includes. ([\#10898](https://github.com/matrix-org/synapse/issues/10898)) - - -Bugfixes --------- - -- Fix a long-standing bug that caused an `AssertionError` when purging history in certain rooms. Contributed by @Kokokokoka. ([\#10690](https://github.com/matrix-org/synapse/issues/10690)) -- Fix a long-standing bug which caused deactivated users that were later reactivated to be missing from the user directory. ([\#10782](https://github.com/matrix-org/synapse/issues/10782)) -- Fix a long-standing bug that caused unbanning a user by sending a membership event to fail. Contributed by @aaronraimist. ([\#10807](https://github.com/matrix-org/synapse/issues/10807)) -- Fix a long-standing bug where logging contexts would go missing when federation requests time out. ([\#10810](https://github.com/matrix-org/synapse/issues/10810)) -- Fix a long-standing bug causing an error in the deprecated `/initialSync` endpoint when using the undocumented `from` and `to` parameters. ([\#10827](https://github.com/matrix-org/synapse/issues/10827)) -- Fix a bug causing the `remove_stale_pushers` background job to repeatedly fail and log errors. This bug affected Synapse servers that had been upgraded from version 1.28 or older and are using SQLite. ([\#10843](https://github.com/matrix-org/synapse/issues/10843)) -- Fix a long-standing bug in Unicode support of the room search admin API breaking search for rooms with non-ASCII characters. ([\#10859](https://github.com/matrix-org/synapse/issues/10859)) -- Fix a bug introduced in Synapse 1.37.0 which caused `knock` membership events which we sent to remote servers to be incorrectly stored in the local database. ([\#10873](https://github.com/matrix-org/synapse/issues/10873)) -- Fix invalidating one-time key count cache after claiming keys. The bug was introduced in Synapse v1.41.0. Contributed by Tulir at Beeper. ([\#10875](https://github.com/matrix-org/synapse/issues/10875)) -- Fix a long-standing bug causing application service users to be subject to MAU blocking if the MAU limit had been reached, even if configured not to be blocked. ([\#10881](https://github.com/matrix-org/synapse/issues/10881)) -- Fix a long-standing bug which could cause events pulled over federation to be incorrectly rejected. ([\#10907](https://github.com/matrix-org/synapse/issues/10907)) -- Fix a long-standing bug causing URL cache files to be stored in storage providers. Server admins may safely delete the `url_cache/` and `url_cache_thumbnails/` directories from any configured storage providers to reclaim space. ([\#10911](https://github.com/matrix-org/synapse/issues/10911)) -- Fix a long-standing bug leading to race conditions when creating media store and config directories. ([\#10913](https://github.com/matrix-org/synapse/issues/10913)) - - -Improved Documentation ----------------------- - -- Fix some crashes in the Module API example code, by adding JSON encoding/decoding. ([\#10845](https://github.com/matrix-org/synapse/issues/10845)) -- Add developer documentation about experimental configuration flags. ([\#10865](https://github.com/matrix-org/synapse/issues/10865)) -- Properly remove deleted files from GitHub pages when generating the documentation. ([\#10869](https://github.com/matrix-org/synapse/issues/10869)) - - -Internal Changes ----------------- - -- Fix GitHub Actions config so we can run sytest on synapse from parallel branches. ([\#10659](https://github.com/matrix-org/synapse/issues/10659)) -- Split out [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) meta events to their own fields in the `/batch_send` response. ([\#10777](https://github.com/matrix-org/synapse/issues/10777)) -- Add missing type hints to REST servlets. ([\#10785](https://github.com/matrix-org/synapse/issues/10785), [\#10817](https://github.com/matrix-org/synapse/issues/10817)) -- Simplify the internal logic which maintains the user directory database tables. ([\#10796](https://github.com/matrix-org/synapse/issues/10796)) -- Use direct references to config flags. ([\#10812](https://github.com/matrix-org/synapse/issues/10812), [\#10885](https://github.com/matrix-org/synapse/issues/10885), [\#10893](https://github.com/matrix-org/synapse/issues/10893), [\#10897](https://github.com/matrix-org/synapse/issues/10897)) -- Specify the type of token in generic "Invalid token" error messages. ([\#10815](https://github.com/matrix-org/synapse/issues/10815)) -- Make `StateFilter` frozen so it is hashable. ([\#10816](https://github.com/matrix-org/synapse/issues/10816)) -- Fix a long-standing bug where an `m.room.message` event containing a null byte would cause an internal server error. ([\#10820](https://github.com/matrix-org/synapse/issues/10820)) -- Add type hints to the state database. ([\#10823](https://github.com/matrix-org/synapse/issues/10823)) -- Opt out of cache expiry for `get_users_who_share_room_with_user`, to hopefully improve `/sync` performance when you - haven't synced recently. ([\#10826](https://github.com/matrix-org/synapse/issues/10826)) -- Track cache eviction rates more finely in Prometheus's monitoring. ([\#10829](https://github.com/matrix-org/synapse/issues/10829)) -- Add missing type hints to `synapse.handlers`. ([\#10831](https://github.com/matrix-org/synapse/issues/10831), [\#10856](https://github.com/matrix-org/synapse/issues/10856)) -- Extend the Module API to let plug-ins check whether an ID is local and to access IP + User Agent data. ([\#10833](https://github.com/matrix-org/synapse/issues/10833)) -- Factor out PNG image data to a constant to be used in several tests. ([\#10834](https://github.com/matrix-org/synapse/issues/10834)) -- Add a test to ensure state events sent by modules get persisted correctly. ([\#10835](https://github.com/matrix-org/synapse/issues/10835)) -- Rename [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) fields and event types from `chunk` to `batch` to match the `/batch_send` endpoint. ([\#10838](https://github.com/matrix-org/synapse/issues/10838)) -- Rename [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) `/batch_send` query parameter from `?prev_event` to more obvious usage with `?prev_event_id`. ([\#10839](https://github.com/matrix-org/synapse/issues/10839)) -- Add type hints to `synapse.http.site`. ([\#10867](https://github.com/matrix-org/synapse/issues/10867)) -- Include outlier status when we log V2 or V3 events. ([\#10879](https://github.com/matrix-org/synapse/issues/10879)) -- Break down Grafana's cache expiry time series based on reason for eviction, c.f. [\#10829](https://github.com/matrix-org/synapse/issues/10829). ([\#10880](https://github.com/matrix-org/synapse/issues/10880)) -- Clean up some of the federation event authentication code for clarity. ([\#10883](https://github.com/matrix-org/synapse/issues/10883), [\#10884](https://github.com/matrix-org/synapse/issues/10884), [\#10896](https://github.com/matrix-org/synapse/issues/10896), [\#10901](https://github.com/matrix-org/synapse/issues/10901)) -- Allow the `.` and `~` characters when creating registration tokens as per the change to [MSC3231](https://github.com/matrix-org/matrix-doc/pull/3231). ([\#10887](https://github.com/matrix-org/synapse/issues/10887)) -- Clean up some unnecessary parentheses in places around the codebase. ([\#10889](https://github.com/matrix-org/synapse/issues/10889)) -- Improve type hinting in the user directory code. ([\#10891](https://github.com/matrix-org/synapse/issues/10891)) -- Update development testing script `test_postgresql.sh` to use a supported Python version and make re-runs quicker. ([\#10906](https://github.com/matrix-org/synapse/issues/10906)) -- Document and summarize changes in schema version `61` – `64`. ([\#10917](https://github.com/matrix-org/synapse/issues/10917)) -- Update release script to sign the newly created git tags. ([\#10925](https://github.com/matrix-org/synapse/issues/10925)) -- Fix Debian builds due to `dh-virtualenv` no longer being able to build their docs. ([\#10931](https://github.com/matrix-org/synapse/issues/10931)) - - -Synapse 1.43.0 (2021-09-21) -=========================== - -This release drops support for the deprecated, unstable API for [MSC2858 (Multiple SSO Identity Providers)](https://github.com/matrix-org/matrix-doc/blob/master/proposals/2858-Multiple-SSO-Identity-Providers.md#unstable-prefix), as well as the undocumented `experimental.msc2858_enabled` config option. Client authors should update their clients to use the stable API, available since Synapse 1.30. - -The documentation has been updated with configuration for routing `/spaces`, `/hierarchy` and `/summary` to workers. See [the upgrade notes](https://github.com/matrix-org/synapse/blob/release-v1.43/docs/upgrade.md#upgrading-to-v1430) for more details. - -No significant changes since 1.43.0rc2. - -Synapse 1.43.0rc2 (2021-09-17) -============================== - -Bugfixes --------- - -- Added opentracing logging to help debug [\#9424](https://github.com/matrix-org/synapse/issues/9424). ([\#10828](https://github.com/matrix-org/synapse/issues/10828)) - - -Synapse 1.43.0rc1 (2021-09-14) -============================== - -Features --------- - -- Allow room creators to send historical events specified by [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) in existing room versions. ([\#10566](https://github.com/matrix-org/synapse/issues/10566)) -- Add config option to use non-default manhole password and keys. ([\#10643](https://github.com/matrix-org/synapse/issues/10643)) -- Skip final GC at shutdown to improve restart performance. ([\#10712](https://github.com/matrix-org/synapse/issues/10712)) -- Allow configuration of the oEmbed URLs used for URL previews. ([\#10714](https://github.com/matrix-org/synapse/issues/10714), [\#10759](https://github.com/matrix-org/synapse/issues/10759)) -- Prefer [room version 9](https://github.com/matrix-org/matrix-doc/pull/3375) for restricted rooms per the [room version capabilities](https://github.com/matrix-org/matrix-doc/pull/3244) API. ([\#10772](https://github.com/matrix-org/synapse/issues/10772)) - - -Bugfixes --------- - -- Fix a long-standing bug where room avatars were not included in email notifications. ([\#10658](https://github.com/matrix-org/synapse/issues/10658)) -- Fix a bug where the ordering algorithm was skipping the `origin_server_ts` step in the spaces summary resulting in unstable room orderings. ([\#10730](https://github.com/matrix-org/synapse/issues/10730)) -- Fix edge case when persisting events into a room where there are multiple events we previously hadn't calculated auth chains for (and hadn't marked as needing to be calculated). ([\#10743](https://github.com/matrix-org/synapse/issues/10743)) -- Fix a bug which prevented calls to `/createRoom` that included the `room_alias_name` parameter from being handled by worker processes. ([\#10757](https://github.com/matrix-org/synapse/issues/10757)) -- Fix a bug which prevented user registration via SSO to require consent tracking for SSO mapping providers that don't prompt for Matrix ID selection. Contributed by @AndrewFerr. ([\#10733](https://github.com/matrix-org/synapse/issues/10733)) -- Only return the stripped state events for the `m.space.child` events in a room for the spaces summary from [MSC2946](https://github.com/matrix-org/matrix-doc/pull/2946). ([\#10760](https://github.com/matrix-org/synapse/issues/10760)) -- Properly handle room upgrades of spaces. ([\#10774](https://github.com/matrix-org/synapse/issues/10774)) -- Fix a bug which generated invalid homeserver config when the `frontend_proxy` worker type was passed to the Synapse Worker-based Complement image. ([\#10783](https://github.com/matrix-org/synapse/issues/10783)) - - -Improved Documentation ----------------------- - -- Minor fix to the `media_repository` developer documentation. Contributed by @cuttingedge1109. ([\#10556](https://github.com/matrix-org/synapse/issues/10556)) -- Update the documentation to note that the `/spaces` and `/hierarchy` endpoints can be routed to workers. ([\#10648](https://github.com/matrix-org/synapse/issues/10648)) -- Clarify admin API documentation on undoing room deletions. ([\#10735](https://github.com/matrix-org/synapse/issues/10735)) -- Split up the modules documentation and add examples for module developers. ([\#10758](https://github.com/matrix-org/synapse/issues/10758)) -- Correct 2 typographical errors in the [Log Contexts documentation](https://matrix-org.github.io/synapse/latest/log_contexts.html). ([\#10795](https://github.com/matrix-org/synapse/issues/10795)) -- Fix a wording mistake in the sample configuration. Contributed by @bramvdnheuvel:nltrix.net. ([\#10804](https://github.com/matrix-org/synapse/issues/10804)) - - -Deprecations and Removals -------------------------- - -- Remove the [unstable MSC2858 API](https://github.com/matrix-org/matrix-doc/blob/master/proposals/2858-Multiple-SSO-Identity-Providers.md#unstable-prefix), including the undocumented `experimental.msc2858_enabled` config option. The unstable API has been deprecated since Synapse 1.35. Client authors should update their clients to use the stable API introduced in Synapse 1.30 if they have not already done so. ([\#10693](https://github.com/matrix-org/synapse/issues/10693)) - - -Internal Changes ----------------- - -- Add OpenTracing logging to help debug stuck messages (as described by issue [#9424](https://github.com/matrix-org/synapse/issues/9424)). ([\#10704](https://github.com/matrix-org/synapse/issues/10704)) -- Add type annotations to the `synapse.util` package. ([\#10601](https://github.com/matrix-org/synapse/issues/10601)) -- Ensure `rooms.creator` field is always populated for easy lookup in [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) usage later. ([\#10697](https://github.com/matrix-org/synapse/issues/10697)) -- Add missing type hints to REST servlets. ([\#10707](https://github.com/matrix-org/synapse/issues/10707), [\#10728](https://github.com/matrix-org/synapse/issues/10728), [\#10736](https://github.com/matrix-org/synapse/issues/10736)) -- Do not include rooms with unknown room versions in the spaces summary results. ([\#10727](https://github.com/matrix-org/synapse/issues/10727)) -- Additional error checking for the `preset` field when creating a room. ([\#10738](https://github.com/matrix-org/synapse/issues/10738)) -- Clean up some of the federation event authentication code for clarity. ([\#10744](https://github.com/matrix-org/synapse/issues/10744), [\#10745](https://github.com/matrix-org/synapse/issues/10745), [\#10746](https://github.com/matrix-org/synapse/issues/10746), [\#10771](https://github.com/matrix-org/synapse/issues/10771), [\#10773](https://github.com/matrix-org/synapse/issues/10773), [\#10781](https://github.com/matrix-org/synapse/issues/10781)) -- Add an index to `presence_stream` to hopefully speed up startups a little. ([\#10748](https://github.com/matrix-org/synapse/issues/10748)) -- Refactor event size checking code to simplify searching the codebase for the origins of certain error strings that are occasionally emitted. ([\#10750](https://github.com/matrix-org/synapse/issues/10750)) -- Move tests relating to rooms having encryption out of the user directory tests. ([\#10752](https://github.com/matrix-org/synapse/issues/10752)) -- Use `attrs` internally for the URL preview code & update documentation. ([\#10753](https://github.com/matrix-org/synapse/issues/10753)) -- Minor speed ups when joining large rooms over federation. ([\#10754](https://github.com/matrix-org/synapse/issues/10754), [\#10755](https://github.com/matrix-org/synapse/issues/10755), [\#10756](https://github.com/matrix-org/synapse/issues/10756), [\#10780](https://github.com/matrix-org/synapse/issues/10780), [\#10784](https://github.com/matrix-org/synapse/issues/10784)) -- Add a constant for `m.federate`. ([\#10775](https://github.com/matrix-org/synapse/issues/10775)) -- Add a script to update the Debian changelog in a Docker container for systems that are not Debian-based. ([\#10778](https://github.com/matrix-org/synapse/issues/10778)) -- Change the format of authenticated users in logs when a user is being puppeted by and admin user. ([\#10779](https://github.com/matrix-org/synapse/issues/10779)) -- Remove fixed and flakey tests from the Sytest blacklist. ([\#10788](https://github.com/matrix-org/synapse/issues/10788)) -- Improve internal details of the user directory code. ([\#10789](https://github.com/matrix-org/synapse/issues/10789)) -- Use direct references to config flags. ([\#10798](https://github.com/matrix-org/synapse/issues/10798)) -- Ensure the Rust reporter passes type checking with jaeger-client 4.7's type annotations. ([\#10799](https://github.com/matrix-org/synapse/issues/10799)) - - -Synapse 1.42.0 (2021-09-07) -=========================== - -This version of Synapse removes deprecated room-management admin APIs, removes out-of-date email pushers, and improves error handling for fallback templates for user-interactive authentication. For more information on these points, server administrators are encouraged to read [the upgrade notes](docs/upgrade.md#upgrading-to-v1420). - -No significant changes since 1.42.0rc2. - - -Synapse 1.42.0rc2 (2021-09-06) -============================== - -Features --------- - -- Support room version 9 from [MSC3375](https://github.com/matrix-org/matrix-doc/pull/3375). ([\#10747](https://github.com/matrix-org/synapse/issues/10747)) - - -Internal Changes ----------------- - -- Print a warning when using one of the deprecated `template_dir` settings. ([\#10768](https://github.com/matrix-org/synapse/issues/10768)) - - -Synapse 1.42.0rc1 (2021-09-01) -============================== - -Features --------- - -- Add support for [MSC3231](https://github.com/matrix-org/matrix-doc/pull/3231): Token authenticated registration. Users can be required to submit a token during registration to authenticate themselves. Contributed by Callum Brown. ([\#10142](https://github.com/matrix-org/synapse/issues/10142)) -- Add support for [MSC3283](https://github.com/matrix-org/matrix-doc/pull/3283): Expose `enable_set_displayname` in capabilities. ([\#10452](https://github.com/matrix-org/synapse/issues/10452)) -- Port the `PresenceRouter` module interface to the new generic interface. ([\#10524](https://github.com/matrix-org/synapse/issues/10524)) -- Add pagination to the spaces summary based on updates to [MSC2946](https://github.com/matrix-org/matrix-doc/pull/2946). ([\#10613](https://github.com/matrix-org/synapse/issues/10613), [\#10725](https://github.com/matrix-org/synapse/issues/10725)) - - -Bugfixes --------- - -- Validate new `m.room.power_levels` events. Contributed by @aaronraimist. ([\#10232](https://github.com/matrix-org/synapse/issues/10232)) -- Display an error on User-Interactive Authentication fallback pages when authentication fails. Contributed by Callum Brown. ([\#10561](https://github.com/matrix-org/synapse/issues/10561)) -- Remove pushers when deleting an e-mail address from an account. Pushers for old unlinked emails will also be deleted. ([\#10581](https://github.com/matrix-org/synapse/issues/10581), [\#10734](https://github.com/matrix-org/synapse/issues/10734)) -- Reject Client-Server `/keys/query` requests which provide `device_ids` incorrectly. ([\#10593](https://github.com/matrix-org/synapse/issues/10593)) -- Rooms with unsupported room versions are no longer returned via `/sync`. ([\#10644](https://github.com/matrix-org/synapse/issues/10644)) -- Enforce the maximum length for per-room display names and avatar URLs. ([\#10654](https://github.com/matrix-org/synapse/issues/10654)) -- Fix a bug which caused the `synapse_user_logins_total` Prometheus metric not to be correctly initialised on restart. ([\#10677](https://github.com/matrix-org/synapse/issues/10677)) -- Improve `ServerNoticeServlet` to avoid duplicate requests and add unit tests. ([\#10679](https://github.com/matrix-org/synapse/issues/10679)) -- Fix long-standing issue which caused an error when a thumbnail is requested and there are multiple thumbnails with the same quality rating. ([\#10684](https://github.com/matrix-org/synapse/issues/10684)) -- Fix a regression introduced in v1.41.0 which affected the performance of concurrent fetches of large sets of events, in extreme cases causing the process to hang. ([\#10703](https://github.com/matrix-org/synapse/issues/10703)) -- Fix a regression introduced in Synapse 1.41 which broke email transmission on Systems using older versions of the Twisted library. ([\#10713](https://github.com/matrix-org/synapse/issues/10713)) - - -Improved Documentation ----------------------- - -- Add documentation on how to connect Django with Synapse using OpenID Connect and django-oauth-toolkit. Contributed by @HugoDelval. ([\#10192](https://github.com/matrix-org/synapse/issues/10192)) -- Advertise https://matrix-org.github.io/synapse documentation in the `README` and `CONTRIBUTING` files. ([\#10595](https://github.com/matrix-org/synapse/issues/10595)) -- Fix some of the titles not rendering in the OpenID Connect documentation. ([\#10639](https://github.com/matrix-org/synapse/issues/10639)) -- Minor clarifications to the documentation for reverse proxies. ([\#10708](https://github.com/matrix-org/synapse/issues/10708)) -- Remove table of contents from the top of installation and contributing documentation pages. ([\#10711](https://github.com/matrix-org/synapse/issues/10711)) - - -Deprecations and Removals -------------------------- - -- Remove deprecated Shutdown Room and Purge Room Admin API. ([\#8830](https://github.com/matrix-org/synapse/issues/8830)) - - -Internal Changes ----------------- - -- Improve type hints for the proxy agent and SRV resolver modules. Contributed by @dklimpel. ([\#10608](https://github.com/matrix-org/synapse/issues/10608)) -- Clean up some of the federation event authentication code for clarity. ([\#10614](https://github.com/matrix-org/synapse/issues/10614), [\#10615](https://github.com/matrix-org/synapse/issues/10615), [\#10624](https://github.com/matrix-org/synapse/issues/10624), [\#10640](https://github.com/matrix-org/synapse/issues/10640)) -- Add a comment asking developers to leave a reason when bumping the database schema version. ([\#10621](https://github.com/matrix-org/synapse/issues/10621)) -- Remove not needed database updates in modify user admin API. ([\#10627](https://github.com/matrix-org/synapse/issues/10627)) -- Convert room member storage tuples to `attrs` classes. ([\#10629](https://github.com/matrix-org/synapse/issues/10629), [\#10642](https://github.com/matrix-org/synapse/issues/10642)) -- Use auto-attribs for the attrs classes used in sync. ([\#10630](https://github.com/matrix-org/synapse/issues/10630)) -- Make `backfill` and `get_missing_events` use the same codepath. ([\#10645](https://github.com/matrix-org/synapse/issues/10645)) -- Improve the performance of the `/hierarchy` API (from [MSC2946](https://github.com/matrix-org/matrix-doc/pull/2946)) by caching responses received over federation. ([\#10647](https://github.com/matrix-org/synapse/issues/10647)) -- Run a nightly CI build against Twisted trunk. ([\#10651](https://github.com/matrix-org/synapse/issues/10651), [\#10672](https://github.com/matrix-org/synapse/issues/10672)) -- Do not print out stack traces for network errors when fetching data over federation. ([\#10662](https://github.com/matrix-org/synapse/issues/10662)) -- Simplify tests for device admin rest API. ([\#10664](https://github.com/matrix-org/synapse/issues/10664)) -- Add missing type hints to REST servlets. ([\#10665](https://github.com/matrix-org/synapse/issues/10665), [\#10666](https://github.com/matrix-org/synapse/issues/10666), [\#10674](https://github.com/matrix-org/synapse/issues/10674)) -- Flatten the `tests.synapse.rests` package by moving the contents of `v1` and `v2_alpha` into the parent. ([\#10667](https://github.com/matrix-org/synapse/issues/10667)) -- Update `complement.sh` to rebuild the base Docker image when run with workers. ([\#10686](https://github.com/matrix-org/synapse/issues/10686)) -- Split the event-processing methods in `FederationHandler` into a separate `FederationEventHandler`. ([\#10692](https://github.com/matrix-org/synapse/issues/10692)) -- Remove unused `compare_digest` function. ([\#10706](https://github.com/matrix-org/synapse/issues/10706)) - - -Synapse 1.41.1 (2021-08-31) -=========================== - -Due to the two security issues highlighted below, server administrators are encouraged to update Synapse. We are not aware of these vulnerabilities being exploited in the wild. - -Security advisory ------------------ - -The following issues are fixed in v1.41.1. - -- **[GHSA-3x4c-pq33-4w3q](https://github.com/matrix-org/synapse/security/advisories/GHSA-3x4c-pq33-4w3q) / [CVE-2021-39164](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-39164): Enumerating a private room's list of members and their display names.** - - If an unauthorized user both knows the Room ID of a private room *and* that room's history visibility is set to `shared`, then they may be able to enumerate the room's members, including their display names. - - The unauthorized user must be on the same homeserver as a user who is a member of the target room. - - Fixed by [52c7a51cf](https://github.com/matrix-org/synapse/commit/52c7a51cf). - -- **[GHSA-jj53-8fmw-f2w2](https://github.com/matrix-org/synapse/security/advisories/GHSA-jj53-8fmw-f2w2) / [CVE-2021-39163](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-39163): Disclosing a private room's name, avatar, topic, and number of members.** - - If an unauthorized user knows the Room ID of a private room, then its name, avatar, topic, and number of members may be disclosed through Group / Community features. - - The unauthorized user must be on the same homeserver as a user who is a member of the target room, and their homeserver must allow non-administrators to create groups (`enable_group_creation` in the Synapse configuration; off by default). - - Fixed by [cb35df940a](https://github.com/matrix-org/synapse/commit/cb35df940a), [\#10723](https://github.com/matrix-org/synapse/issues/10723). - -Bugfixes --------- - -- Fix a regression introduced in Synapse 1.41 which broke email transmission on systems using older versions of the Twisted library. ([\#10713](https://github.com/matrix-org/synapse/issues/10713)) - -Synapse 1.41.0 (2021-08-24) -=========================== - -This release adds support for Debian 12 (Bookworm), but **removes support for Ubuntu 20.10 (Groovy Gorilla)**, which reached End of Life last month. - -Note that when using workers the `/_synapse/admin/v1/users/{userId}/media` must now be handled by media workers. See the [upgrade notes](https://matrix-org.github.io/synapse/latest/upgrade.html) for more information. - - -Features --------- - -- Enable room capabilities ([MSC3244](https://github.com/matrix-org/matrix-doc/pull/3244)) by default and set room version 8 as the preferred room version when creating restricted rooms. ([\#10571](https://github.com/matrix-org/synapse/issues/10571)) - - -Synapse 1.41.0rc1 (2021-08-18) -============================== - -Features --------- - -- Add `get_userinfo_by_id` method to ModuleApi. ([\#9581](https://github.com/matrix-org/synapse/issues/9581)) -- Initial local support for [MSC3266](https://github.com/matrix-org/synapse/pull/10394), Room Summary over the unstable `/rooms/{roomIdOrAlias}/summary` API. ([\#10394](https://github.com/matrix-org/synapse/issues/10394)) -- Experimental support for [MSC3288](https://github.com/matrix-org/matrix-doc/pull/3288), sending `room_type` to the identity server for 3pid invites over the `/store-invite` API. ([\#10435](https://github.com/matrix-org/synapse/issues/10435)) -- Add support for sending federation requests through a proxy. Contributed by @Bubu and @dklimpel. See the [upgrade notes](https://matrix-org.github.io/synapse/latest/upgrade.html) for more information. ([\#10596](https://github.com/matrix-org/synapse/issues/10596)). ([\#10475](https://github.com/matrix-org/synapse/issues/10475)) -- Add support for "marker" events which makes historical events discoverable for servers that already have all of the scrollback history (part of [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716)). ([\#10498](https://github.com/matrix-org/synapse/issues/10498)) -- Add a configuration setting for the time a `/sync` response is cached for. ([\#10513](https://github.com/matrix-org/synapse/issues/10513)) -- The default logging handler for new installations is now `PeriodicallyFlushingMemoryHandler`, a buffered logging handler which periodically flushes itself. ([\#10518](https://github.com/matrix-org/synapse/issues/10518)) -- Add support for new redaction rules for historical events specified in [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716). ([\#10538](https://github.com/matrix-org/synapse/issues/10538)) -- Add a setting to disable TLS when sending email. ([\#10546](https://github.com/matrix-org/synapse/issues/10546)) -- Add pagination to the spaces summary based on updates to [MSC2946](https://github.com/matrix-org/matrix-doc/pull/2946). ([\#10549](https://github.com/matrix-org/synapse/issues/10549), [\#10560](https://github.com/matrix-org/synapse/issues/10560), [\#10569](https://github.com/matrix-org/synapse/issues/10569), [\#10574](https://github.com/matrix-org/synapse/issues/10574), [\#10575](https://github.com/matrix-org/synapse/issues/10575), [\#10579](https://github.com/matrix-org/synapse/issues/10579), [\#10583](https://github.com/matrix-org/synapse/issues/10583)) -- Admin API to delete several media for a specific user. Contributed by @dklimpel. ([\#10558](https://github.com/matrix-org/synapse/issues/10558), [\#10628](https://github.com/matrix-org/synapse/issues/10628)) -- Add support for routing `/createRoom` to workers. ([\#10564](https://github.com/matrix-org/synapse/issues/10564)) -- Update the Synapse Grafana dashboard. ([\#10570](https://github.com/matrix-org/synapse/issues/10570)) -- Add an admin API (`GET /_synapse/admin/username_available`) to check if a username is available (regardless of registration settings). ([\#10578](https://github.com/matrix-org/synapse/issues/10578)) -- Allow editing a user's `external_ids` via the "Edit User" admin API. Contributed by @dklimpel. ([\#10598](https://github.com/matrix-org/synapse/issues/10598)) -- The Synapse manhole no longer needs coroutines to be wrapped in `defer.ensureDeferred`. ([\#10602](https://github.com/matrix-org/synapse/issues/10602)) -- Add option to allow modules to run periodic tasks on all instances, rather than just the one configured to run background tasks. ([\#10638](https://github.com/matrix-org/synapse/issues/10638)) - - -Bugfixes --------- - -- Add some clarification to the sample config file. Contributed by @Kentokamoto. ([\#10129](https://github.com/matrix-org/synapse/issues/10129)) -- Fix a long-standing bug where protocols which are not implemented by any appservices were incorrectly returned via `GET /_matrix/client/r0/thirdparty/protocols`. ([\#10532](https://github.com/matrix-org/synapse/issues/10532)) -- Fix exceptions in logs when failing to get remote room list. ([\#10541](https://github.com/matrix-org/synapse/issues/10541)) -- Fix longstanding bug which caused the user's presence "status message" to be reset when the user went offline. Contributed by @dklimpel. ([\#10550](https://github.com/matrix-org/synapse/issues/10550)) -- Allow public rooms to be previewed in the spaces summary APIs from [MSC2946](https://github.com/matrix-org/matrix-doc/pull/2946). ([\#10580](https://github.com/matrix-org/synapse/issues/10580)) -- Fix a bug introduced in v1.37.1 where an error could occur in the asynchronous processing of PDUs when the queue was empty. ([\#10592](https://github.com/matrix-org/synapse/issues/10592)) -- Fix errors on /sync when read receipt data is a string. Only affects homeservers with the experimental flag for [MSC2285](https://github.com/matrix-org/matrix-doc/pull/2285) enabled. Contributed by @SimonBrandner. ([\#10606](https://github.com/matrix-org/synapse/issues/10606)) -- Additional validation for the spaces summary API to avoid errors like `ValueError: Stop argument for islice() must be None or an integer`. The missing validation has existed since v1.31.0. ([\#10611](https://github.com/matrix-org/synapse/issues/10611)) -- Revert behaviour introduced in v1.38.0 that strips `org.matrix.msc2732.device_unused_fallback_key_types` from `/sync` when its value is empty. This field should instead always be present according to [MSC2732](https://github.com/matrix-org/matrix-doc/blob/master/proposals/2732-olm-fallback-keys.md). ([\#10623](https://github.com/matrix-org/synapse/issues/10623)) - - -Improved Documentation ----------------------- - -- Add documentation for configuring a forward proxy. ([\#10443](https://github.com/matrix-org/synapse/issues/10443)) -- Updated the reverse proxy documentation to highlight the homserver configuration that is needed to make Synapse aware that is is intentionally reverse proxied. ([\#10551](https://github.com/matrix-org/synapse/issues/10551)) -- Update CONTRIBUTING.md to fix index links and the instructions for SyTest in docker. ([\#10599](https://github.com/matrix-org/synapse/issues/10599)) - - -Deprecations and Removals -------------------------- - -- No longer build `.deb` packages for Ubuntu 20.10 Groovy Gorilla, which has now EOLed. ([\#10588](https://github.com/matrix-org/synapse/issues/10588)) -- The `template_dir` configuration settings in the `sso`, `account_validity` and `email` sections of the configuration file are now deprecated in favour of the global `templates.custom_template_directory` setting. See the [upgrade notes](https://matrix-org.github.io/synapse/latest/upgrade.html) for more information. ([\#10596](https://github.com/matrix-org/synapse/issues/10596)) - - -Internal Changes ----------------- - -- Improve event caching mechanism to avoid having multiple copies of an event in memory at a time. ([\#10119](https://github.com/matrix-org/synapse/issues/10119)) -- Reduce errors in PostgreSQL logs due to concurrent serialization errors. ([\#10504](https://github.com/matrix-org/synapse/issues/10504)) -- Include room ID in ignored EDU log messages. Contributed by @ilmari. ([\#10507](https://github.com/matrix-org/synapse/issues/10507)) -- Add pagination to the spaces summary based on updates to [MSC2946](https://github.com/matrix-org/matrix-doc/pull/2946). ([\#10527](https://github.com/matrix-org/synapse/issues/10527), [\#10530](https://github.com/matrix-org/synapse/issues/10530)) -- Fix CI to not break when run against branches rather than pull requests. ([\#10529](https://github.com/matrix-org/synapse/issues/10529)) -- Mark all events stemming from the [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) `/batch_send` endpoint as historical. ([\#10537](https://github.com/matrix-org/synapse/issues/10537)) -- Clean up some of the federation event authentication code for clarity. ([\#10539](https://github.com/matrix-org/synapse/issues/10539), [\#10591](https://github.com/matrix-org/synapse/issues/10591)) -- Convert `Transaction` and `Edu` objects to attrs. ([\#10542](https://github.com/matrix-org/synapse/issues/10542)) -- Update `/batch_send` endpoint to only return `state_events` created by the `state_events_from_before` passed in. ([\#10552](https://github.com/matrix-org/synapse/issues/10552)) -- Update contributing.md to warn against rebasing an open PR. ([\#10563](https://github.com/matrix-org/synapse/issues/10563)) -- Remove the unused public rooms replication stream. ([\#10565](https://github.com/matrix-org/synapse/issues/10565)) -- Clarify error message when failing to join a restricted room. ([\#10572](https://github.com/matrix-org/synapse/issues/10572)) -- Remove references to BuildKite in favour of GitHub Actions. ([\#10573](https://github.com/matrix-org/synapse/issues/10573)) -- Move `/batch_send` endpoint defined by [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) to the `/v2_alpha` directory. ([\#10576](https://github.com/matrix-org/synapse/issues/10576)) -- Allow multiple custom directories in `read_templates`. ([\#10587](https://github.com/matrix-org/synapse/issues/10587)) -- Re-organize the `synapse.federation.transport.server` module to create smaller files. ([\#10590](https://github.com/matrix-org/synapse/issues/10590)) -- Flatten the `synapse.rest.client` package by moving the contents of `v1` and `v2_alpha` into the parent. ([\#10600](https://github.com/matrix-org/synapse/issues/10600)) -- Build Debian packages for Debian 12 (Bookworm). ([\#10612](https://github.com/matrix-org/synapse/issues/10612)) -- Fix up a couple of links to the database schema documentation. ([\#10620](https://github.com/matrix-org/synapse/issues/10620)) -- Fix a broken link to the upgrade notes. ([\#10631](https://github.com/matrix-org/synapse/issues/10631)) - - -Synapse 1.40.0 (2021-08-10) -=========================== - -No significant changes. - - -Synapse 1.40.0rc3 (2021-08-09) -============================== - -Features --------- - -- Support [MSC3289: room version 8](https://github.com/matrix-org/matrix-doc/pull/3289). ([\#10449](https://github.com/matrix-org/synapse/issues/10449)) - - -Bugfixes --------- - -- Mark the experimental room version from [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) as unstable. ([\#10449](https://github.com/matrix-org/synapse/issues/10449)) - - -Improved Documentation ----------------------- - -- Fix broken links in `upgrade.md`. Contributed by @dklimpel. ([\#10543](https://github.com/matrix-org/synapse/issues/10543)) - - -Synapse 1.40.0rc2 (2021-08-04) -============================== - -Bugfixes --------- - -- Fix the `PeriodicallyFlushingMemoryHandler` inhibiting application shutdown because of its background thread. ([\#10517](https://github.com/matrix-org/synapse/issues/10517)) -- Fix a bug introduced in Synapse v1.40.0rc1 that could cause Synapse to respond with an error when clients would update read receipts. ([\#10531](https://github.com/matrix-org/synapse/issues/10531)) - - -Internal Changes ----------------- - -- Fix release script to open the correct URL for the release. ([\#10516](https://github.com/matrix-org/synapse/issues/10516)) - - -Synapse 1.40.0rc1 (2021-08-03) -============================== - -Features --------- - -- Add support for [MSC2033](https://github.com/matrix-org/matrix-doc/pull/2033): `device_id` on `/account/whoami`. ([\#9918](https://github.com/matrix-org/synapse/issues/9918)) -- Update support for [MSC2716 - Incrementally importing history into existing rooms](https://github.com/matrix-org/matrix-doc/pull/2716). ([\#10245](https://github.com/matrix-org/synapse/issues/10245), [\#10432](https://github.com/matrix-org/synapse/issues/10432), [\#10463](https://github.com/matrix-org/synapse/issues/10463)) -- Update support for [MSC3083](https://github.com/matrix-org/matrix-doc/pull/3083) to consider changes in the MSC around which servers can issue join events. ([\#10254](https://github.com/matrix-org/synapse/issues/10254), [\#10447](https://github.com/matrix-org/synapse/issues/10447), [\#10489](https://github.com/matrix-org/synapse/issues/10489)) -- Initial support for [MSC3244](https://github.com/matrix-org/matrix-doc/pull/3244), Room version capabilities over the /capabilities API. ([\#10283](https://github.com/matrix-org/synapse/issues/10283)) -- Add a buffered logging handler which periodically flushes itself. ([\#10407](https://github.com/matrix-org/synapse/issues/10407), [\#10515](https://github.com/matrix-org/synapse/issues/10515)) -- Add support for https connections to a proxy server. Contributed by @Bubu and @dklimpel. ([\#10411](https://github.com/matrix-org/synapse/issues/10411)) -- Support for [MSC2285 (hidden read receipts)](https://github.com/matrix-org/matrix-doc/pull/2285). Contributed by @SimonBrandner. ([\#10413](https://github.com/matrix-org/synapse/issues/10413)) -- Email notifications now state whether an invitation is to a room or a space. ([\#10426](https://github.com/matrix-org/synapse/issues/10426)) -- Allow setting transaction limit for database connections. ([\#10440](https://github.com/matrix-org/synapse/issues/10440), [\#10511](https://github.com/matrix-org/synapse/issues/10511)) -- Add `creation_ts` to "list users" admin API. ([\#10448](https://github.com/matrix-org/synapse/issues/10448)) - - -Bugfixes --------- - -- Improve character set detection in URL previews by supporting underscores (in addition to hyphens). Contributed by @srividyut. ([\#10410](https://github.com/matrix-org/synapse/issues/10410)) -- Fix events being incorrectly rejected over federation if they reference auth events that the server needed to fetch. ([\#10439](https://github.com/matrix-org/synapse/issues/10439)) -- Fix `synapse_federation_server_oldest_inbound_pdu_in_staging` Prometheus metric to not report a max age of 51 years when the queue is empty. ([\#10455](https://github.com/matrix-org/synapse/issues/10455)) -- Fix a bug which caused an explicit assignment of power-level 0 to a user to be misinterpreted in rare circumstances. ([\#10499](https://github.com/matrix-org/synapse/issues/10499)) - - -Improved Documentation ----------------------- - -- Fix hierarchy of providers on the OpenID page. ([\#10445](https://github.com/matrix-org/synapse/issues/10445)) -- Consolidate development documentation to `docs/development/`. ([\#10453](https://github.com/matrix-org/synapse/issues/10453)) -- Add some developer docs to explain room DAG concepts like `outliers`, `state_groups`, `depth`, etc. ([\#10464](https://github.com/matrix-org/synapse/issues/10464)) -- Document how to use Complement while developing a new Synapse feature. ([\#10483](https://github.com/matrix-org/synapse/issues/10483)) - - -Internal Changes ----------------- - -- Prune inbound federation queues for a room if they get too large. ([\#10390](https://github.com/matrix-org/synapse/issues/10390)) -- Add type hints to `synapse.federation.transport.client` module. ([\#10408](https://github.com/matrix-org/synapse/issues/10408)) -- Remove shebang line from module files. ([\#10415](https://github.com/matrix-org/synapse/issues/10415)) -- Drop backwards-compatibility code that was required to support Ubuntu Xenial. ([\#10429](https://github.com/matrix-org/synapse/issues/10429)) -- Use a docker image cache for the prerequisites for the debian package build. ([\#10431](https://github.com/matrix-org/synapse/issues/10431)) -- Improve servlet type hints. ([\#10437](https://github.com/matrix-org/synapse/issues/10437), [\#10438](https://github.com/matrix-org/synapse/issues/10438)) -- Replace usage of `or_ignore` in `simple_insert` with `simple_upsert` usage, to stop spamming postgres logs with spurious ERROR messages. ([\#10442](https://github.com/matrix-org/synapse/issues/10442)) -- Update the `tests-done` Github Actions status. ([\#10444](https://github.com/matrix-org/synapse/issues/10444), [\#10512](https://github.com/matrix-org/synapse/issues/10512)) -- Update type annotations to work with forthcoming Twisted 21.7.0 release. ([\#10446](https://github.com/matrix-org/synapse/issues/10446), [\#10450](https://github.com/matrix-org/synapse/issues/10450)) -- Cancel redundant GHA workflows when a new commit is pushed. ([\#10451](https://github.com/matrix-org/synapse/issues/10451)) -- Mitigate media repo XSS attacks on IE11 via the non-standard X-Content-Security-Policy header. ([\#10468](https://github.com/matrix-org/synapse/issues/10468)) -- Additional type hints in the state handler. ([\#10482](https://github.com/matrix-org/synapse/issues/10482)) -- Update syntax used to run complement tests. ([\#10488](https://github.com/matrix-org/synapse/issues/10488)) -- Fix up type annotations to work with Twisted 21.7. ([\#10490](https://github.com/matrix-org/synapse/issues/10490)) -- Improve type annotations for `ObservableDeferred`. ([\#10491](https://github.com/matrix-org/synapse/issues/10491)) -- Extend release script to also tag and create GitHub releases. ([\#10496](https://github.com/matrix-org/synapse/issues/10496)) -- Fix a bug which caused production debian packages to be incorrectly marked as 'prerelease'. ([\#10500](https://github.com/matrix-org/synapse/issues/10500)) - - -Synapse 1.39.0 (2021-07-29) -=========================== - -No significant changes. - - -Synapse 1.39.0rc3 (2021-07-28) -============================== - -Bugfixes --------- - -- Fix a bug introduced in Synapse 1.38 which caused an exception at startup when SAML authentication was enabled. ([\#10477](https://github.com/matrix-org/synapse/issues/10477)) -- Fix a long-standing bug where Synapse would not inform clients that a device had exhausted its one-time-key pool, potentially causing problems decrypting events. ([\#10485](https://github.com/matrix-org/synapse/issues/10485)) -- Fix reporting old R30 stats as R30v2 stats. Introduced in v1.39.0rc1. ([\#10486](https://github.com/matrix-org/synapse/issues/10486)) - - -Internal Changes ----------------- - -- Fix an error which prevented the Github Actions workflow to build the docker images from running. ([\#10461](https://github.com/matrix-org/synapse/issues/10461)) -- Fix release script to correctly version debian changelog when doing RCs. ([\#10465](https://github.com/matrix-org/synapse/issues/10465)) - - -Synapse 1.39.0rc2 (2021-07-22) -============================== - -This release also includes the changes in v1.38.1. - - -Internal Changes ----------------- - -- Move docker image build to Github Actions. ([\#10416](https://github.com/matrix-org/synapse/issues/10416)) - - -Synapse 1.38.1 (2021-07-22) -=========================== - -Bugfixes --------- - -- Always include `device_one_time_keys_count` key in `/sync` response to work around a bug in Element Android that broke encryption for new devices. ([\#10457](https://github.com/matrix-org/synapse/issues/10457)) - - -Synapse 1.39.0rc1 (2021-07-20) -============================== - -The Third-Party Event Rules module interface has been deprecated in favour of the generic module interface introduced in Synapse v1.37.0. Support for the old interface is planned to be removed in September 2021. See the [upgrade notes](https://matrix-org.github.io/synapse/latest/upgrade.html#upgrading-to-v1390) for more information. - -Features --------- - -- Add the ability to override the account validity feature with a module. ([\#9884](https://github.com/matrix-org/synapse/issues/9884)) -- The spaces summary API now returns any joinable rooms, not only rooms which are world-readable. ([\#10298](https://github.com/matrix-org/synapse/issues/10298), [\#10305](https://github.com/matrix-org/synapse/issues/10305)) -- Add a new version of the R30 phone-home metric, which removes a false impression of retention given by the old R30 metric. ([\#10332](https://github.com/matrix-org/synapse/issues/10332), [\#10427](https://github.com/matrix-org/synapse/issues/10427)) -- Allow providing credentials to `http_proxy`. ([\#10360](https://github.com/matrix-org/synapse/issues/10360)) - - -Bugfixes --------- - -- Fix error while dropping locks on shutdown. Introduced in v1.38.0. ([\#10433](https://github.com/matrix-org/synapse/issues/10433)) -- Add base starting insertion event when no chunk ID is specified in the historical batch send API. ([\#10250](https://github.com/matrix-org/synapse/issues/10250)) -- Fix historical batch send endpoint (MSC2716) rejecting batches with messages from multiple senders. ([\#10276](https://github.com/matrix-org/synapse/issues/10276)) -- Fix purging rooms that other homeservers are still sending events for. Contributed by @ilmari. ([\#10317](https://github.com/matrix-org/synapse/issues/10317)) -- Fix errors during backfill caused by previously purged redaction events. Contributed by Andreas Rammhold (@andir). ([\#10343](https://github.com/matrix-org/synapse/issues/10343)) -- Fix the user directory becoming broken (and noisy errors being logged) when knocking and room statistics are in use. ([\#10344](https://github.com/matrix-org/synapse/issues/10344)) -- Fix newly added `synapse_federation_server_oldest_inbound_pdu_in_staging` prometheus metric to measure age rather than timestamp. ([\#10355](https://github.com/matrix-org/synapse/issues/10355)) -- Fix PostgreSQL sometimes using table scans for queries against `state_groups_state` table, taking a long time and a large amount of IO. ([\#10359](https://github.com/matrix-org/synapse/issues/10359)) -- Fix `make_room_admin` failing for users that have left a private room. ([\#10367](https://github.com/matrix-org/synapse/issues/10367)) -- Fix a number of logged errors caused by remote servers being down. ([\#10400](https://github.com/matrix-org/synapse/issues/10400), [\#10414](https://github.com/matrix-org/synapse/issues/10414)) -- Responses from `/make_{join,leave,knock}` no longer include signatures, which will turn out to be invalid after events are returned to `/send_{join,leave,knock}`. ([\#10404](https://github.com/matrix-org/synapse/issues/10404)) - - -Improved Documentation ----------------------- - -- Updated installation dependencies for newer macOS versions and ARM Macs. Contributed by Luke Walsh. ([\#9971](https://github.com/matrix-org/synapse/issues/9971)) -- Simplify structure of room admin API. ([\#10313](https://github.com/matrix-org/synapse/issues/10313)) -- Refresh the logcontext dev documentation. ([\#10353](https://github.com/matrix-org/synapse/issues/10353)), ([\#10337](https://github.com/matrix-org/synapse/issues/10337)) -- Add delegation example for caddy in the reverse proxy documentation. Contributed by @moritzdietz. ([\#10368](https://github.com/matrix-org/synapse/issues/10368)) -- Fix and clarify some links in `docs` and `contrib`. ([\#10370](https://github.com/matrix-org/synapse/issues/10370)), ([\#10322](https://github.com/matrix-org/synapse/issues/10322)), ([\#10399](https://github.com/matrix-org/synapse/issues/10399)) -- Make deprecation notice of the spam checker doc more obvious. ([\#10395](https://github.com/matrix-org/synapse/issues/10395)) -- Add instructions on installing Debian packages for release candidates. ([\#10396](https://github.com/matrix-org/synapse/issues/10396)) - - -Deprecations and Removals -------------------------- - -- Remove functionality associated with the unused `room_stats_historical` and `user_stats_historical` tables. Contributed by @xmunoz. ([\#9721](https://github.com/matrix-org/synapse/issues/9721)) -- The third-party event rules module interface is deprecated in favour of the generic module interface introduced in Synapse v1.37.0. See the [upgrade notes](https://matrix-org.github.io/synapse/latest/upgrade.html#upgrading-to-v1390) for more information. ([\#10386](https://github.com/matrix-org/synapse/issues/10386)) - - -Internal Changes ----------------- - -- Convert `room_depth.min_depth` column to a `BIGINT`. ([\#10289](https://github.com/matrix-org/synapse/issues/10289)) -- Add tests to characterise the current behaviour of R30 phone-home metrics. ([\#10315](https://github.com/matrix-org/synapse/issues/10315)) -- Rebuild event context and auth when processing specific results from `ThirdPartyEventRules` modules. ([\#10316](https://github.com/matrix-org/synapse/issues/10316)) -- Minor change to the code that populates `user_daily_visits`. ([\#10324](https://github.com/matrix-org/synapse/issues/10324)) -- Re-enable Sytests that were disabled for the 1.37.1 release. ([\#10345](https://github.com/matrix-org/synapse/issues/10345), [\#10357](https://github.com/matrix-org/synapse/issues/10357)) -- Run `pyupgrade` on the codebase. ([\#10347](https://github.com/matrix-org/synapse/issues/10347), [\#10348](https://github.com/matrix-org/synapse/issues/10348)) -- Switch `application_services_txns.txn_id` database column to `BIGINT`. ([\#10349](https://github.com/matrix-org/synapse/issues/10349)) -- Convert internal type variable syntax to reflect wider ecosystem use. ([\#10350](https://github.com/matrix-org/synapse/issues/10350), [\#10380](https://github.com/matrix-org/synapse/issues/10380), [\#10381](https://github.com/matrix-org/synapse/issues/10381), [\#10382](https://github.com/matrix-org/synapse/issues/10382), [\#10418](https://github.com/matrix-org/synapse/issues/10418)) -- Make the Github Actions workflow configuration more efficient. ([\#10383](https://github.com/matrix-org/synapse/issues/10383)) -- Add type hints to `get_{domain,localpart}_from_id`. ([\#10385](https://github.com/matrix-org/synapse/issues/10385)) -- When building Debian packages for prerelease versions, set the Section accordingly. ([\#10391](https://github.com/matrix-org/synapse/issues/10391)) -- Add type hints and comments to event auth code. ([\#10393](https://github.com/matrix-org/synapse/issues/10393)) -- Stagger sending of presence update to remote servers, reducing CPU spikes caused by starting many connections to remote servers at once. ([\#10398](https://github.com/matrix-org/synapse/issues/10398)) -- Remove unused `events_by_room` code (tech debt). ([\#10421](https://github.com/matrix-org/synapse/issues/10421)) -- Add a github actions job which records success of other jobs. ([\#10430](https://github.com/matrix-org/synapse/issues/10430)) - - -Synapse 1.38.0 (2021-07-13) -=========================== - -This release includes a database schema update which could result in elevated disk usage. See the [upgrade notes](https://matrix-org.github.io/synapse/develop/upgrade#upgrading-to-v1380) for more information. - -No significant changes since 1.38.0rc3. - - -Synapse 1.38.0rc3 (2021-07-13) -============================== - -Internal Changes ----------------- - -- Build the Debian packages in CI. ([\#10247](https://github.com/matrix-org/synapse/issues/10247), [\#10379](https://github.com/matrix-org/synapse/issues/10379)) - - -Synapse 1.38.0rc2 (2021-07-09) -============================== - -Bugfixes --------- - -- Fix bug where inbound federation in a room could be delayed due to not correctly dropping a lock. Introduced in v1.37.1. ([\#10336](https://github.com/matrix-org/synapse/issues/10336)) - - -Improved Documentation ----------------------- - -- Update links to documentation in the sample config. Contributed by @dklimpel. ([\#10287](https://github.com/matrix-org/synapse/issues/10287)) -- Fix broken links in [INSTALL.md](INSTALL.md). Contributed by @dklimpel. ([\#10331](https://github.com/matrix-org/synapse/issues/10331)) - - -Synapse 1.38.0rc1 (2021-07-06) -============================== - -Features --------- - -- Implement refresh tokens as specified by [MSC2918](https://github.com/matrix-org/matrix-doc/pull/2918). ([\#9450](https://github.com/matrix-org/synapse/issues/9450)) -- Add support for evicting cache entries based on last access time. ([\#10205](https://github.com/matrix-org/synapse/issues/10205)) -- Omit empty fields from the `/sync` response. Contributed by @deepbluev7. ([\#10214](https://github.com/matrix-org/synapse/issues/10214)) -- Improve validation on federation `send_{join,leave,knock}` endpoints. ([\#10225](https://github.com/matrix-org/synapse/issues/10225), [\#10243](https://github.com/matrix-org/synapse/issues/10243)) -- Add SSO `external_ids` to the Query User Account admin API. ([\#10261](https://github.com/matrix-org/synapse/issues/10261)) -- Mark events received over federation which fail a spam check as "soft-failed". ([\#10263](https://github.com/matrix-org/synapse/issues/10263)) -- Add metrics for new inbound federation staging area. ([\#10284](https://github.com/matrix-org/synapse/issues/10284)) -- Add script to print information about recently registered users. ([\#10290](https://github.com/matrix-org/synapse/issues/10290)) - - -Bugfixes --------- - -- Fix a long-standing bug which meant that invite rejections and knocks were not sent out over federation in a timely manner. ([\#10223](https://github.com/matrix-org/synapse/issues/10223)) -- Fix a bug introduced in v1.26.0 where only users who have set profile information could be deactivated with erasure enabled. ([\#10252](https://github.com/matrix-org/synapse/issues/10252)) -- Fix a long-standing bug where Synapse would return errors after 231 events were handled by the server. ([\#10264](https://github.com/matrix-org/synapse/issues/10264), [\#10267](https://github.com/matrix-org/synapse/issues/10267), [\#10282](https://github.com/matrix-org/synapse/issues/10282), [\#10286](https://github.com/matrix-org/synapse/issues/10286), [\#10291](https://github.com/matrix-org/synapse/issues/10291), [\#10314](https://github.com/matrix-org/synapse/issues/10314), [\#10326](https://github.com/matrix-org/synapse/issues/10326)) -- Fix the prometheus `synapse_federation_server_pdu_process_time` metric. Broke in v1.37.1. ([\#10279](https://github.com/matrix-org/synapse/issues/10279)) -- Ensure that inbound events from federation that were being processed when Synapse was restarted get promptly processed on start up. ([\#10303](https://github.com/matrix-org/synapse/issues/10303)) - - -Improved Documentation ----------------------- - -- Move the upgrade notes to [docs/upgrade.md](https://github.com/matrix-org/synapse/blob/develop/docs/upgrade.md) and convert them to markdown. ([\#10166](https://github.com/matrix-org/synapse/issues/10166)) -- Choose Welcome & Overview as the default page for synapse documentation website. ([\#10242](https://github.com/matrix-org/synapse/issues/10242)) -- Adjust the URL in the README.rst file to point to irc.libera.chat. ([\#10258](https://github.com/matrix-org/synapse/issues/10258)) -- Fix homeserver config option name in presence router documentation. ([\#10288](https://github.com/matrix-org/synapse/issues/10288)) -- Fix link pointing at the wrong section in the modules documentation page. ([\#10302](https://github.com/matrix-org/synapse/issues/10302)) - - -Internal Changes ----------------- - -- Drop `Origin` and `Accept` from the value of the `Access-Control-Allow-Headers` response header. ([\#10114](https://github.com/matrix-org/synapse/issues/10114)) -- Add type hints to the federation servlets. ([\#10213](https://github.com/matrix-org/synapse/issues/10213)) -- Improve the reliability of auto-joining remote rooms. ([\#10237](https://github.com/matrix-org/synapse/issues/10237)) -- Update the release script to use the semver terminology and determine the release branch based on the next version. ([\#10239](https://github.com/matrix-org/synapse/issues/10239)) -- Fix type hints for computing auth events. ([\#10253](https://github.com/matrix-org/synapse/issues/10253)) -- Improve the performance of the spaces summary endpoint by only recursing into spaces (and not rooms in general). ([\#10256](https://github.com/matrix-org/synapse/issues/10256)) -- Move event authentication methods from `Auth` to `EventAuthHandler`. ([\#10268](https://github.com/matrix-org/synapse/issues/10268)) -- Re-enable a SyTest after it has been fixed. ([\#10292](https://github.com/matrix-org/synapse/issues/10292)) - - -Synapse 1.37.1 (2021-06-30) -=========================== - -This release resolves issues (such as [#9490](https://github.com/matrix-org/synapse/issues/9490)) where one busy room could cause head-of-line blocking, starving Synapse from processing events in other rooms, and causing all federated traffic to fall behind. Synapse 1.37.1 processes inbound federation traffic asynchronously, ensuring that one busy room won't impact others. Please upgrade to Synapse 1.37.1 as soon as possible, in order to increase resilience to other traffic spikes. - -No significant changes since v1.37.1rc1. - - -Synapse 1.37.1rc1 (2021-06-29) -============================== - -Features --------- - -- Handle inbound events from federation asynchronously. ([\#10269](https://github.com/matrix-org/synapse/issues/10269), [\#10272](https://github.com/matrix-org/synapse/issues/10272)) - - -Synapse 1.37.0 (2021-06-29) -=========================== - -This release deprecates the current spam checker interface. See the [upgrade notes](https://matrix-org.github.io/synapse/develop/upgrade#deprecation-of-the-current-spam-checker-interface) for more information on how to update to the new generic module interface. - -This release also removes support for fetching and renewing TLS certificates using the ACME v1 protocol, which has been fully decommissioned by Let's Encrypt on June 1st 2021. Admins previously using this feature should use a [reverse proxy](https://matrix-org.github.io/synapse/develop/reverse_proxy.html) to handle TLS termination, or use an external ACME client (such as [certbot](https://certbot.eff.org/)) to retrieve a certificate and key and provide them to Synapse using the `tls_certificate_path` and `tls_private_key_path` configuration settings. - -Synapse 1.37.0rc1 (2021-06-24) -============================== - -Features --------- - -- Implement "room knocking" as per [MSC2403](https://github.com/matrix-org/matrix-doc/pull/2403). Contributed by @Sorunome and anoa. ([\#6739](https://github.com/matrix-org/synapse/issues/6739), [\#9359](https://github.com/matrix-org/synapse/issues/9359), [\#10167](https://github.com/matrix-org/synapse/issues/10167), [\#10212](https://github.com/matrix-org/synapse/issues/10212), [\#10227](https://github.com/matrix-org/synapse/issues/10227)) -- Add experimental support for backfilling history into rooms ([MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716)). ([\#9247](https://github.com/matrix-org/synapse/issues/9247)) -- Implement a generic interface for third-party plugin modules. ([\#10062](https://github.com/matrix-org/synapse/issues/10062), [\#10206](https://github.com/matrix-org/synapse/issues/10206)) -- Implement config option `sso.update_profile_information` to sync SSO users' profile information with the identity provider each time they login. Currently only displayname is supported. ([\#10108](https://github.com/matrix-org/synapse/issues/10108)) -- Ensure that errors during startup are written to the logs and the console. ([\#10191](https://github.com/matrix-org/synapse/issues/10191)) - - -Bugfixes --------- - -- Fix a bug introduced in Synapse v1.25.0 that prevented the `ip_range_whitelist` configuration option from working for federation and identity servers. Contributed by @mikure. ([\#10115](https://github.com/matrix-org/synapse/issues/10115)) -- Remove a broken import line in Synapse's `admin_cmd` worker. Broke in Synapse v1.33.0. ([\#10154](https://github.com/matrix-org/synapse/issues/10154)) -- Fix a bug introduced in Synapse v1.21.0 which could cause `/sync` to return immediately with an empty response. ([\#10157](https://github.com/matrix-org/synapse/issues/10157), [\#10158](https://github.com/matrix-org/synapse/issues/10158)) -- Fix a minor bug in the response to `/_matrix/client/r0/user/{user}/openid/request_token` causing `expires_in` to be a float instead of an integer. Contributed by @lukaslihotzki. ([\#10175](https://github.com/matrix-org/synapse/issues/10175)) -- Always require users to re-authenticate for dangerous operations: deactivating an account, modifying an account password, and adding 3PIDs. ([\#10184](https://github.com/matrix-org/synapse/issues/10184)) -- Fix a bug introduced in Synpase v1.7.2 where remote server count metrics collection would be incorrectly delayed on startup. Found by @heftig. ([\#10195](https://github.com/matrix-org/synapse/issues/10195)) -- Fix a bug introduced in Synapse v1.35.1 where an `allow` key of a `m.room.join_rules` event could be applied for incorrect room versions and configurations. ([\#10208](https://github.com/matrix-org/synapse/issues/10208)) -- Fix performance regression in responding to user key requests over federation. Introduced in Synapse v1.34.0rc1. ([\#10221](https://github.com/matrix-org/synapse/issues/10221)) - - -Improved Documentation ----------------------- - -- Add a new guide to decoding request logs. ([\#8436](https://github.com/matrix-org/synapse/issues/8436)) -- Mention in the sample homeserver config that you may need to configure max upload size in your reverse proxy. Contributed by @aaronraimist. ([\#10122](https://github.com/matrix-org/synapse/issues/10122)) -- Fix broken links in documentation. ([\#10180](https://github.com/matrix-org/synapse/issues/10180)) -- Deploy a snapshot of the documentation website upon each new Synapse release. ([\#10198](https://github.com/matrix-org/synapse/issues/10198)) - - -Deprecations and Removals -------------------------- - -- The current spam checker interface is deprecated in favour of a new generic modules system. See the [upgrade notes](https://matrix-org.github.io/synapse/develop/upgrade#deprecation-of-the-current-spam-checker-interface) for more information on how to update to the new system. ([\#10062](https://github.com/matrix-org/synapse/issues/10062), [\#10210](https://github.com/matrix-org/synapse/issues/10210), [\#10238](https://github.com/matrix-org/synapse/issues/10238)) -- Stop supporting the unstable spaces prefixes from MSC1772. ([\#10161](https://github.com/matrix-org/synapse/issues/10161)) -- Remove Synapse's support for automatically fetching and renewing certificates using the ACME v1 protocol. This protocol has been fully turned off by Let's Encrypt for existing installations on June 1st 2021. Admins previously using this feature should use a [reverse proxy](https://matrix-org.github.io/synapse/develop/reverse_proxy.html) to handle TLS termination, or use an external ACME client (such as [certbot](https://certbot.eff.org/)) to retrieve a certificate and key and provide them to Synapse using the `tls_certificate_path` and `tls_private_key_path` configuration settings. ([\#10194](https://github.com/matrix-org/synapse/issues/10194)) - - -Internal Changes ----------------- - -- Update the database schema versioning to support gradual migration away from legacy tables. ([\#9933](https://github.com/matrix-org/synapse/issues/9933)) -- Add type hints to the federation servlets. ([\#10080](https://github.com/matrix-org/synapse/issues/10080)) -- Improve OpenTracing for event persistence. ([\#10134](https://github.com/matrix-org/synapse/issues/10134), [\#10193](https://github.com/matrix-org/synapse/issues/10193)) -- Clean up the interface for injecting OpenTracing over HTTP. ([\#10143](https://github.com/matrix-org/synapse/issues/10143)) -- Limit the number of in-flight `/keys/query` requests from a single device. ([\#10144](https://github.com/matrix-org/synapse/issues/10144)) -- Refactor EventPersistenceQueue. ([\#10145](https://github.com/matrix-org/synapse/issues/10145)) -- Document `SYNAPSE_TEST_LOG_LEVEL` to see the logger output when running tests. ([\#10148](https://github.com/matrix-org/synapse/issues/10148)) -- Update the Complement build tags in GitHub Actions to test currently experimental features. ([\#10155](https://github.com/matrix-org/synapse/issues/10155)) -- Add a `synapse_federation_soft_failed_events_total` metric to track how often events are soft failed. ([\#10156](https://github.com/matrix-org/synapse/issues/10156)) -- Fetch the corresponding complement branch when performing CI. ([\#10160](https://github.com/matrix-org/synapse/issues/10160)) -- Add some developer documentation about boolean columns in database schemas. ([\#10164](https://github.com/matrix-org/synapse/issues/10164)) -- Add extra logging fields to better debug where events are being soft failed. ([\#10168](https://github.com/matrix-org/synapse/issues/10168)) -- Add debug logging for when we enter and exit `Measure` blocks. ([\#10183](https://github.com/matrix-org/synapse/issues/10183)) -- Improve comments in structured logging code. ([\#10188](https://github.com/matrix-org/synapse/issues/10188)) -- Update [MSC3083](https://github.com/matrix-org/matrix-doc/pull/3083) support with modifications from the MSC. ([\#10189](https://github.com/matrix-org/synapse/issues/10189)) -- Remove redundant DNS lookup limiter. ([\#10190](https://github.com/matrix-org/synapse/issues/10190)) -- Upgrade `black` linting tool to 21.6b0. ([\#10197](https://github.com/matrix-org/synapse/issues/10197)) -- Expose OpenTracing trace id in response headers. ([\#10199](https://github.com/matrix-org/synapse/issues/10199)) - - -Synapse 1.36.0 (2021-06-15) -=========================== - -No significant changes. - - -Synapse 1.36.0rc2 (2021-06-11) -============================== - -Bugfixes --------- - -- Fix a bug which caused presence updates to stop working some time after a restart, when using a presence writer worker. Broke in v1.33.0. ([\#10149](https://github.com/matrix-org/synapse/issues/10149)) -- Fix a bug when using federation sender worker where it would send out more presence updates than necessary, leading to high resource usage. Broke in v1.33.0. ([\#10163](https://github.com/matrix-org/synapse/issues/10163)) -- Fix a bug where Synapse could send the same presence update to a remote twice. ([\#10165](https://github.com/matrix-org/synapse/issues/10165)) - - -Synapse 1.36.0rc1 (2021-06-08) -============================== - -Features --------- - -- Add new endpoint `/_matrix/client/r0/rooms/{roomId}/aliases` from Client-Server API r0.6.1 (previously [MSC2432](https://github.com/matrix-org/matrix-doc/pull/2432)). ([\#9224](https://github.com/matrix-org/synapse/issues/9224)) -- Improve performance of incoming federation transactions in large rooms. ([\#9953](https://github.com/matrix-org/synapse/issues/9953), [\#9973](https://github.com/matrix-org/synapse/issues/9973)) -- Rewrite logic around verifying JSON object and fetching server keys to be more performant and use less memory. ([\#10035](https://github.com/matrix-org/synapse/issues/10035)) -- Add new admin APIs for unprotecting local media from quarantine. Contributed by @dklimpel. ([\#10040](https://github.com/matrix-org/synapse/issues/10040)) -- Add new admin APIs to remove media by media ID from quarantine. Contributed by @dklimpel. ([\#10044](https://github.com/matrix-org/synapse/issues/10044)) -- Make reason and score parameters optional for reporting content. Implements [MSC2414](https://github.com/matrix-org/matrix-doc/pull/2414). Contributed by Callum Brown. ([\#10077](https://github.com/matrix-org/synapse/issues/10077)) -- Add support for routing more requests to workers. ([\#10084](https://github.com/matrix-org/synapse/issues/10084)) -- Report OpenTracing spans for database activity. ([\#10113](https://github.com/matrix-org/synapse/issues/10113), [\#10136](https://github.com/matrix-org/synapse/issues/10136), [\#10141](https://github.com/matrix-org/synapse/issues/10141)) -- Significantly reduce memory usage of joining large remote rooms. ([\#10117](https://github.com/matrix-org/synapse/issues/10117)) - - -Bugfixes --------- - -- Fixed a bug causing replication requests to fail when receiving a lot of events via federation. ([\#10082](https://github.com/matrix-org/synapse/issues/10082)) -- Fix a bug in the `force_tracing_for_users` option introduced in Synapse v1.35 which meant that the OpenTracing spans produced were missing most tags. ([\#10092](https://github.com/matrix-org/synapse/issues/10092)) -- Fixed a bug that could cause Synapse to stop notifying application services. Contributed by Willem Mulder. ([\#10107](https://github.com/matrix-org/synapse/issues/10107)) -- Fix bug where the server would attempt to fetch the same history in the room from a remote server multiple times in parallel. ([\#10116](https://github.com/matrix-org/synapse/issues/10116)) -- Fix a bug introduced in Synapse 1.33.0 which caused replication requests to fail when receiving a lot of very large events via federation. ([\#10118](https://github.com/matrix-org/synapse/issues/10118)) -- Fix bug when using workers where pagination requests failed if a remote server returned zero events from `/backfill`. Introduced in 1.35.0. ([\#10133](https://github.com/matrix-org/synapse/issues/10133)) - - -Improved Documentation ----------------------- - -- Clarify security note regarding hosting Synapse on the same domain as other web applications. ([\#9221](https://github.com/matrix-org/synapse/issues/9221)) -- Update CAPTCHA documentation to mention turning off the verify origin feature. Contributed by @aaronraimist. ([\#10046](https://github.com/matrix-org/synapse/issues/10046)) -- Tweak wording of database recommendation in `INSTALL.md`. Contributed by @aaronraimist. ([\#10057](https://github.com/matrix-org/synapse/issues/10057)) -- Add initial infrastructure for rendering Synapse documentation with mdbook. ([\#10086](https://github.com/matrix-org/synapse/issues/10086)) -- Convert the remaining Admin API documentation files to markdown. ([\#10089](https://github.com/matrix-org/synapse/issues/10089)) -- Make a link in docs use HTTPS. Contributed by @RhnSharma. ([\#10130](https://github.com/matrix-org/synapse/issues/10130)) -- Fix broken link in Docker docs. ([\#10132](https://github.com/matrix-org/synapse/issues/10132)) - - -Deprecations and Removals -------------------------- - -- Remove the experimental `spaces_enabled` flag. The spaces features are always available now. ([\#10063](https://github.com/matrix-org/synapse/issues/10063)) - - -Internal Changes ----------------- - -- Tell CircleCI to build Docker images from `main` branch. ([\#9906](https://github.com/matrix-org/synapse/issues/9906)) -- Simplify naming convention for release branches to only include the major and minor version numbers. ([\#10013](https://github.com/matrix-org/synapse/issues/10013)) -- Add `parse_strings_from_args` for parsing an array from query parameters. ([\#10048](https://github.com/matrix-org/synapse/issues/10048), [\#10137](https://github.com/matrix-org/synapse/issues/10137)) -- Remove some dead code regarding TLS certificate handling. ([\#10054](https://github.com/matrix-org/synapse/issues/10054)) -- Remove redundant, unmaintained `convert_server_keys` script. ([\#10055](https://github.com/matrix-org/synapse/issues/10055)) -- Improve the error message printed by synctl when synapse fails to start. ([\#10059](https://github.com/matrix-org/synapse/issues/10059)) -- Fix GitHub Actions lint for newsfragments. ([\#10069](https://github.com/matrix-org/synapse/issues/10069)) -- Update opentracing to inject the right context into the carrier. ([\#10074](https://github.com/matrix-org/synapse/issues/10074)) -- Fix up `BatchingQueue` implementation. ([\#10078](https://github.com/matrix-org/synapse/issues/10078)) -- Log method and path when dropping request due to size limit. ([\#10091](https://github.com/matrix-org/synapse/issues/10091)) -- In Github Actions workflows, summarize the Sytest results in an easy-to-read format. ([\#10094](https://github.com/matrix-org/synapse/issues/10094)) -- Make `/sync` do fewer state resolutions. ([\#10102](https://github.com/matrix-org/synapse/issues/10102)) -- Add missing type hints to the admin API servlets. ([\#10105](https://github.com/matrix-org/synapse/issues/10105)) -- Improve opentracing annotations for `Notifier`. ([\#10111](https://github.com/matrix-org/synapse/issues/10111)) -- Enable Prometheus metrics for the jaeger client library. ([\#10112](https://github.com/matrix-org/synapse/issues/10112)) -- Work to improve the responsiveness of `/sync` requests. ([\#10124](https://github.com/matrix-org/synapse/issues/10124)) -- OpenTracing: use a consistent name for background processes. ([\#10135](https://github.com/matrix-org/synapse/issues/10135)) - - -Synapse 1.35.1 (2021-06-03) -=========================== - -Bugfixes --------- - -- Fix a bug introduced in v1.35.0 where invite-only rooms would be shown to all users in a space, regardless of if the user had access to it. ([\#10109](https://github.com/matrix-org/synapse/issues/10109)) - - -Synapse 1.35.0 (2021-06-01) -=========================== - -Note that [the tag](https://github.com/matrix-org/synapse/releases/tag/v1.35.0rc3) and [docker images](https://hub.docker.com/layers/matrixdotorg/synapse/v1.35.0rc3/images/sha256-34ccc87bd99a17e2cbc0902e678b5937d16bdc1991ead097eee6096481ecf2c4?context=explore) for `v1.35.0rc3` were incorrectly built. If you are experiencing issues with either, it is recommended to upgrade to the equivalent tag or docker image for the `v1.35.0` release. - -Deprecations and Removals -------------------------- - -- The core Synapse development team plan to drop support for the [unstable API of MSC2858](https://github.com/matrix-org/matrix-doc/blob/master/proposals/2858-Multiple-SSO-Identity-Providers.md#unstable-prefix), including the undocumented `experimental.msc2858_enabled` config option, in August 2021. Client authors should ensure that their clients are updated to use the stable API (which has been supported since Synapse 1.30) well before that time, to give their users time to upgrade. ([\#10101](https://github.com/matrix-org/synapse/issues/10101)) - -Bugfixes --------- - -- Fixed a bug causing replication requests to fail when receiving a lot of events via federation. Introduced in v1.33.0. ([\#10082](https://github.com/matrix-org/synapse/issues/10082)) -- Fix HTTP response size limit to allow joining very large rooms over federation. Introduced in v1.33.0. ([\#10093](https://github.com/matrix-org/synapse/issues/10093)) - - -Internal Changes ----------------- - -- Log method and path when dropping request due to size limit. ([\#10091](https://github.com/matrix-org/synapse/issues/10091)) - - -Synapse 1.35.0rc2 (2021-05-27) -============================== - -Bugfixes --------- - -- Fix a bug introduced in v1.35.0rc1 when calling the spaces summary API via a GET request. ([\#10079](https://github.com/matrix-org/synapse/issues/10079)) - - -Synapse 1.35.0rc1 (2021-05-25) -============================== - -Features --------- - -- Add experimental support to allow a user who could join a restricted room to view it in the spaces summary. ([\#9922](https://github.com/matrix-org/synapse/issues/9922), [\#10007](https://github.com/matrix-org/synapse/issues/10007), [\#10038](https://github.com/matrix-org/synapse/issues/10038)) -- Reduce memory usage when joining very large rooms over federation. ([\#9958](https://github.com/matrix-org/synapse/issues/9958)) -- Add a configuration option which allows enabling opentracing by user id. ([\#9978](https://github.com/matrix-org/synapse/issues/9978)) -- Enable experimental support for [MSC2946](https://github.com/matrix-org/matrix-doc/pull/2946) (spaces summary API) and [MSC3083](https://github.com/matrix-org/matrix-doc/pull/3083) (restricted join rules) by default. ([\#10011](https://github.com/matrix-org/synapse/issues/10011)) - - -Bugfixes --------- - -- Fix a bug introduced in v1.26.0 which meant that `synapse_port_db` would not correctly initialise some postgres sequences, requiring manual updates afterwards. ([\#9991](https://github.com/matrix-org/synapse/issues/9991)) -- Fix `synctl`'s `--no-daemonize` parameter to work correctly with worker processes. ([\#9995](https://github.com/matrix-org/synapse/issues/9995)) -- Fix a validation bug introduced in v1.34.0 in the ordering of spaces in the space summary API. ([\#10002](https://github.com/matrix-org/synapse/issues/10002)) -- Fixed deletion of new presence stream states from database. ([\#10014](https://github.com/matrix-org/synapse/issues/10014), [\#10033](https://github.com/matrix-org/synapse/issues/10033)) -- Fixed a bug with very high resolution image uploads throwing internal server errors. ([\#10029](https://github.com/matrix-org/synapse/issues/10029)) - - -Updates to the Docker image ---------------------------- - -- Fix bug introduced in Synapse 1.33.0 which caused a `Permission denied: '/homeserver.log'` error when starting Synapse with the generated log configuration. Contributed by Sergio Miguéns Iglesias. ([\#10045](https://github.com/matrix-org/synapse/issues/10045)) - - -Improved Documentation ----------------------- - -- Add hardened systemd files as proposed in [#9760](https://github.com/matrix-org/synapse/issues/9760) and added them to `contrib/`. Change the docs to reflect the presence of these files. ([\#9803](https://github.com/matrix-org/synapse/issues/9803)) -- Clarify documentation around SSO mapping providers generating unique IDs and localparts. ([\#9980](https://github.com/matrix-org/synapse/issues/9980)) -- Updates to the PostgreSQL documentation (`postgres.md`). ([\#9988](https://github.com/matrix-org/synapse/issues/9988), [\#9989](https://github.com/matrix-org/synapse/issues/9989)) -- Fix broken link in user directory documentation. Contributed by @junquera. ([\#10016](https://github.com/matrix-org/synapse/issues/10016)) -- Add missing room state entry to the table of contents of room admin API. ([\#10043](https://github.com/matrix-org/synapse/issues/10043)) - - -Deprecations and Removals -------------------------- - -- Removed support for the deprecated `tls_fingerprints` configuration setting. Contributed by Jerin J Titus. ([\#9280](https://github.com/matrix-org/synapse/issues/9280)) - - -Internal Changes ----------------- - -- Allow sending full presence to users via workers other than the one that called `ModuleApi.send_local_online_presence_to`. ([\#9823](https://github.com/matrix-org/synapse/issues/9823)) -- Update comments in the space summary handler. ([\#9974](https://github.com/matrix-org/synapse/issues/9974)) -- Minor enhancements to the `@cachedList` descriptor. ([\#9975](https://github.com/matrix-org/synapse/issues/9975)) -- Split multipart email sending into a dedicated handler. ([\#9977](https://github.com/matrix-org/synapse/issues/9977)) -- Run `black` on files in the `scripts` directory. ([\#9981](https://github.com/matrix-org/synapse/issues/9981)) -- Add missing type hints to `synapse.util` module. ([\#9982](https://github.com/matrix-org/synapse/issues/9982)) -- Simplify a few helper functions. ([\#9984](https://github.com/matrix-org/synapse/issues/9984), [\#9985](https://github.com/matrix-org/synapse/issues/9985), [\#9986](https://github.com/matrix-org/synapse/issues/9986)) -- Remove unnecessary property from SQLBaseStore. ([\#9987](https://github.com/matrix-org/synapse/issues/9987)) -- Remove `keylen` param on `LruCache`. ([\#9993](https://github.com/matrix-org/synapse/issues/9993)) -- Update the Grafana dashboard in `contrib/`. ([\#10001](https://github.com/matrix-org/synapse/issues/10001)) -- Add a batching queue implementation. ([\#10017](https://github.com/matrix-org/synapse/issues/10017)) -- Reduce memory usage when verifying signatures on large numbers of events at once. ([\#10018](https://github.com/matrix-org/synapse/issues/10018)) -- Properly invalidate caches for destination retry timings every (instead of expiring entries every 5 minutes). ([\#10036](https://github.com/matrix-org/synapse/issues/10036)) -- Fix running complement tests with Synapse workers. ([\#10039](https://github.com/matrix-org/synapse/issues/10039)) -- Fix typo in `get_state_ids_for_event` docstring where the return type was incorrect. ([\#10050](https://github.com/matrix-org/synapse/issues/10050)) - - -Synapse 1.34.0 (2021-05-17) -=========================== - -This release deprecates the `room_invite_state_types` configuration setting. See the [upgrade notes](https://github.com/matrix-org/synapse/blob/release-v1.34.0/UPGRADE.rst#upgrading-to-v1340) for instructions on updating your configuration file to use the new `room_prejoin_state` setting. - -This release also deprecates the `POST /_synapse/admin/v1/rooms//delete` admin API route. Server administrators are encouraged to update their scripts to use the new `DELETE /_synapse/admin/v1/rooms/` route instead. - - -No significant changes since v1.34.0rc1. - - -Synapse 1.34.0rc1 (2021-05-12) -============================== - -Features --------- - -- Add experimental option to track memory usage of the caches. ([\#9881](https://github.com/matrix-org/synapse/issues/9881)) -- Add support for `DELETE /_synapse/admin/v1/rooms/`. ([\#9889](https://github.com/matrix-org/synapse/issues/9889)) -- Add limits to how often Synapse will GC, ensuring that large servers do not end up GC thrashing if `gc_thresholds` has not been correctly set. ([\#9902](https://github.com/matrix-org/synapse/issues/9902)) -- Improve performance of sending events for worker-based deployments using Redis. ([\#9905](https://github.com/matrix-org/synapse/issues/9905), [\#9950](https://github.com/matrix-org/synapse/issues/9950), [\#9951](https://github.com/matrix-org/synapse/issues/9951)) -- Improve performance after joining a large room when presence is enabled. ([\#9910](https://github.com/matrix-org/synapse/issues/9910), [\#9916](https://github.com/matrix-org/synapse/issues/9916)) -- Support stable identifiers for [MSC1772](https://github.com/matrix-org/matrix-doc/pull/1772) Spaces. `m.space.child` events will now be taken into account when populating the experimental spaces summary response. Please see [the upgrade notes](https://github.com/matrix-org/synapse/blob/release-v1.34.0/UPGRADE.rst#upgrading-to-v1340) if you have customised `room_invite_state_types` in your configuration. ([\#9915](https://github.com/matrix-org/synapse/issues/9915), [\#9966](https://github.com/matrix-org/synapse/issues/9966)) -- Improve performance of backfilling in large rooms. ([\#9935](https://github.com/matrix-org/synapse/issues/9935)) -- Add a config option to allow you to prevent device display names from being shared over federation. Contributed by @aaronraimist. ([\#9945](https://github.com/matrix-org/synapse/issues/9945)) -- Update support for [MSC2946](https://github.com/matrix-org/matrix-doc/pull/2946): Spaces Summary. ([\#9947](https://github.com/matrix-org/synapse/issues/9947), [\#9954](https://github.com/matrix-org/synapse/issues/9954)) - - -Bugfixes --------- - -- Fix a bug introduced in v1.32.0 where the associated connection was improperly logged for SQL logging statements. ([\#9895](https://github.com/matrix-org/synapse/issues/9895)) -- Correct the type hint for the `user_may_create_room_alias` method of spam checkers. It is provided a `RoomAlias`, not a `str`. ([\#9896](https://github.com/matrix-org/synapse/issues/9896)) -- Fix bug where user directory could get out of sync if room visibility and membership changed in quick succession. ([\#9910](https://github.com/matrix-org/synapse/issues/9910)) -- Include the `origin_server_ts` property in the experimental [MSC2946](https://github.com/matrix-org/matrix-doc/pull/2946) support to allow clients to properly sort rooms. ([\#9928](https://github.com/matrix-org/synapse/issues/9928)) -- Fix bugs introduced in v1.23.0 which made the PostgreSQL port script fail when run with a newly-created SQLite database. ([\#9930](https://github.com/matrix-org/synapse/issues/9930)) -- Fix a bug introduced in Synapse 1.29.0 which caused `m.room_key_request` to-device messages sent from one user to another to be dropped. ([\#9961](https://github.com/matrix-org/synapse/issues/9961), [\#9965](https://github.com/matrix-org/synapse/issues/9965)) -- Fix a bug introduced in v1.27.0 preventing users and appservices exempt from ratelimiting from creating rooms with many invitees. ([\#9968](https://github.com/matrix-org/synapse/issues/9968)) - - -Updates to the Docker image ---------------------------- - -- Add `startup_delay` to docker healthcheck to reduce waiting time for coming online and update the documentation with extra options. Contributed by @Maquis196. ([\#9913](https://github.com/matrix-org/synapse/issues/9913)) - - -Improved Documentation ----------------------- - -- Add `port` argument to the Postgres database sample config section. ([\#9911](https://github.com/matrix-org/synapse/issues/9911)) - - -Deprecations and Removals -------------------------- - -- Mark as deprecated `POST /_synapse/admin/v1/rooms//delete`. ([\#9889](https://github.com/matrix-org/synapse/issues/9889)) - - -Internal Changes ----------------- - -- Reduce the length of Synapse's access tokens. ([\#5588](https://github.com/matrix-org/synapse/issues/5588)) -- Export jemalloc stats to Prometheus if it is being used. ([\#9882](https://github.com/matrix-org/synapse/issues/9882)) -- Add type hints to presence handler. ([\#9885](https://github.com/matrix-org/synapse/issues/9885)) -- Reduce memory usage of the LRU caches. ([\#9886](https://github.com/matrix-org/synapse/issues/9886)) -- Add type hints to the `synapse.handlers` module. ([\#9896](https://github.com/matrix-org/synapse/issues/9896)) -- Time response time for external cache requests. ([\#9904](https://github.com/matrix-org/synapse/issues/9904)) -- Minor fixes to the `make_full_schema.sh` script. ([\#9931](https://github.com/matrix-org/synapse/issues/9931)) -- Move database schema files into a common directory. ([\#9932](https://github.com/matrix-org/synapse/issues/9932)) -- Add debug logging for lost/delayed to-device messages. ([\#9959](https://github.com/matrix-org/synapse/issues/9959)) - - -Synapse 1.33.2 (2021-05-11) -=========================== - -Due to the security issue highlighted below, server administrators are encouraged to update Synapse. We are not aware of these vulnerabilities being exploited in the wild. - -Security advisory ------------------ - -This release fixes a denial of service attack ([CVE-2021-29471](https://github.com/matrix-org/synapse/security/advisories/GHSA-x345-32rc-8h85)) against Synapse's push rules implementation. Server admins are encouraged to upgrade. - -Internal Changes ----------------- - -- Unpin attrs dependency. ([\#9946](https://github.com/matrix-org/synapse/issues/9946)) - - -Synapse 1.33.1 (2021-05-06) -=========================== - -Bugfixes --------- - -- Fix bug where `/sync` would break if using the latest version of `attrs` dependency, by pinning to a previous version. ([\#9937](https://github.com/matrix-org/synapse/issues/9937)) - - -Synapse 1.33.0 (2021-05-05) -=========================== - -Features --------- - -- Build Debian packages for Ubuntu 21.04 (Hirsute Hippo). ([\#9909](https://github.com/matrix-org/synapse/issues/9909)) - - -Synapse 1.33.0rc2 (2021-04-29) -============================== - -Bugfixes --------- - -- Fix tight loop when handling presence replication when using workers. Introduced in v1.33.0rc1. ([\#9900](https://github.com/matrix-org/synapse/issues/9900)) - - -Synapse 1.33.0rc1 (2021-04-28) -============================== - -Features --------- - -- Update experimental support for [MSC3083](https://github.com/matrix-org/matrix-doc/pull/3083): restricting room access via group membership. ([\#9800](https://github.com/matrix-org/synapse/issues/9800), [\#9814](https://github.com/matrix-org/synapse/issues/9814)) -- Add experimental support for handling presence on a worker. ([\#9819](https://github.com/matrix-org/synapse/issues/9819), [\#9820](https://github.com/matrix-org/synapse/issues/9820), [\#9828](https://github.com/matrix-org/synapse/issues/9828), [\#9850](https://github.com/matrix-org/synapse/issues/9850)) -- Return a new template when an user attempts to renew their account multiple times with the same token, stating that their account is set to expire. This replaces the invalid token template that would previously be shown in this case. This change concerns the optional account validity feature. ([\#9832](https://github.com/matrix-org/synapse/issues/9832)) - - -Bugfixes --------- - -- Fixes the OIDC SSO flow when using a `public_baseurl` value including a non-root URL path. ([\#9726](https://github.com/matrix-org/synapse/issues/9726)) -- Fix thumbnail generation for some sites with non-standard content types. Contributed by @rkfg. ([\#9788](https://github.com/matrix-org/synapse/issues/9788)) -- Add some sanity checks to identity server passed to 3PID bind/unbind endpoints. ([\#9802](https://github.com/matrix-org/synapse/issues/9802)) -- Limit the size of HTTP responses read over federation. ([\#9833](https://github.com/matrix-org/synapse/issues/9833)) -- Fix a bug which could cause Synapse to get stuck in a loop of resyncing device lists. ([\#9867](https://github.com/matrix-org/synapse/issues/9867)) -- Fix a long-standing bug where errors from federation did not propagate to the client. ([\#9868](https://github.com/matrix-org/synapse/issues/9868)) - - -Improved Documentation ----------------------- - -- Add a note to the docker docs mentioning that we mirror upstream's supported Docker platforms. ([\#9801](https://github.com/matrix-org/synapse/issues/9801)) - - -Internal Changes ----------------- - -- Add a dockerfile for running Synapse in worker-mode under Complement. ([\#9162](https://github.com/matrix-org/synapse/issues/9162)) -- Apply `pyupgrade` across the codebase. ([\#9786](https://github.com/matrix-org/synapse/issues/9786)) -- Move some replication processing out of `generic_worker`. ([\#9796](https://github.com/matrix-org/synapse/issues/9796)) -- Replace `HomeServer.get_config()` with inline references. ([\#9815](https://github.com/matrix-org/synapse/issues/9815)) -- Rename some handlers and config modules to not duplicate the top-level module. ([\#9816](https://github.com/matrix-org/synapse/issues/9816)) -- Fix a long-standing bug which caused `max_upload_size` to not be correctly enforced. ([\#9817](https://github.com/matrix-org/synapse/issues/9817)) -- Reduce CPU usage of the user directory by reusing existing calculated room membership. ([\#9821](https://github.com/matrix-org/synapse/issues/9821)) -- Small speed up for joining large remote rooms. ([\#9825](https://github.com/matrix-org/synapse/issues/9825)) -- Introduce flake8-bugbear to the test suite and fix some of its lint violations. ([\#9838](https://github.com/matrix-org/synapse/issues/9838)) -- Only store the raw data in the in-memory caches, rather than objects that include references to e.g. the data stores. ([\#9845](https://github.com/matrix-org/synapse/issues/9845)) -- Limit length of accepted email addresses. ([\#9855](https://github.com/matrix-org/synapse/issues/9855)) -- Remove redundant `synapse.types.Collection` type definition. ([\#9856](https://github.com/matrix-org/synapse/issues/9856)) -- Handle recently added rate limits correctly when using `--no-rate-limit` with the demo scripts. ([\#9858](https://github.com/matrix-org/synapse/issues/9858)) -- Disable invite rate-limiting by default when running the unit tests. ([\#9871](https://github.com/matrix-org/synapse/issues/9871)) -- Pass a reactor into `SynapseSite` to make testing easier. ([\#9874](https://github.com/matrix-org/synapse/issues/9874)) -- Make `DomainSpecificString` an `attrs` class. ([\#9875](https://github.com/matrix-org/synapse/issues/9875)) -- Add type hints to `synapse.api.auth` and `synapse.api.auth_blocking` modules. ([\#9876](https://github.com/matrix-org/synapse/issues/9876)) -- Remove redundant `_PushHTTPChannel` test class. ([\#9878](https://github.com/matrix-org/synapse/issues/9878)) -- Remove backwards-compatibility code for Python versions < 3.6. ([\#9879](https://github.com/matrix-org/synapse/issues/9879)) -- Small performance improvement around handling new local presence updates. ([\#9887](https://github.com/matrix-org/synapse/issues/9887)) - - -Synapse 1.32.2 (2021-04-22) -=========================== - -This release includes a fix for a regression introduced in 1.32.0. - -Bugfixes --------- - -- Fix a regression in Synapse 1.32.0 and 1.32.1 which caused `LoggingContext` errors in plugins. ([\#9857](https://github.com/matrix-org/synapse/issues/9857)) - - -Synapse 1.32.1 (2021-04-21) -=========================== - -This release fixes [a regression](https://github.com/matrix-org/synapse/issues/9853) -in Synapse 1.32.0 that caused connected Prometheus instances to become unstable. - -However, as this release is still subject to the `LoggingContext` change in 1.32.0, -it is recommended to remain on or downgrade to 1.31.0. - -Bugfixes --------- - -- Fix a regression in Synapse 1.32.0 which caused Synapse to report large numbers of Prometheus time series, potentially overwhelming Prometheus instances. ([\#9854](https://github.com/matrix-org/synapse/issues/9854)) - - -Synapse 1.32.0 (2021-04-20) -=========================== - -**Note:** This release introduces [a regression](https://github.com/matrix-org/synapse/issues/9853) -that can overwhelm connected Prometheus instances. This issue was not present in -1.32.0rc1. If affected, it is recommended to downgrade to 1.31.0 in the meantime, and -follow [these instructions](https://github.com/matrix-org/synapse/pull/9854#issuecomment-823472183) -to clean up any excess writeahead logs. - -**Note:** This release also mistakenly included a change that may affected Synapse -modules that import `synapse.logging.context.LoggingContext`, such as -[synapse-s3-storage-provider](https://github.com/matrix-org/synapse-s3-storage-provider). -This will be fixed in a later Synapse version. - -**Note:** This release requires Python 3.6+ and Postgres 9.6+ or SQLite 3.22+. - -This release removes the deprecated `GET /_synapse/admin/v1/users/` admin API. Please use the [v2 API](https://github.com/matrix-org/synapse/blob/develop/docs/admin_api/user_admin_api.rst#query-user-account) instead, which has improved capabilities. - -This release requires Application Services to use type `m.login.application_service` when registering users via the `/_matrix/client/r0/register` endpoint to comply with the spec. Please ensure your Application Services are up to date. - -If you are using the `packages.matrix.org` Debian repository for Synapse packages, -note that we have recently updated the expiry date on the gpg signing key. If you see an -error similar to `The following signatures were invalid: EXPKEYSIG F473DD4473365DE1`, you -will need to get a fresh copy of the keys. You can do so with: - -```sh -sudo wget -O /usr/share/keyrings/matrix-org-archive-keyring.gpg https://packages.matrix.org/debian/matrix-org-archive-keyring.gpg -``` - -Bugfixes --------- - -- Fix the log lines of nested logging contexts. Broke in 1.32.0rc1. ([\#9829](https://github.com/matrix-org/synapse/issues/9829)) - - -Synapse 1.32.0rc1 (2021-04-13) -============================== - -Features --------- - -- Add a Synapse module for routing presence updates between users. ([\#9491](https://github.com/matrix-org/synapse/issues/9491)) -- Add an admin API to manage ratelimit for a specific user. ([\#9648](https://github.com/matrix-org/synapse/issues/9648)) -- Include request information in structured logging output. ([\#9654](https://github.com/matrix-org/synapse/issues/9654)) -- Add `order_by` to the admin API `GET /_synapse/admin/v2/users`. Contributed by @dklimpel. ([\#9691](https://github.com/matrix-org/synapse/issues/9691)) -- Replace the `room_invite_state_types` configuration setting with `room_prejoin_state`. ([\#9700](https://github.com/matrix-org/synapse/issues/9700)) -- Add experimental support for [MSC3083](https://github.com/matrix-org/matrix-doc/pull/3083): restricting room access via group membership. ([\#9717](https://github.com/matrix-org/synapse/issues/9717), [\#9735](https://github.com/matrix-org/synapse/issues/9735)) -- Update experimental support for Spaces: include `m.room.create` in the room state sent with room-invites. ([\#9710](https://github.com/matrix-org/synapse/issues/9710)) -- Synapse now requires Python 3.6 or later. It also requires Postgres 9.6 or later or SQLite 3.22 or later. ([\#9766](https://github.com/matrix-org/synapse/issues/9766)) - - -Bugfixes --------- - -- Prevent `synapse_forward_extremities` and `synapse_excess_extremity_events` Prometheus metrics from initially reporting zero-values after startup. ([\#8926](https://github.com/matrix-org/synapse/issues/8926)) -- Fix recently added ratelimits to correctly honour the application service `rate_limited` flag. ([\#9711](https://github.com/matrix-org/synapse/issues/9711)) -- Fix longstanding bug which caused `duplicate key value violates unique constraint "remote_media_cache_thumbnails_media_origin_media_id_thumbna_key"` errors. ([\#9725](https://github.com/matrix-org/synapse/issues/9725)) -- Fix bug where sharded federation senders could get stuck repeatedly querying the DB in a loop, using lots of CPU. ([\#9770](https://github.com/matrix-org/synapse/issues/9770)) -- Fix duplicate logging of exceptions thrown during federation transaction processing. ([\#9780](https://github.com/matrix-org/synapse/issues/9780)) - - -Updates to the Docker image ---------------------------- - -- Move opencontainers labels to the final Docker image such that users can inspect them. ([\#9765](https://github.com/matrix-org/synapse/issues/9765)) - - -Improved Documentation ----------------------- - -- Make the `allowed_local_3pids` regex example in the sample config stricter. ([\#9719](https://github.com/matrix-org/synapse/issues/9719)) - - -Deprecations and Removals -------------------------- - -- Remove old admin API `GET /_synapse/admin/v1/users/`. ([\#9401](https://github.com/matrix-org/synapse/issues/9401)) -- Make `/_matrix/client/r0/register` expect a type of `m.login.application_service` when an Application Service registers a user, to align with [the relevant spec](https://spec.matrix.org/unstable/application-service-api/#server-admin-style-permissions). ([\#9548](https://github.com/matrix-org/synapse/issues/9548)) - - -Internal Changes ----------------- - -- Replace deprecated `imp` module with successor `importlib`. Contributed by Cristina Muñoz. ([\#9718](https://github.com/matrix-org/synapse/issues/9718)) -- Experiment with GitHub Actions for CI. ([\#9661](https://github.com/matrix-org/synapse/issues/9661)) -- Introduce flake8-bugbear to the test suite and fix some of its lint violations. ([\#9682](https://github.com/matrix-org/synapse/issues/9682)) -- Update `scripts-dev/complement.sh` to use a local checkout of Complement, allow running a subset of tests and have it use Synapse's Complement test blacklist. ([\#9685](https://github.com/matrix-org/synapse/issues/9685)) -- Improve Jaeger tracing for `to_device` messages. ([\#9686](https://github.com/matrix-org/synapse/issues/9686)) -- Add release helper script for automating part of the Synapse release process. ([\#9713](https://github.com/matrix-org/synapse/issues/9713)) -- Add type hints to expiring cache. ([\#9730](https://github.com/matrix-org/synapse/issues/9730)) -- Convert various testcases to `HomeserverTestCase`. ([\#9736](https://github.com/matrix-org/synapse/issues/9736)) -- Start linting mypy with `no_implicit_optional`. ([\#9742](https://github.com/matrix-org/synapse/issues/9742)) -- Add missing type hints to federation handler and server. ([\#9743](https://github.com/matrix-org/synapse/issues/9743)) -- Check that a `ConfigError` is raised, rather than simply `Exception`, when appropriate in homeserver config file generation tests. ([\#9753](https://github.com/matrix-org/synapse/issues/9753)) -- Fix incompatibility with `tox` 2.5. ([\#9769](https://github.com/matrix-org/synapse/issues/9769)) -- Enable Complement tests for [MSC2946](https://github.com/matrix-org/matrix-doc/pull/2946): Spaces Summary API. ([\#9771](https://github.com/matrix-org/synapse/issues/9771)) -- Use mock from the standard library instead of a separate package. ([\#9772](https://github.com/matrix-org/synapse/issues/9772)) -- Update Black configuration to target Python 3.6. ([\#9781](https://github.com/matrix-org/synapse/issues/9781)) -- Add option to skip unit tests when building Debian packages. ([\#9793](https://github.com/matrix-org/synapse/issues/9793)) - - -Synapse 1.31.0 (2021-04-06) -=========================== - -**Note:** As announced in v1.25.0, and in line with the deprecation policy for platform dependencies, this is the last release to support Python 3.5 and PostgreSQL 9.5. Future versions of Synapse will require Python 3.6+ and PostgreSQL 9.6+, as per our [deprecation policy](docs/deprecation_policy.md). - -This is also the last release that the Synapse team will be publishing packages for Debian Stretch and Ubuntu Xenial. - - -Improved Documentation ----------------------- - -- Add a document describing the deprecation policy for platform dependencies. ([\#9723](https://github.com/matrix-org/synapse/issues/9723)) - - -Internal Changes ----------------- - -- Revert using `dmypy run` in lint script. ([\#9720](https://github.com/matrix-org/synapse/issues/9720)) -- Pin flake8-bugbear's version. ([\#9734](https://github.com/matrix-org/synapse/issues/9734)) - - -Synapse 1.31.0rc1 (2021-03-30) -============================== - -Features --------- - -- Add support to OpenID Connect login for requiring attributes on the `userinfo` response. Contributed by Hubbe King. ([\#9609](https://github.com/matrix-org/synapse/issues/9609)) -- Add initial experimental support for a "space summary" API. ([\#9643](https://github.com/matrix-org/synapse/issues/9643), [\#9652](https://github.com/matrix-org/synapse/issues/9652), [\#9653](https://github.com/matrix-org/synapse/issues/9653)) -- Add support for the busy presence state as described in [MSC3026](https://github.com/matrix-org/matrix-doc/pull/3026). ([\#9644](https://github.com/matrix-org/synapse/issues/9644)) -- Add support for credentials for proxy authentication in the `HTTPS_PROXY` environment variable. ([\#9657](https://github.com/matrix-org/synapse/issues/9657)) - - -Bugfixes --------- - -- Fix a longstanding bug that could cause issues when editing a reply to a message. ([\#9585](https://github.com/matrix-org/synapse/issues/9585)) -- Fix the `/capabilities` endpoint to return `m.change_password` as disabled if the local password database is not used for authentication. Contributed by @dklimpel. ([\#9588](https://github.com/matrix-org/synapse/issues/9588)) -- Check if local passwords are enabled before setting them for the user. ([\#9636](https://github.com/matrix-org/synapse/issues/9636)) -- Fix a bug where federation sending can stall due to `concurrent access` database exceptions when it falls behind. ([\#9639](https://github.com/matrix-org/synapse/issues/9639)) -- Fix a bug introduced in Synapse 1.30.1 which meant the suggested `pip` incantation to install an updated `cryptography` was incorrect. ([\#9699](https://github.com/matrix-org/synapse/issues/9699)) - - -Updates to the Docker image ---------------------------- - -- Speed up Docker builds and make it nicer to test against Complement while developing (install all dependencies before copying the project). ([\#9610](https://github.com/matrix-org/synapse/issues/9610)) -- Include [opencontainers labels](https://github.com/opencontainers/image-spec/blob/master/annotations.md#pre-defined-annotation-keys) in the Docker image. ([\#9612](https://github.com/matrix-org/synapse/issues/9612)) - - -Improved Documentation ----------------------- - -- Clarify that `register_new_matrix_user` is present also when installed via non-pip package. ([\#9074](https://github.com/matrix-org/synapse/issues/9074)) -- Update source install documentation to mention platform prerequisites before the source install steps. ([\#9667](https://github.com/matrix-org/synapse/issues/9667)) -- Improve worker documentation for fallback/web auth endpoints. ([\#9679](https://github.com/matrix-org/synapse/issues/9679)) -- Update the sample configuration for OIDC authentication. ([\#9695](https://github.com/matrix-org/synapse/issues/9695)) - - -Internal Changes ----------------- - -- Preparatory steps for removing redundant `outlier` data from `event_json.internal_metadata` column. ([\#9411](https://github.com/matrix-org/synapse/issues/9411)) -- Add type hints to the caching module. ([\#9442](https://github.com/matrix-org/synapse/issues/9442)) -- Introduce flake8-bugbear to the test suite and fix some of its lint violations. ([\#9499](https://github.com/matrix-org/synapse/issues/9499), [\#9659](https://github.com/matrix-org/synapse/issues/9659)) -- Add additional type hints to the Homeserver object. ([\#9631](https://github.com/matrix-org/synapse/issues/9631), [\#9638](https://github.com/matrix-org/synapse/issues/9638), [\#9675](https://github.com/matrix-org/synapse/issues/9675), [\#9681](https://github.com/matrix-org/synapse/issues/9681)) -- Only save remote cross-signing and device keys if they're different from the current ones. ([\#9634](https://github.com/matrix-org/synapse/issues/9634)) -- Rename storage function to fix spelling and not conflict with another function's name. ([\#9637](https://github.com/matrix-org/synapse/issues/9637)) -- Improve performance of federation catch up by sending the latest events in the room to the remote, rather than just the last event sent by the local server. ([\#9640](https://github.com/matrix-org/synapse/issues/9640), [\#9664](https://github.com/matrix-org/synapse/issues/9664)) -- In the `federation_client` commandline client, stop automatically adding the URL prefix, so that servlets on other prefixes can be tested. ([\#9645](https://github.com/matrix-org/synapse/issues/9645)) -- In the `federation_client` commandline client, handle inline `signing_key`s in `homeserver.yaml`. ([\#9647](https://github.com/matrix-org/synapse/issues/9647)) -- Fixed some antipattern issues to improve code quality. ([\#9649](https://github.com/matrix-org/synapse/issues/9649)) -- Add a storage method for pulling all current user presence state from the database. ([\#9650](https://github.com/matrix-org/synapse/issues/9650)) -- Import `HomeServer` from the proper module. ([\#9665](https://github.com/matrix-org/synapse/issues/9665)) -- Increase default join ratelimiting burst rate. ([\#9674](https://github.com/matrix-org/synapse/issues/9674)) -- Add type hints to third party event rules and visibility modules. ([\#9676](https://github.com/matrix-org/synapse/issues/9676)) -- Bump mypy-zope to 0.2.13 to fix "Cannot determine consistent method resolution order (MRO)" errors when running mypy a second time. ([\#9678](https://github.com/matrix-org/synapse/issues/9678)) -- Use interpreter from `$PATH` via `/usr/bin/env` instead of absolute paths in various scripts. ([\#9689](https://github.com/matrix-org/synapse/issues/9689)) -- Make it possible to use `dmypy`. ([\#9692](https://github.com/matrix-org/synapse/issues/9692)) -- Suppress "CryptographyDeprecationWarning: int_from_bytes is deprecated". ([\#9698](https://github.com/matrix-org/synapse/issues/9698)) -- Use `dmypy run` in lint script for improved performance in type-checking while developing. ([\#9701](https://github.com/matrix-org/synapse/issues/9701)) -- Fix undetected mypy error when using Python 3.6. ([\#9703](https://github.com/matrix-org/synapse/issues/9703)) -- Fix type-checking CI on develop. ([\#9709](https://github.com/matrix-org/synapse/issues/9709)) - - -Synapse 1.30.1 (2021-03-26) -=========================== - -This release is identical to Synapse 1.30.0, with the exception of explicitly -setting a minimum version of Python's Cryptography library to ensure that users -of Synapse are protected from the recent [OpenSSL security advisories](https://mta.openssl.org/pipermail/openssl-announce/2021-March/000198.html), -especially CVE-2021-3449. - -Note that Cryptography defaults to bundling its own statically linked copy of -OpenSSL, which means that you may not be protected by your operating system's -security updates. - -It's also worth noting that Cryptography no longer supports Python 3.5, so -admins deploying to older environments may not be protected against this or -future vulnerabilities. Synapse will be dropping support for Python 3.5 at the -end of March. - - -Updates to the Docker image ---------------------------- - -- Ensure that the docker container has up to date versions of openssl. ([\#9697](https://github.com/matrix-org/synapse/issues/9697)) - - -Internal Changes ----------------- - -- Enforce that `cryptography` dependency is up to date to ensure it has the most recent openssl patches. ([\#9697](https://github.com/matrix-org/synapse/issues/9697)) - - -Synapse 1.30.0 (2021-03-22) -=========================== - -Note that this release deprecates the ability for appservices to -call `POST /_matrix/client/r0/register` without the body parameter `type`. Appservice -developers should use a `type` value of `m.login.application_service` as -per [the spec](https://matrix.org/docs/spec/application_service/r0.1.2#server-admin-style-permissions). -In future releases, calling this endpoint with an access token - but without a `m.login.application_service` -type - will fail. - - -No significant changes. - - -Synapse 1.30.0rc1 (2021-03-16) -============================== - -Features --------- - -- Add prometheus metrics for number of users successfully registering and logging in. ([\#9510](https://github.com/matrix-org/synapse/issues/9510), [\#9511](https://github.com/matrix-org/synapse/issues/9511), [\#9573](https://github.com/matrix-org/synapse/issues/9573)) -- Add `synapse_federation_last_sent_pdu_time` and `synapse_federation_last_received_pdu_time` prometheus metrics, which monitor federation delays by reporting the timestamps of messages sent and received to a set of remote servers. ([\#9540](https://github.com/matrix-org/synapse/issues/9540)) -- Add support for generating JSON Web Tokens dynamically for use as OIDC client secrets. ([\#9549](https://github.com/matrix-org/synapse/issues/9549)) -- Optimise handling of incomplete room history for incoming federation. ([\#9601](https://github.com/matrix-org/synapse/issues/9601)) -- Finalise support for allowing clients to pick an SSO Identity Provider ([MSC2858](https://github.com/matrix-org/matrix-doc/pull/2858)). ([\#9617](https://github.com/matrix-org/synapse/issues/9617)) -- Tell spam checker modules about the SSO IdP a user registered through if one was used. ([\#9626](https://github.com/matrix-org/synapse/issues/9626)) - - -Bugfixes --------- - -- Fix long-standing bug when generating thumbnails for some images with transparency: `TypeError: cannot unpack non-iterable int object`. ([\#9473](https://github.com/matrix-org/synapse/issues/9473)) -- Purge chain cover indexes for events that were purged prior to Synapse v1.29.0. ([\#9542](https://github.com/matrix-org/synapse/issues/9542), [\#9583](https://github.com/matrix-org/synapse/issues/9583)) -- Fix bug where federation requests were not correctly retried on 5xx responses. ([\#9567](https://github.com/matrix-org/synapse/issues/9567)) -- Fix re-activating an account via the admin API when local passwords are disabled. ([\#9587](https://github.com/matrix-org/synapse/issues/9587)) -- Fix a bug introduced in Synapse 1.20 which caused incoming federation transactions to stack up, causing slow recovery from outages. ([\#9597](https://github.com/matrix-org/synapse/issues/9597)) -- Fix a bug introduced in v1.28.0 where the OpenID Connect callback endpoint could error with a `MacaroonInitException`. ([\#9620](https://github.com/matrix-org/synapse/issues/9620)) -- Fix Internal Server Error on `GET /_synapse/client/saml2/authn_response` request. ([\#9623](https://github.com/matrix-org/synapse/issues/9623)) - - -Updates to the Docker image ---------------------------- - -- Make use of an improved malloc implementation (`jemalloc`) in the docker image. ([\#8553](https://github.com/matrix-org/synapse/issues/8553)) - - -Improved Documentation ----------------------- - -- Add relayd entry to reverse proxy example configurations. ([\#9508](https://github.com/matrix-org/synapse/issues/9508)) -- Improve the SAML2 upgrade notes for 1.27.0. ([\#9550](https://github.com/matrix-org/synapse/issues/9550)) -- Link to the "List user's media" admin API from the media admin API docs. ([\#9571](https://github.com/matrix-org/synapse/issues/9571)) -- Clarify the spam checker modules documentation example to mention that `parse_config` is a required method. ([\#9580](https://github.com/matrix-org/synapse/issues/9580)) -- Clarify the sample configuration for `stats` settings. ([\#9604](https://github.com/matrix-org/synapse/issues/9604)) - - -Deprecations and Removals -------------------------- - -- The `synapse_federation_last_sent_pdu_age` and `synapse_federation_last_received_pdu_age` prometheus metrics have been removed. They are replaced by `synapse_federation_last_sent_pdu_time` and `synapse_federation_last_received_pdu_time`. ([\#9540](https://github.com/matrix-org/synapse/issues/9540)) -- Registering an Application Service user without using the `m.login.application_service` login type will be unsupported in an upcoming Synapse release. ([\#9559](https://github.com/matrix-org/synapse/issues/9559)) - - -Internal Changes ----------------- - -- Add tests to ResponseCache. ([\#9458](https://github.com/matrix-org/synapse/issues/9458)) -- Add type hints to purge room and server notice admin API. ([\#9520](https://github.com/matrix-org/synapse/issues/9520)) -- Add extra logging to ObservableDeferred when callbacks throw exceptions. ([\#9523](https://github.com/matrix-org/synapse/issues/9523)) -- Fix incorrect type hints. ([\#9528](https://github.com/matrix-org/synapse/issues/9528), [\#9543](https://github.com/matrix-org/synapse/issues/9543), [\#9591](https://github.com/matrix-org/synapse/issues/9591), [\#9608](https://github.com/matrix-org/synapse/issues/9608), [\#9618](https://github.com/matrix-org/synapse/issues/9618)) -- Add an additional test for purging a room. ([\#9541](https://github.com/matrix-org/synapse/issues/9541)) -- Add a `.git-blame-ignore-revs` file with the hashes of auto-formatting. ([\#9560](https://github.com/matrix-org/synapse/issues/9560)) -- Increase the threshold before which outbound federation to a server goes into "catch up" mode, which is expensive for the remote server to handle. ([\#9561](https://github.com/matrix-org/synapse/issues/9561)) -- Fix spurious errors reported by the `config-lint.sh` script. ([\#9562](https://github.com/matrix-org/synapse/issues/9562)) -- Fix type hints and tests for BlacklistingAgentWrapper and BlacklistingReactorWrapper. ([\#9563](https://github.com/matrix-org/synapse/issues/9563)) -- Do not have mypy ignore type hints from unpaddedbase64. ([\#9568](https://github.com/matrix-org/synapse/issues/9568)) -- Improve efficiency of calculating the auth chain in large rooms. ([\#9576](https://github.com/matrix-org/synapse/issues/9576)) -- Convert `synapse.types.Requester` to an `attrs` class. ([\#9586](https://github.com/matrix-org/synapse/issues/9586)) -- Add logging for redis connection setup. ([\#9590](https://github.com/matrix-org/synapse/issues/9590)) -- Improve logging when processing incoming transactions. ([\#9596](https://github.com/matrix-org/synapse/issues/9596)) -- Remove unused `stats.retention` setting, and emit a warning if stats are disabled. ([\#9604](https://github.com/matrix-org/synapse/issues/9604)) -- Prevent attempting to bundle aggregations for state events in /context APIs. ([\#9619](https://github.com/matrix-org/synapse/issues/9619)) - - -Synapse 1.29.0 (2021-03-08) -=========================== - -Note that synapse now expects an `X-Forwarded-Proto` header when used with a reverse proxy. Please see the [upgrade notes](docs/upgrade.md#upgrading-to-v1290) for more details on this change. - - -No significant changes. - - -Synapse 1.29.0rc1 (2021-03-04) -============================== - -Features --------- - -- Add rate limiters to cross-user key sharing requests. ([\#8957](https://github.com/matrix-org/synapse/issues/8957)) -- Add `order_by` to the admin API `GET /_synapse/admin/v1/users//media`. Contributed by @dklimpel. ([\#8978](https://github.com/matrix-org/synapse/issues/8978)) -- Add some configuration settings to make users' profile data more private. ([\#9203](https://github.com/matrix-org/synapse/issues/9203)) -- The `no_proxy` and `NO_PROXY` environment variables are now respected in proxied HTTP clients with the lowercase form taking precedence if both are present. Additionally, the lowercase `https_proxy` environment variable is now respected in proxied HTTP clients on top of existing support for the uppercase `HTTPS_PROXY` form and takes precedence if both are present. Contributed by Timothy Leung. ([\#9372](https://github.com/matrix-org/synapse/issues/9372)) -- Add a configuration option, `user_directory.prefer_local_users`, which when enabled will make it more likely for users on the same server as you to appear above other users. ([\#9383](https://github.com/matrix-org/synapse/issues/9383), [\#9385](https://github.com/matrix-org/synapse/issues/9385)) -- Add support for regenerating thumbnails if they have been deleted but the original image is still stored. ([\#9438](https://github.com/matrix-org/synapse/issues/9438)) -- Add support for `X-Forwarded-Proto` header when using a reverse proxy. ([\#9472](https://github.com/matrix-org/synapse/issues/9472), [\#9501](https://github.com/matrix-org/synapse/issues/9501), [\#9512](https://github.com/matrix-org/synapse/issues/9512), [\#9539](https://github.com/matrix-org/synapse/issues/9539)) - - -Bugfixes --------- - -- Fix a bug where users' pushers were not all deleted when they deactivated their account. ([\#9285](https://github.com/matrix-org/synapse/issues/9285), [\#9516](https://github.com/matrix-org/synapse/issues/9516)) -- Fix a bug where a lot of unnecessary presence updates were sent when joining a room. ([\#9402](https://github.com/matrix-org/synapse/issues/9402)) -- Fix a bug that caused multiple calls to the experimental `shared_rooms` endpoint to return stale results. ([\#9416](https://github.com/matrix-org/synapse/issues/9416)) -- Fix a bug in single sign-on which could cause a "No session cookie found" error. ([\#9436](https://github.com/matrix-org/synapse/issues/9436)) -- Fix bug introduced in v1.27.0 where allowing a user to choose their own username when logging in via single sign-on did not work unless an `idp_icon` was defined. ([\#9440](https://github.com/matrix-org/synapse/issues/9440)) -- Fix a bug introduced in v1.26.0 where some sequences were not properly configured when running `synapse_port_db`. ([\#9449](https://github.com/matrix-org/synapse/issues/9449)) -- Fix deleting pushers when using sharded pushers. ([\#9465](https://github.com/matrix-org/synapse/issues/9465), [\#9466](https://github.com/matrix-org/synapse/issues/9466), [\#9479](https://github.com/matrix-org/synapse/issues/9479), [\#9536](https://github.com/matrix-org/synapse/issues/9536)) -- Fix missing startup checks for the consistency of certain PostgreSQL sequences. ([\#9470](https://github.com/matrix-org/synapse/issues/9470)) -- Fix a long-standing bug where the media repository could leak file descriptors while previewing media. ([\#9497](https://github.com/matrix-org/synapse/issues/9497)) -- Properly purge the event chain cover index when purging history. ([\#9498](https://github.com/matrix-org/synapse/issues/9498)) -- Fix missing chain cover index due to a schema delta not being applied correctly. Only affected servers that ran development versions. ([\#9503](https://github.com/matrix-org/synapse/issues/9503)) -- Fix a bug introduced in v1.25.0 where `/_synapse/admin/join/` would fail when given a room alias. ([\#9506](https://github.com/matrix-org/synapse/issues/9506)) -- Prevent presence background jobs from running when presence is disabled. ([\#9530](https://github.com/matrix-org/synapse/issues/9530)) -- Fix rare edge case that caused a background update to fail if the server had rejected an event that had duplicate auth events. ([\#9537](https://github.com/matrix-org/synapse/issues/9537)) - - -Improved Documentation ----------------------- - -- Update the example systemd config to propagate reloads to individual units. ([\#9463](https://github.com/matrix-org/synapse/issues/9463)) - - -Internal Changes ----------------- - -- Add documentation and type hints to `parse_duration`. ([\#9432](https://github.com/matrix-org/synapse/issues/9432)) -- Remove vestiges of `uploads_path` configuration setting. ([\#9462](https://github.com/matrix-org/synapse/issues/9462)) -- Add a comment about systemd-python. ([\#9464](https://github.com/matrix-org/synapse/issues/9464)) -- Test that we require validated email for email pushers. ([\#9496](https://github.com/matrix-org/synapse/issues/9496)) -- Allow python to generate bytecode for synapse. ([\#9502](https://github.com/matrix-org/synapse/issues/9502)) -- Fix incorrect type hints. ([\#9515](https://github.com/matrix-org/synapse/issues/9515), [\#9518](https://github.com/matrix-org/synapse/issues/9518)) -- Add type hints to device and event report admin API. ([\#9519](https://github.com/matrix-org/synapse/issues/9519)) -- Add type hints to user admin API. ([\#9521](https://github.com/matrix-org/synapse/issues/9521)) -- Bump the versions of mypy and mypy-zope used for static type checking. ([\#9529](https://github.com/matrix-org/synapse/issues/9529)) - - -Synapse 1.28.0 (2021-02-25) -=========================== - -Note that this release drops support for ARMv7 in the official Docker images, due to repeated problems building for ARMv7 (and the associated maintenance burden this entails). - -This release also fixes the documentation included in v1.27.0 around the callback URI for SAML2 identity providers. If your server is configured to use single sign-on via a SAML2 IdP, you may need to make configuration changes. Please review the [upgrade notes](docs/upgrade.md) for more details on these changes. - - -Internal Changes ----------------- - -- Revert change in v1.28.0rc1 to remove the deprecated SAML endpoint. ([\#9474](https://github.com/matrix-org/synapse/issues/9474)) - - -Synapse 1.28.0rc1 (2021-02-19) -============================== - -Removal warning ---------------- - -The v1 list accounts API is deprecated and will be removed in a future release. -This API was undocumented and misleading. It can be replaced by the -[v2 list accounts API](https://github.com/matrix-org/synapse/blob/release-v1.28.0/docs/admin_api/user_admin_api.rst#list-accounts), -which has been available since Synapse 1.7.0 (2019-12-13). - -Please check if you're using any scripts which use the admin API and replace -`GET /_synapse/admin/v1/users/` with `GET /_synapse/admin/v2/users`. - - -Features --------- - -- New admin API to get the context of an event: `/_synapse/admin/rooms/{roomId}/context/{eventId}`. ([\#9150](https://github.com/matrix-org/synapse/issues/9150)) -- Further improvements to the user experience of registration via single sign-on. ([\#9300](https://github.com/matrix-org/synapse/issues/9300), [\#9301](https://github.com/matrix-org/synapse/issues/9301)) -- Add hook to spam checker modules that allow checking file uploads and remote downloads. ([\#9311](https://github.com/matrix-org/synapse/issues/9311)) -- Add support for receiving OpenID Connect authentication responses via form `POST`s rather than `GET`s. ([\#9376](https://github.com/matrix-org/synapse/issues/9376)) -- Add the shadow-banning status to the admin API for user info. ([\#9400](https://github.com/matrix-org/synapse/issues/9400)) - - -Bugfixes --------- - -- Fix long-standing bug where sending email notifications would fail for rooms that the server had since left. ([\#9257](https://github.com/matrix-org/synapse/issues/9257)) -- Fix bug introduced in Synapse 1.27.0rc1 which meant the "session expired" error page during SSO registration was badly formatted. ([\#9296](https://github.com/matrix-org/synapse/issues/9296)) -- Assert a maximum length for some parameters for spec compliance. ([\#9321](https://github.com/matrix-org/synapse/issues/9321), [\#9393](https://github.com/matrix-org/synapse/issues/9393)) -- Fix additional errors when previewing URLs: "AttributeError 'NoneType' object has no attribute 'xpath'" and "ValueError: Unicode strings with encoding declaration are not supported. Please use bytes input or XML fragments without declaration.". ([\#9333](https://github.com/matrix-org/synapse/issues/9333)) -- Fix a bug causing Synapse to impose the wrong type constraints on fields when processing responses from appservices to `/_matrix/app/v1/thirdparty/user/{protocol}`. ([\#9361](https://github.com/matrix-org/synapse/issues/9361)) -- Fix bug where Synapse would occasionally stop reconnecting to Redis after the connection was lost. ([\#9391](https://github.com/matrix-org/synapse/issues/9391)) -- Fix a long-standing bug when upgrading a room: "TypeError: '>' not supported between instances of 'NoneType' and 'int'". ([\#9395](https://github.com/matrix-org/synapse/issues/9395)) -- Reduce the amount of memory used when generating the URL preview of a file that is larger than the `max_spider_size`. ([\#9421](https://github.com/matrix-org/synapse/issues/9421)) -- Fix a long-standing bug in the deduplication of old presence, resulting in no deduplication. ([\#9425](https://github.com/matrix-org/synapse/issues/9425)) -- The `ui_auth.session_timeout` config option can now be specified in terms of number of seconds/minutes/etc/. Contributed by Rishabh Arya. ([\#9426](https://github.com/matrix-org/synapse/issues/9426)) -- Fix a bug introduced in v1.27.0: "TypeError: int() argument must be a string, a bytes-like object or a number, not 'NoneType." related to the user directory. ([\#9428](https://github.com/matrix-org/synapse/issues/9428)) - - -Updates to the Docker image ---------------------------- - -- Drop support for ARMv7 in Docker images. ([\#9433](https://github.com/matrix-org/synapse/issues/9433)) - - -Improved Documentation ----------------------- - -- Reorganize CHANGELOG.md. ([\#9281](https://github.com/matrix-org/synapse/issues/9281)) -- Add note to `auto_join_rooms` config option explaining existing rooms must be publicly joinable. ([\#9291](https://github.com/matrix-org/synapse/issues/9291)) -- Correct name of Synapse's service file in TURN howto. ([\#9308](https://github.com/matrix-org/synapse/issues/9308)) -- Fix the braces in the `oidc_providers` section of the sample config. ([\#9317](https://github.com/matrix-org/synapse/issues/9317)) -- Update installation instructions on Fedora. ([\#9322](https://github.com/matrix-org/synapse/issues/9322)) -- Add HTTP/2 support to the nginx example configuration. Contributed by David Vo. ([\#9390](https://github.com/matrix-org/synapse/issues/9390)) -- Update docs for using Gitea as OpenID provider. ([\#9404](https://github.com/matrix-org/synapse/issues/9404)) -- Document that pusher instances are shardable. ([\#9407](https://github.com/matrix-org/synapse/issues/9407)) -- Fix erroneous documentation from v1.27.0 about updating the SAML2 callback URL. ([\#9434](https://github.com/matrix-org/synapse/issues/9434)) - - -Deprecations and Removals -------------------------- - -- Deprecate old admin API `GET /_synapse/admin/v1/users/`. ([\#9429](https://github.com/matrix-org/synapse/issues/9429)) - - -Internal Changes ----------------- - -- Fix 'object name reserved for internal use' errors with recent versions of SQLite. ([\#9003](https://github.com/matrix-org/synapse/issues/9003)) -- Add experimental support for running Synapse with PyPy. ([\#9123](https://github.com/matrix-org/synapse/issues/9123)) -- Deny access to additional IP addresses by default. ([\#9240](https://github.com/matrix-org/synapse/issues/9240)) -- Update the `Cursor` type hints to better match PEP 249. ([\#9299](https://github.com/matrix-org/synapse/issues/9299)) -- Add debug logging for SRV lookups. Contributed by @Bubu. ([\#9305](https://github.com/matrix-org/synapse/issues/9305)) -- Improve logging for OIDC login flow. ([\#9307](https://github.com/matrix-org/synapse/issues/9307)) -- Share the code for handling required attributes between the CAS and SAML handlers. ([\#9326](https://github.com/matrix-org/synapse/issues/9326)) -- Clean up the code to load the metadata for OpenID Connect identity providers. ([\#9362](https://github.com/matrix-org/synapse/issues/9362)) -- Convert tests to use `HomeserverTestCase`. ([\#9377](https://github.com/matrix-org/synapse/issues/9377), [\#9396](https://github.com/matrix-org/synapse/issues/9396)) -- Update the version of black used to 20.8b1. ([\#9381](https://github.com/matrix-org/synapse/issues/9381)) -- Allow OIDC config to override discovered values. ([\#9384](https://github.com/matrix-org/synapse/issues/9384)) -- Remove some dead code from the acceptance of room invites path. ([\#9394](https://github.com/matrix-org/synapse/issues/9394)) -- Clean up an unused method in the presence handler code. ([\#9408](https://github.com/matrix-org/synapse/issues/9408)) - - -Synapse 1.27.0 (2021-02-16) -=========================== - -Note that this release includes a change in Synapse to use Redis as a cache ─ as well as a pub/sub mechanism ─ if Redis support is enabled for workers. No action is needed by server administrators, and we do not expect resource usage of the Redis instance to change dramatically. - -This release also changes the callback URI for OpenID Connect (OIDC) and SAML2 identity providers. If your server is configured to use single sign-on via an OIDC/OAuth2 or SAML2 IdP, you may need to make configuration changes. Please review the [upgrade notes](docs/upgrade.md) for more details on these changes. - -This release also changes escaping of variables in the HTML templates for SSO or email notifications. If you have customised these templates, please review the [upgrade notes](docs/upgrade.md) for more details on these changes. - - -Bugfixes --------- - -- Fix building Docker images for armv7. ([\#9405](https://github.com/matrix-org/synapse/issues/9405)) - - -Synapse 1.27.0rc2 (2021-02-11) -============================== - -Features --------- - -- Further improvements to the user experience of registration via single sign-on. ([\#9297](https://github.com/matrix-org/synapse/issues/9297)) - - -Bugfixes --------- - -- Fix ratelimiting introduced in v1.27.0rc1 for invites to respect the `ratelimit` flag on application services. ([\#9302](https://github.com/matrix-org/synapse/issues/9302)) -- Do not automatically calculate `public_baseurl` since it can be wrong in some situations. Reverts behaviour introduced in v1.26.0. ([\#9313](https://github.com/matrix-org/synapse/issues/9313)) - - -Improved Documentation ----------------------- - -- Clarify the sample configuration for changes made to the template loading code. ([\#9310](https://github.com/matrix-org/synapse/issues/9310)) - - -Synapse 1.27.0rc1 (2021-02-02) -============================== - -Features --------- - -- Add an admin API for getting and deleting forward extremities for a room. ([\#9062](https://github.com/matrix-org/synapse/issues/9062)) -- Add an admin API for retrieving the current room state of a room. ([\#9168](https://github.com/matrix-org/synapse/issues/9168)) -- Add experimental support for allowing clients to pick an SSO Identity Provider ([MSC2858](https://github.com/matrix-org/matrix-doc/pull/2858)). ([\#9183](https://github.com/matrix-org/synapse/issues/9183), [\#9242](https://github.com/matrix-org/synapse/issues/9242)) -- Add an admin API endpoint for shadow-banning users. ([\#9209](https://github.com/matrix-org/synapse/issues/9209)) -- Add ratelimits to the 3PID `/requestToken` APIs. ([\#9238](https://github.com/matrix-org/synapse/issues/9238)) -- Add support to the OpenID Connect integration for adding the user's email address. ([\#9245](https://github.com/matrix-org/synapse/issues/9245)) -- Add ratelimits to invites in rooms and to specific users. ([\#9258](https://github.com/matrix-org/synapse/issues/9258)) -- Improve the user experience of setting up an account via single-sign on. ([\#9262](https://github.com/matrix-org/synapse/issues/9262), [\#9272](https://github.com/matrix-org/synapse/issues/9272), [\#9275](https://github.com/matrix-org/synapse/issues/9275), [\#9276](https://github.com/matrix-org/synapse/issues/9276), [\#9277](https://github.com/matrix-org/synapse/issues/9277), [\#9286](https://github.com/matrix-org/synapse/issues/9286), [\#9287](https://github.com/matrix-org/synapse/issues/9287)) -- Add phone home stats for encrypted messages. ([\#9283](https://github.com/matrix-org/synapse/issues/9283)) -- Update the redirect URI for OIDC authentication. ([\#9288](https://github.com/matrix-org/synapse/issues/9288)) - - -Bugfixes --------- - -- Fix spurious errors in logs when deleting a non-existant pusher. ([\#9121](https://github.com/matrix-org/synapse/issues/9121)) -- Fix a long-standing bug where Synapse would return a 500 error when a thumbnail did not exist (and auto-generation of thumbnails was not enabled). ([\#9163](https://github.com/matrix-org/synapse/issues/9163)) -- Fix a long-standing bug where an internal server error was raised when attempting to preview an HTML document in an unknown character encoding. ([\#9164](https://github.com/matrix-org/synapse/issues/9164)) -- Fix a long-standing bug where invalid data could cause errors when calculating the presentable room name for push. ([\#9165](https://github.com/matrix-org/synapse/issues/9165)) -- Fix bug where we sometimes didn't detect that Redis connections had died, causing workers to not see new data. ([\#9218](https://github.com/matrix-org/synapse/issues/9218)) -- Fix a bug where `None` was passed to Synapse modules instead of an empty dictionary if an empty module `config` block was provided in the homeserver config. ([\#9229](https://github.com/matrix-org/synapse/issues/9229)) -- Fix a bug in the `make_room_admin` admin API where it failed if the admin with the greatest power level was not in the room. Contributed by Pankaj Yadav. ([\#9235](https://github.com/matrix-org/synapse/issues/9235)) -- Prevent password hashes from getting dropped if a client failed threepid validation during a User Interactive Auth stage. Removes a workaround for an ancient bug in Riot Web /joined_rooms` to work for both local and remote users. ([\#8948](https://github.com/matrix-org/synapse/issues/8948)) -- Add experimental support for handling to-device messages on worker processes. ([\#9042](https://github.com/matrix-org/synapse/issues/9042), [\#9043](https://github.com/matrix-org/synapse/issues/9043), [\#9044](https://github.com/matrix-org/synapse/issues/9044), [\#9130](https://github.com/matrix-org/synapse/issues/9130)) -- Add experimental support for handling `/keys/claim` and `/room_keys` APIs on worker processes. ([\#9068](https://github.com/matrix-org/synapse/issues/9068)) -- Add experimental support for handling `/devices` API on worker processes. ([\#9092](https://github.com/matrix-org/synapse/issues/9092)) -- Add experimental support for moving off receipts and account data persistence off master. ([\#9104](https://github.com/matrix-org/synapse/issues/9104), [\#9166](https://github.com/matrix-org/synapse/issues/9166)) - - -Bugfixes --------- - -- Fix a long-standing issue where an internal server error would occur when requesting a profile over federation that did not include a display name / avatar URL. ([\#9023](https://github.com/matrix-org/synapse/issues/9023)) -- Fix a long-standing bug where some caches could grow larger than configured. ([\#9028](https://github.com/matrix-org/synapse/issues/9028)) -- Fix error handling during insertion of client IPs into the database. ([\#9051](https://github.com/matrix-org/synapse/issues/9051)) -- Fix bug where we didn't correctly record CPU time spent in `on_new_event` block. ([\#9053](https://github.com/matrix-org/synapse/issues/9053)) -- Fix a minor bug which could cause confusing error messages from invalid configurations. ([\#9054](https://github.com/matrix-org/synapse/issues/9054)) -- Fix incorrect exit code when there is an error at startup. ([\#9059](https://github.com/matrix-org/synapse/issues/9059)) -- Fix `JSONDecodeError` spamming the logs when sending transactions to remote servers. ([\#9070](https://github.com/matrix-org/synapse/issues/9070)) -- Fix "Failed to send request" errors when a client provides an invalid room alias. ([\#9071](https://github.com/matrix-org/synapse/issues/9071)) -- Fix bugs in federation catchup logic that caused outbound federation to be delayed for large servers after start up. Introduced in v1.8.0 and v1.21.0. ([\#9114](https://github.com/matrix-org/synapse/issues/9114), [\#9116](https://github.com/matrix-org/synapse/issues/9116)) -- Fix corruption of `pushers` data when a postgres bouncer is used. ([\#9117](https://github.com/matrix-org/synapse/issues/9117)) -- Fix minor bugs in handling the `clientRedirectUrl` parameter for SSO login. ([\#9128](https://github.com/matrix-org/synapse/issues/9128)) -- Fix "Unhandled error in Deferred: BodyExceededMaxSize" errors when .well-known files that are too large. ([\#9108](https://github.com/matrix-org/synapse/issues/9108)) -- Fix "UnboundLocalError: local variable 'length' referenced before assignment" errors when the response body exceeds the expected size. This bug was introduced in v1.25.0. ([\#9145](https://github.com/matrix-org/synapse/issues/9145)) -- Fix a long-standing bug "ValueError: invalid literal for int() with base 10" when `/publicRooms` is requested with an invalid `server` parameter. ([\#9161](https://github.com/matrix-org/synapse/issues/9161)) - - -Improved Documentation ----------------------- - -- Add some extra docs for getting Synapse running on macOS. ([\#8997](https://github.com/matrix-org/synapse/issues/8997)) -- Correct a typo in the `systemd-with-workers` documentation. ([\#9035](https://github.com/matrix-org/synapse/issues/9035)) -- Correct a typo in `INSTALL.md`. ([\#9040](https://github.com/matrix-org/synapse/issues/9040)) -- Add missing `user_mapping_provider` configuration to the Keycloak OIDC example. Contributed by @chris-ruecker. ([\#9057](https://github.com/matrix-org/synapse/issues/9057)) -- Quote `pip install` packages when extras are used to avoid shells interpreting bracket characters. ([\#9151](https://github.com/matrix-org/synapse/issues/9151)) - - -Deprecations and Removals -------------------------- - -- Remove broken and unmaintained `demo/webserver.py` script. ([\#9039](https://github.com/matrix-org/synapse/issues/9039)) - - -Internal Changes ----------------- - -- Improve efficiency of large state resolutions. ([\#8868](https://github.com/matrix-org/synapse/issues/8868), [\#9029](https://github.com/matrix-org/synapse/issues/9029), [\#9115](https://github.com/matrix-org/synapse/issues/9115), [\#9118](https://github.com/matrix-org/synapse/issues/9118), [\#9124](https://github.com/matrix-org/synapse/issues/9124)) -- Various clean-ups to the structured logging and logging context code. ([\#8939](https://github.com/matrix-org/synapse/issues/8939)) -- Ensure rejected events get added to some metadata tables. ([\#9016](https://github.com/matrix-org/synapse/issues/9016)) -- Ignore date-rotated homeserver logs saved to disk. ([\#9018](https://github.com/matrix-org/synapse/issues/9018)) -- Remove an unused column from `access_tokens` table. ([\#9025](https://github.com/matrix-org/synapse/issues/9025)) -- Add a `-noextras` factor to `tox.ini`, to support running the tests with no optional dependencies. ([\#9030](https://github.com/matrix-org/synapse/issues/9030)) -- Fix running unit tests when optional dependencies are not installed. ([\#9031](https://github.com/matrix-org/synapse/issues/9031)) -- Allow bumping schema version when using split out state database. ([\#9033](https://github.com/matrix-org/synapse/issues/9033)) -- Configure the linters to run on a consistent set of files. ([\#9038](https://github.com/matrix-org/synapse/issues/9038)) -- Various cleanups to device inbox store. ([\#9041](https://github.com/matrix-org/synapse/issues/9041)) -- Drop unused database tables. ([\#9055](https://github.com/matrix-org/synapse/issues/9055)) -- Remove unused `SynapseService` class. ([\#9058](https://github.com/matrix-org/synapse/issues/9058)) -- Remove unnecessary declarations in the tests for the admin API. ([\#9063](https://github.com/matrix-org/synapse/issues/9063)) -- Remove `SynapseRequest.get_user_agent`. ([\#9069](https://github.com/matrix-org/synapse/issues/9069)) -- Remove redundant `Homeserver.get_ip_from_request` method. ([\#9080](https://github.com/matrix-org/synapse/issues/9080)) -- Add type hints to media repository. ([\#9093](https://github.com/matrix-org/synapse/issues/9093)) -- Fix the wrong arguments being passed to `BlacklistingAgentWrapper` from `MatrixFederationAgent`. Contributed by Timothy Leung. ([\#9098](https://github.com/matrix-org/synapse/issues/9098)) -- Reduce the scope of caught exceptions in `BlacklistingAgentWrapper`. ([\#9106](https://github.com/matrix-org/synapse/issues/9106)) -- Improve `UsernamePickerTestCase`. ([\#9112](https://github.com/matrix-org/synapse/issues/9112)) -- Remove dependency on `distutils`. ([\#9125](https://github.com/matrix-org/synapse/issues/9125)) -- Enforce that replication HTTP clients are called with keyword arguments only. ([\#9144](https://github.com/matrix-org/synapse/issues/9144)) -- Fix the Python 3.5 / old dependencies build in CI. ([\#9146](https://github.com/matrix-org/synapse/issues/9146)) -- Replace the old `perspectives` option in the Synapse docker config file template with `trusted_key_servers`. ([\#9157](https://github.com/matrix-org/synapse/issues/9157)) - - -Synapse 1.25.0 (2021-01-13) -=========================== - -Ending Support for Python 3.5 and Postgres 9.5 ----------------------------------------------- - -With this release, the Synapse team is announcing a formal deprecation policy for our platform dependencies, like Python and PostgreSQL: - -All future releases of Synapse will follow the upstream end-of-life schedules. - -Which means: - -* This is the last release which guarantees support for Python 3.5. -* We will end support for PostgreSQL 9.5 early next month. -* We will end support for Python 3.6 and PostgreSQL 9.6 near the end of the year. - -Crucially, this means __we will not produce .deb packages for Debian 9 (Stretch) or Ubuntu 16.04 (Xenial)__ beyond the transition period described below. - -The website https://endoflife.date/ has convenient summaries of the support schedules for projects like [Python](https://endoflife.date/python) and [PostgreSQL](https://endoflife.date/postgresql). - -If you are unable to upgrade your environment to a supported version of Python or -Postgres, we encourage you to consider using the -[Synapse Docker images](https://matrix-org.github.io/synapse/latest/setup/installation.html#docker-images-and-ansible-playbooks) -instead. - -### Transition Period - -We will make a good faith attempt to avoid breaking compatibility in all releases through the end of March 2021. However, critical security vulnerabilities in dependencies or other unanticipated circumstances may arise which necessitate breaking compatibility earlier. - -We intend to continue producing .deb packages for Debian 9 (Stretch) and Ubuntu 16.04 (Xenial) through the transition period. - -Removal warning ---------------- - -The old [Purge Room API](https://github.com/matrix-org/synapse/tree/master/docs/admin_api/purge_room.md) -and [Shutdown Room API](https://github.com/matrix-org/synapse/tree/master/docs/admin_api/shutdown_room.md) -are deprecated and will be removed in a future release. They will be replaced by the -[Delete Room API](https://github.com/matrix-org/synapse/tree/master/docs/admin_api/rooms.md#delete-room-api). - -`POST /_synapse/admin/v1/rooms//delete` replaces `POST /_synapse/admin/v1/purge_room` and -`POST /_synapse/admin/v1/shutdown_room/`. - -Bugfixes --------- - -- Fix HTTP proxy support when using a proxy that is on a blacklisted IP. Introduced in v1.25.0rc1. Contributed by @Bubu. ([\#9084](https://github.com/matrix-org/synapse/issues/9084)) - - -Synapse 1.25.0rc1 (2021-01-06) -============================== - -Features --------- - -- Add an admin API that lets server admins get power in rooms in which local users have power. ([\#8756](https://github.com/matrix-org/synapse/issues/8756)) -- Add optional HTTP authentication to replication endpoints. ([\#8853](https://github.com/matrix-org/synapse/issues/8853)) -- Improve the error messages printed as a result of configuration problems for extension modules. ([\#8874](https://github.com/matrix-org/synapse/issues/8874)) -- Add the number of local devices to Room Details Admin API. Contributed by @dklimpel. ([\#8886](https://github.com/matrix-org/synapse/issues/8886)) -- Add `X-Robots-Tag` header to stop web crawlers from indexing media. Contributed by Aaron Raimist. ([\#8887](https://github.com/matrix-org/synapse/issues/8887)) -- Spam-checkers may now define their methods as `async`. ([\#8890](https://github.com/matrix-org/synapse/issues/8890)) -- Add support for allowing users to pick their own user ID during a single-sign-on login. ([\#8897](https://github.com/matrix-org/synapse/issues/8897), [\#8900](https://github.com/matrix-org/synapse/issues/8900), [\#8911](https://github.com/matrix-org/synapse/issues/8911), [\#8938](https://github.com/matrix-org/synapse/issues/8938), [\#8941](https://github.com/matrix-org/synapse/issues/8941), [\#8942](https://github.com/matrix-org/synapse/issues/8942), [\#8951](https://github.com/matrix-org/synapse/issues/8951)) -- Add an `email.invite_client_location` configuration option to send a web client location to the invite endpoint on the identity server which allows customisation of the email template. ([\#8930](https://github.com/matrix-org/synapse/issues/8930)) -- The search term in the list room and list user Admin APIs is now treated as case-insensitive. ([\#8931](https://github.com/matrix-org/synapse/issues/8931)) -- Apply an IP range blacklist to push and key revocation requests. ([\#8821](https://github.com/matrix-org/synapse/issues/8821), [\#8870](https://github.com/matrix-org/synapse/issues/8870), [\#8954](https://github.com/matrix-org/synapse/issues/8954)) -- Add an option to allow re-use of user-interactive authentication sessions for a period of time. ([\#8970](https://github.com/matrix-org/synapse/issues/8970)) -- Allow running the redact endpoint on workers. ([\#8994](https://github.com/matrix-org/synapse/issues/8994)) - - -Bugfixes --------- - -- Fix bug where we might not correctly calculate the current state for rooms with multiple extremities. ([\#8827](https://github.com/matrix-org/synapse/issues/8827)) -- Fix a long-standing bug in the register admin endpoint (`/_synapse/admin/v1/register`) when the `mac` field was not provided. The endpoint now properly returns a 400 error. Contributed by @edwargix. ([\#8837](https://github.com/matrix-org/synapse/issues/8837)) -- Fix a long-standing bug on Synapse instances supporting Single-Sign-On, where users would be prompted to enter their password to confirm certain actions, even though they have not set a password. ([\#8858](https://github.com/matrix-org/synapse/issues/8858)) -- Fix a longstanding bug where a 500 error would be returned if the `Content-Length` header was not provided to the upload media resource. ([\#8862](https://github.com/matrix-org/synapse/issues/8862)) -- Add additional validation to pusher URLs to be compliant with the specification. ([\#8865](https://github.com/matrix-org/synapse/issues/8865)) -- Fix the error code that is returned when a user tries to register on a homeserver on which new-user registration has been disabled. ([\#8867](https://github.com/matrix-org/synapse/issues/8867)) -- Fix a bug where `PUT /_synapse/admin/v2/users/` failed to create a new user when `avatar_url` is specified. Bug introduced in Synapse v1.9.0. ([\#8872](https://github.com/matrix-org/synapse/issues/8872)) -- Fix a 500 error when attempting to preview an empty HTML file. ([\#8883](https://github.com/matrix-org/synapse/issues/8883)) -- Fix occasional deadlock when handling SIGHUP. ([\#8918](https://github.com/matrix-org/synapse/issues/8918)) -- Fix login API to not ratelimit application services that have ratelimiting disabled. ([\#8920](https://github.com/matrix-org/synapse/issues/8920)) -- Fix bug where we ratelimited auto joining of rooms on registration (using `auto_join_rooms` config). ([\#8921](https://github.com/matrix-org/synapse/issues/8921)) -- Fix a bug where deactivated users appeared in the user directory when their profile information was updated. ([\#8933](https://github.com/matrix-org/synapse/issues/8933), [\#8964](https://github.com/matrix-org/synapse/issues/8964)) -- Fix bug introduced in Synapse v1.24.0 which would cause an exception on startup if both `enabled` and `localdb_enabled` were set to `False` in the `password_config` setting of the configuration file. ([\#8937](https://github.com/matrix-org/synapse/issues/8937)) -- Fix a bug where 500 errors would be returned if the `m.room_history_visibility` event had invalid content. ([\#8945](https://github.com/matrix-org/synapse/issues/8945)) -- Fix a bug causing common English words to not be considered for a user directory search. ([\#8959](https://github.com/matrix-org/synapse/issues/8959)) -- Fix bug where application services couldn't register new ghost users if the server had reached its MAU limit. ([\#8962](https://github.com/matrix-org/synapse/issues/8962)) -- Fix a long-standing bug where a `m.image` event without a `url` would cause errors on push. ([\#8965](https://github.com/matrix-org/synapse/issues/8965)) -- Fix a small bug in v2 state resolution algorithm, which could also cause performance issues for rooms with large numbers of power levels. ([\#8971](https://github.com/matrix-org/synapse/issues/8971)) -- Add validation to the `sendToDevice` API to raise a missing parameters error instead of a 500 error. ([\#8975](https://github.com/matrix-org/synapse/issues/8975)) -- Add validation of group IDs to raise a 400 error instead of a 500 eror. ([\#8977](https://github.com/matrix-org/synapse/issues/8977)) - - -Improved Documentation ----------------------- - -- Fix the "Event persist rate" section of the included grafana dashboard by adding missing prometheus rules. ([\#8802](https://github.com/matrix-org/synapse/issues/8802)) -- Combine related media admin API docs. ([\#8839](https://github.com/matrix-org/synapse/issues/8839)) -- Fix an error in the documentation for the SAML username mapping provider. ([\#8873](https://github.com/matrix-org/synapse/issues/8873)) -- Clarify comments around template directories in `sample_config.yaml`. ([\#8891](https://github.com/matrix-org/synapse/issues/8891)) -- Move instructions for database setup, adjusted heading levels and improved syntax highlighting in [INSTALL.md](../INSTALL.md). Contributed by @fossterer. ([\#8987](https://github.com/matrix-org/synapse/issues/8987)) -- Update the example value of `group_creation_prefix` in the sample configuration. ([\#8992](https://github.com/matrix-org/synapse/issues/8992)) -- Link the Synapse developer room to the development section in the docs. ([\#9002](https://github.com/matrix-org/synapse/issues/9002)) - - -Deprecations and Removals -------------------------- - -- Deprecate Shutdown Room and Purge Room Admin APIs. ([\#8829](https://github.com/matrix-org/synapse/issues/8829)) - - -Internal Changes ----------------- - -- Properly store the mapping of external ID to Matrix ID for CAS users. ([\#8856](https://github.com/matrix-org/synapse/issues/8856), [\#8958](https://github.com/matrix-org/synapse/issues/8958)) -- Remove some unnecessary stubbing from unit tests. ([\#8861](https://github.com/matrix-org/synapse/issues/8861)) -- Remove unused `FakeResponse` class from unit tests. ([\#8864](https://github.com/matrix-org/synapse/issues/8864)) -- Pass `room_id` to `get_auth_chain_difference`. ([\#8879](https://github.com/matrix-org/synapse/issues/8879)) -- Add type hints to push module. ([\#8880](https://github.com/matrix-org/synapse/issues/8880), [\#8882](https://github.com/matrix-org/synapse/issues/8882), [\#8901](https://github.com/matrix-org/synapse/issues/8901), [\#8940](https://github.com/matrix-org/synapse/issues/8940), [\#8943](https://github.com/matrix-org/synapse/issues/8943), [\#9020](https://github.com/matrix-org/synapse/issues/9020)) -- Simplify logic for handling user-interactive-auth via single-sign-on servers. ([\#8881](https://github.com/matrix-org/synapse/issues/8881)) -- Skip the SAML tests if the requirements (`pysaml2` and `xmlsec1`) aren't available. ([\#8905](https://github.com/matrix-org/synapse/issues/8905)) -- Fix multiarch docker image builds. ([\#8906](https://github.com/matrix-org/synapse/issues/8906)) -- Don't publish `latest` docker image until all archs are built. ([\#8909](https://github.com/matrix-org/synapse/issues/8909)) -- Various clean-ups to the structured logging and logging context code. ([\#8916](https://github.com/matrix-org/synapse/issues/8916), [\#8935](https://github.com/matrix-org/synapse/issues/8935)) -- Automatically drop stale forward-extremities under some specific conditions. ([\#8929](https://github.com/matrix-org/synapse/issues/8929)) -- Refactor test utilities for injecting HTTP requests. ([\#8946](https://github.com/matrix-org/synapse/issues/8946)) -- Add a maximum size of 50 kilobytes to .well-known lookups. ([\#8950](https://github.com/matrix-org/synapse/issues/8950)) -- Fix bug in `generate_log_config` script which made it write empty files. ([\#8952](https://github.com/matrix-org/synapse/issues/8952)) -- Clean up tox.ini file; disable coverage checking for non-test runs. ([\#8963](https://github.com/matrix-org/synapse/issues/8963)) -- Add type hints to the admin and room list handlers. ([\#8973](https://github.com/matrix-org/synapse/issues/8973)) -- Add type hints to the receipts and user directory handlers. ([\#8976](https://github.com/matrix-org/synapse/issues/8976)) -- Drop the unused `local_invites` table. ([\#8979](https://github.com/matrix-org/synapse/issues/8979)) -- Add type hints to the base storage code. ([\#8980](https://github.com/matrix-org/synapse/issues/8980)) -- Support using PyJWT v2.0.0 in the test suite. ([\#8986](https://github.com/matrix-org/synapse/issues/8986)) -- Fix `tests.federation.transport.RoomDirectoryFederationTests` and ensure it runs in CI. ([\#8998](https://github.com/matrix-org/synapse/issues/8998)) -- Add type hints to the crypto module. ([\#8999](https://github.com/matrix-org/synapse/issues/8999)) - - -Synapse 1.24.0 (2020-12-09) -=========================== - -Due to the two security issues highlighted below, server administrators are -encouraged to update Synapse. We are not aware of these vulnerabilities being -exploited in the wild. - -Security advisory ------------------ - -The following issues are fixed in v1.23.1 and v1.24.0. - -- There is a denial of service attack - ([CVE-2020-26257](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-26257)) - against the federation APIs in which future events will not be correctly sent - to other servers over federation. This affects all servers that participate in - open federation. (Fixed in [#8776](https://github.com/matrix-org/synapse/pull/8776)). - -- Synapse may be affected by OpenSSL - [CVE-2020-1971](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-1971). - Synapse administrators should ensure that they have the latest versions of - the cryptography Python package installed. - -To upgrade Synapse along with the cryptography package: - -* Administrators using the [`matrix.org` Docker - image](https://hub.docker.com/r/matrixdotorg/synapse/) or the [Debian/Ubuntu - packages from - `matrix.org`](https://matrix-org.github.io/synapse/latest/setup/installation.html#matrixorg-packages) - should ensure that they have version 1.24.0 or 1.23.1 installed: these images include - the updated packages. -* Administrators who have [installed Synapse from - source](https://matrix-org.github.io/synapse/latest/setup/installation.html#installing-from-source) - should upgrade the cryptography package within their virtualenv by running: - ```sh - /bin/pip install 'cryptography>=3.3' - ``` -* Administrators who have installed Synapse from distribution packages should - consult the information from their distributions. - -Internal Changes ----------------- - -- Add a maximum version for pysaml2 on Python 3.5. ([\#8898](https://github.com/matrix-org/synapse/issues/8898)) - - -Synapse 1.23.1 (2020-12-09) -=========================== - -Due to the two security issues highlighted below, server administrators are -encouraged to update Synapse. We are not aware of these vulnerabilities being -exploited in the wild. - -Security advisory ------------------ - -The following issues are fixed in v1.23.1 and v1.24.0. - -- There is a denial of service attack - ([CVE-2020-26257](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-26257)) - against the federation APIs in which future events will not be correctly sent - to other servers over federation. This affects all servers that participate in - open federation. (Fixed in [#8776](https://github.com/matrix-org/synapse/pull/8776)). - -- Synapse may be affected by OpenSSL - [CVE-2020-1971](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-1971). - Synapse administrators should ensure that they have the latest versions of - the cryptography Python package installed. - -To upgrade Synapse along with the cryptography package: - -* Administrators using the [`matrix.org` Docker - image](https://hub.docker.com/r/matrixdotorg/synapse/) or the [Debian/Ubuntu - packages from - `matrix.org`](https://matrix-org.github.io/synapse/latest/setup/installation.html#matrixorg-packages) - should ensure that they have version 1.24.0 or 1.23.1 installed: these images include - the updated packages. -* Administrators who have [installed Synapse from - source](https://matrix-org.github.io/synapse/latest/setup/installation.html#installing-from-source) - should upgrade the cryptography package within their virtualenv by running: - ```sh - /bin/pip install 'cryptography>=3.3' - ``` -* Administrators who have installed Synapse from distribution packages should - consult the information from their distributions. - -Bugfixes --------- - -- Fix a bug in some federation APIs which could lead to unexpected behaviour if different parameters were set in the URI and the request body. ([\#8776](https://github.com/matrix-org/synapse/issues/8776)) - - -Internal Changes ----------------- - -- Add a maximum version for pysaml2 on Python 3.5. ([\#8898](https://github.com/matrix-org/synapse/issues/8898)) - - -Synapse 1.24.0rc2 (2020-12-04) -============================== - -Bugfixes --------- - -- Fix a regression in v1.24.0rc1 which failed to allow SAML mapping providers which were unable to redirect users to an additional page. ([\#8878](https://github.com/matrix-org/synapse/issues/8878)) - - -Internal Changes ----------------- - -- Add support for the `prometheus_client` newer than 0.9.0. Contributed by Jordan Bancino. ([\#8875](https://github.com/matrix-org/synapse/issues/8875)) - - -Synapse 1.24.0rc1 (2020-12-02) -============================== - -Features --------- - -- Add admin API for logging in as a user. ([\#8617](https://github.com/matrix-org/synapse/issues/8617)) -- Allow specification of the SAML IdP if the metadata returns multiple IdPs. ([\#8630](https://github.com/matrix-org/synapse/issues/8630)) -- Add support for re-trying generation of a localpart for OpenID Connect mapping providers. ([\#8801](https://github.com/matrix-org/synapse/issues/8801), [\#8855](https://github.com/matrix-org/synapse/issues/8855)) -- Allow the `Date` header through CORS. Contributed by Nicolas Chamo. ([\#8804](https://github.com/matrix-org/synapse/issues/8804)) -- Add a config option, `push.group_by_unread_count`, which controls whether unread message counts in push notifications are defined as "the number of rooms with unread messages" or "total unread messages". ([\#8820](https://github.com/matrix-org/synapse/issues/8820)) -- Add `force_purge` option to delete-room admin api. ([\#8843](https://github.com/matrix-org/synapse/issues/8843)) - - -Bugfixes --------- - -- Fix a bug where appservices may be sent an excessive amount of read receipts and presence. Broke in v1.22.0. ([\#8744](https://github.com/matrix-org/synapse/issues/8744)) -- Fix a bug in some federation APIs which could lead to unexpected behaviour if different parameters were set in the URI and the request body. ([\#8776](https://github.com/matrix-org/synapse/issues/8776)) -- Fix a bug where synctl could spawn duplicate copies of a worker. Contributed by Waylon Cude. ([\#8798](https://github.com/matrix-org/synapse/issues/8798)) -- Allow per-room profiles to be used for the server notice user. ([\#8799](https://github.com/matrix-org/synapse/issues/8799)) -- Fix a bug where logging could break after a call to SIGHUP. ([\#8817](https://github.com/matrix-org/synapse/issues/8817)) -- Fix `register_new_matrix_user` failing with "Bad Request" when trailing slash is included in server URL. Contributed by @angdraug. ([\#8823](https://github.com/matrix-org/synapse/issues/8823)) -- Fix a minor long-standing bug in login, where we would offer the `password` login type if a custom auth provider supported it, even if password login was disabled. ([\#8835](https://github.com/matrix-org/synapse/issues/8835)) -- Fix a long-standing bug which caused Synapse to require unspecified parameters during user-interactive authentication. ([\#8848](https://github.com/matrix-org/synapse/issues/8848)) -- Fix a bug introduced in v1.20.0 where the user-agent and IP address reported during user registration for CAS, OpenID Connect, and SAML were of the wrong form. ([\#8784](https://github.com/matrix-org/synapse/issues/8784)) - - -Improved Documentation ----------------------- - -- Clarify the usecase for a msisdn delegate. Contributed by Adrian Wannenmacher. ([\#8734](https://github.com/matrix-org/synapse/issues/8734)) -- Remove extraneous comma from JSON example in User Admin API docs. ([\#8771](https://github.com/matrix-org/synapse/issues/8771)) -- Update `turn-howto.md` with troubleshooting notes. ([\#8779](https://github.com/matrix-org/synapse/issues/8779)) -- Fix the example on how to set the `Content-Type` header in nginx for the Client Well-Known URI. ([\#8793](https://github.com/matrix-org/synapse/issues/8793)) -- Improve the documentation for the admin API to list all media in a room with respect to encrypted events. ([\#8795](https://github.com/matrix-org/synapse/issues/8795)) -- Update the formatting of the `push` section of the homeserver config file to better align with the [code style guidelines](https://github.com/matrix-org/synapse/blob/develop/docs/code_style.md#configuration-file-format). ([\#8818](https://github.com/matrix-org/synapse/issues/8818)) -- Improve documentation how to configure prometheus for workers. ([\#8822](https://github.com/matrix-org/synapse/issues/8822)) -- Update example prometheus console. ([\#8824](https://github.com/matrix-org/synapse/issues/8824)) - - -Deprecations and Removals -------------------------- - -- Remove old `/_matrix/client/*/admin` endpoints which were deprecated since Synapse 1.20.0. ([\#8785](https://github.com/matrix-org/synapse/issues/8785)) -- Disable pretty printing JSON responses for curl. Users who want pretty-printed output should use [jq](https://stedolan.github.io/jq/) in combination with curl. Contributed by @tulir. ([\#8833](https://github.com/matrix-org/synapse/issues/8833)) - - -Internal Changes ----------------- - -- Simplify the way the `HomeServer` object caches its internal attributes. ([\#8565](https://github.com/matrix-org/synapse/issues/8565), [\#8851](https://github.com/matrix-org/synapse/issues/8851)) -- Add an example and documentation for clock skew to the SAML2 sample configuration to allow for clock/time difference between the homserver and IdP. Contributed by @localguru. ([\#8731](https://github.com/matrix-org/synapse/issues/8731)) -- Generalise `RoomMemberHandler._locally_reject_invite` to apply to more flows than just invite. ([\#8751](https://github.com/matrix-org/synapse/issues/8751)) -- Generalise `RoomStore.maybe_store_room_on_invite` to handle other, non-invite membership events. ([\#8754](https://github.com/matrix-org/synapse/issues/8754)) -- Refactor test utilities for injecting HTTP requests. ([\#8757](https://github.com/matrix-org/synapse/issues/8757), [\#8758](https://github.com/matrix-org/synapse/issues/8758), [\#8759](https://github.com/matrix-org/synapse/issues/8759), [\#8760](https://github.com/matrix-org/synapse/issues/8760), [\#8761](https://github.com/matrix-org/synapse/issues/8761), [\#8777](https://github.com/matrix-org/synapse/issues/8777)) -- Consolidate logic between the OpenID Connect and SAML code. ([\#8765](https://github.com/matrix-org/synapse/issues/8765)) -- Use `TYPE_CHECKING` instead of magic `MYPY` variable. ([\#8770](https://github.com/matrix-org/synapse/issues/8770)) -- Add a commandline script to sign arbitrary json objects. ([\#8772](https://github.com/matrix-org/synapse/issues/8772)) -- Minor log line improvements for the SSO mapping code used to generate Matrix IDs from SSO IDs. ([\#8773](https://github.com/matrix-org/synapse/issues/8773)) -- Add additional error checking for OpenID Connect and SAML mapping providers. ([\#8774](https://github.com/matrix-org/synapse/issues/8774), [\#8800](https://github.com/matrix-org/synapse/issues/8800)) -- Add type hints to HTTP abstractions. ([\#8806](https://github.com/matrix-org/synapse/issues/8806), [\#8812](https://github.com/matrix-org/synapse/issues/8812)) -- Remove unnecessary function arguments and add typing to several membership replication classes. ([\#8809](https://github.com/matrix-org/synapse/issues/8809)) -- Optimise the lookup for an invite from another homeserver when trying to reject it. ([\#8815](https://github.com/matrix-org/synapse/issues/8815)) -- Add tests for `password_auth_provider`s. ([\#8819](https://github.com/matrix-org/synapse/issues/8819)) -- Drop redundant database index on `event_json`. ([\#8845](https://github.com/matrix-org/synapse/issues/8845)) -- Simplify `uk.half-shot.msc2778.login.application_service` login handler. ([\#8847](https://github.com/matrix-org/synapse/issues/8847)) -- Refactor `password_auth_provider` support code. ([\#8849](https://github.com/matrix-org/synapse/issues/8849)) -- Add missing `ordering` to background database updates. ([\#8850](https://github.com/matrix-org/synapse/issues/8850)) -- Allow for specifying a room version when creating a room in unit tests via `RestHelper.create_room_as`. ([\#8854](https://github.com/matrix-org/synapse/issues/8854)) - - -Synapse 1.23.0 (2020-11-18) -=========================== - -This release changes the way structured logging is configured. See the [upgrade notes](docs/upgrade.md#upgrading-to-v1230) for details. - -**Note**: We are aware of a trivially exploitable denial of service vulnerability in versions of Synapse prior to 1.20.0. Complete details will be disclosed on Monday, November 23rd. If you have not upgraded recently, please do so. - -Bugfixes --------- - -- Fix a dependency versioning bug in the Dockerfile that prevented Synapse from starting. ([\#8767](https://github.com/matrix-org/synapse/issues/8767)) - - -Synapse 1.23.0rc1 (2020-11-13) -============================== - -Features --------- - -- Add a push rule that highlights when a jitsi conference is created in a room. ([\#8286](https://github.com/matrix-org/synapse/issues/8286)) -- Add an admin api to delete a single file or files that were not used for a defined time from server. Contributed by @dklimpel. ([\#8519](https://github.com/matrix-org/synapse/issues/8519)) -- Split admin API for reported events (`GET /_synapse/admin/v1/event_reports`) into detail and list endpoints. This is a breaking change to #8217 which was introduced in Synapse v1.21.0. Those who already use this API should check their scripts. Contributed by @dklimpel. ([\#8539](https://github.com/matrix-org/synapse/issues/8539)) -- Support generating structured logs via the standard logging configuration. ([\#8607](https://github.com/matrix-org/synapse/issues/8607), [\#8685](https://github.com/matrix-org/synapse/issues/8685)) -- Add an admin API to allow server admins to list users' pushers. Contributed by @dklimpel. ([\#8610](https://github.com/matrix-org/synapse/issues/8610), [\#8689](https://github.com/matrix-org/synapse/issues/8689)) -- Add an admin API `GET /_synapse/admin/v1/users//media` to get information about uploaded media. Contributed by @dklimpel. ([\#8647](https://github.com/matrix-org/synapse/issues/8647)) -- Add an admin API for local user media statistics. Contributed by @dklimpel. ([\#8700](https://github.com/matrix-org/synapse/issues/8700)) -- Add `displayname` to Shared-Secret Registration for admins. ([\#8722](https://github.com/matrix-org/synapse/issues/8722)) - - -Bugfixes --------- - -- Fix fetching of E2E cross signing keys over federation when only one of the master key and device signing key is cached already. ([\#8455](https://github.com/matrix-org/synapse/issues/8455)) -- Fix a bug where Synapse would blindly forward bad responses from federation to clients when retrieving profile information. ([\#8580](https://github.com/matrix-org/synapse/issues/8580)) -- Fix a bug where the account validity endpoint would silently fail if the user ID did not have an expiration time. It now returns a 400 error. ([\#8620](https://github.com/matrix-org/synapse/issues/8620)) -- Fix email notifications for invites without local state. ([\#8627](https://github.com/matrix-org/synapse/issues/8627)) -- Fix handling of invalid group IDs to return a 400 rather than log an exception and return a 500. ([\#8628](https://github.com/matrix-org/synapse/issues/8628)) -- Fix handling of User-Agent headers that are invalid UTF-8, which caused user agents of users to not get correctly recorded. ([\#8632](https://github.com/matrix-org/synapse/issues/8632)) -- Fix a bug in the `joined_rooms` admin API if the user has never joined any rooms. The bug was introduced, along with the API, in v1.21.0. ([\#8643](https://github.com/matrix-org/synapse/issues/8643)) -- Fix exception during handling multiple concurrent requests for remote media when using multiple media repositories. ([\#8682](https://github.com/matrix-org/synapse/issues/8682)) -- Fix bug that prevented Synapse from recovering after losing connection to the database. ([\#8726](https://github.com/matrix-org/synapse/issues/8726)) -- Fix bug where the `/_synapse/admin/v1/send_server_notice` API could send notices to non-notice rooms. ([\#8728](https://github.com/matrix-org/synapse/issues/8728)) -- Fix PostgreSQL port script fails when DB has no backfilled events. Broke in v1.21.0. ([\#8729](https://github.com/matrix-org/synapse/issues/8729)) -- Fix PostgreSQL port script to correctly handle foreign key constraints. Broke in v1.21.0. ([\#8730](https://github.com/matrix-org/synapse/issues/8730)) -- Fix PostgreSQL port script so that it can be run again after a failure. Broke in v1.21.0. ([\#8755](https://github.com/matrix-org/synapse/issues/8755)) - - -Improved Documentation ----------------------- - -- Instructions for Azure AD in the OpenID Connect documentation. Contributed by peterk. ([\#8582](https://github.com/matrix-org/synapse/issues/8582)) -- Improve the sample configuration for single sign-on providers. ([\#8635](https://github.com/matrix-org/synapse/issues/8635)) -- Fix the filepath of Dex's example config and the link to Dex's Getting Started guide in the OpenID Connect docs. ([\#8657](https://github.com/matrix-org/synapse/issues/8657)) -- Note support for Python 3.9. ([\#8665](https://github.com/matrix-org/synapse/issues/8665)) -- Minor updates to docs on running tests. ([\#8666](https://github.com/matrix-org/synapse/issues/8666)) -- Interlink prometheus/grafana documentation. ([\#8667](https://github.com/matrix-org/synapse/issues/8667)) -- Notes on SSO logins and media_repository worker. ([\#8701](https://github.com/matrix-org/synapse/issues/8701)) -- Document experimental support for running multiple event persisters. ([\#8706](https://github.com/matrix-org/synapse/issues/8706)) -- Add information regarding the various sources of, and expected contributions to, Synapse's documentation to `CONTRIBUTING.md`. ([\#8714](https://github.com/matrix-org/synapse/issues/8714)) -- Migrate documentation `docs/admin_api/event_reports` to markdown. ([\#8742](https://github.com/matrix-org/synapse/issues/8742)) -- Add some helpful hints to the README for new Synapse developers. Contributed by @chagai95. ([\#8746](https://github.com/matrix-org/synapse/issues/8746)) - - -Internal Changes ----------------- - -- Optimise `/createRoom` with multiple invited users. ([\#8559](https://github.com/matrix-org/synapse/issues/8559)) -- Implement and use an `@lru_cache` decorator. ([\#8595](https://github.com/matrix-org/synapse/issues/8595)) -- Don't instansiate Requester directly. ([\#8614](https://github.com/matrix-org/synapse/issues/8614)) -- Type hints for `RegistrationStore`. ([\#8615](https://github.com/matrix-org/synapse/issues/8615)) -- Change schema to support access tokens belonging to one user but granting access to another. ([\#8616](https://github.com/matrix-org/synapse/issues/8616)) -- Remove unused OPTIONS handlers. ([\#8621](https://github.com/matrix-org/synapse/issues/8621)) -- Run `mypy` as part of the lint.sh script. ([\#8633](https://github.com/matrix-org/synapse/issues/8633)) -- Correct Synapse's PyPI package name in the OpenID Connect installation instructions. ([\#8634](https://github.com/matrix-org/synapse/issues/8634)) -- Catch exceptions during initialization of `password_providers`. Contributed by Nicolai Søborg. ([\#8636](https://github.com/matrix-org/synapse/issues/8636)) -- Fix typos and spelling errors in the code. ([\#8639](https://github.com/matrix-org/synapse/issues/8639)) -- Reduce number of OpenTracing spans started. ([\#8640](https://github.com/matrix-org/synapse/issues/8640), [\#8668](https://github.com/matrix-org/synapse/issues/8668), [\#8670](https://github.com/matrix-org/synapse/issues/8670)) -- Add field `total` to device list in admin API. ([\#8644](https://github.com/matrix-org/synapse/issues/8644)) -- Add more type hints to the application services code. ([\#8655](https://github.com/matrix-org/synapse/issues/8655), [\#8693](https://github.com/matrix-org/synapse/issues/8693)) -- Tell Black to format code for Python 3.5. ([\#8664](https://github.com/matrix-org/synapse/issues/8664)) -- Don't pull event from DB when handling replication traffic. ([\#8669](https://github.com/matrix-org/synapse/issues/8669)) -- Abstract some invite-related code in preparation for landing knocking. ([\#8671](https://github.com/matrix-org/synapse/issues/8671), [\#8688](https://github.com/matrix-org/synapse/issues/8688)) -- Clarify representation of events in logfiles. ([\#8679](https://github.com/matrix-org/synapse/issues/8679)) -- Don't require `hiredis` package to be installed to run unit tests. ([\#8680](https://github.com/matrix-org/synapse/issues/8680)) -- Fix typing info on cache call signature to accept `on_invalidate`. ([\#8684](https://github.com/matrix-org/synapse/issues/8684)) -- Fail tests if they do not await coroutines. ([\#8690](https://github.com/matrix-org/synapse/issues/8690)) -- Improve start time by adding an index to `e2e_cross_signing_keys.stream_id`. ([\#8694](https://github.com/matrix-org/synapse/issues/8694)) -- Re-organize the structured logging code to separate the TCP transport handling from the JSON formatting. ([\#8697](https://github.com/matrix-org/synapse/issues/8697)) -- Use Python 3.8 in Docker images by default. ([\#8698](https://github.com/matrix-org/synapse/issues/8698)) -- Remove the "draft" status of the Room Details Admin API. ([\#8702](https://github.com/matrix-org/synapse/issues/8702)) -- Improve the error returned when a non-string displayname or avatar_url is used when updating a user's profile. ([\#8705](https://github.com/matrix-org/synapse/issues/8705)) -- Block attempts by clients to send server ACLs, or redactions of server ACLs, that would result in the local server being blocked from the room. ([\#8708](https://github.com/matrix-org/synapse/issues/8708)) -- Add metrics the allow the local sysadmin to track 3PID `/requestToken` requests. ([\#8712](https://github.com/matrix-org/synapse/issues/8712)) -- Consolidate duplicated lists of purged tables that are checked in tests. ([\#8713](https://github.com/matrix-org/synapse/issues/8713)) -- Add some `mdui:UIInfo` element examples for `saml2_config` in the homeserver config. ([\#8718](https://github.com/matrix-org/synapse/issues/8718)) -- Improve the error message returned when a remote server incorrectly sets the `Content-Type` header in response to a JSON request. ([\#8719](https://github.com/matrix-org/synapse/issues/8719)) -- Speed up repeated state resolutions on the same room by caching event ID to auth event ID lookups. ([\#8752](https://github.com/matrix-org/synapse/issues/8752)) - - -Synapse 1.22.1 (2020-10-30) -=========================== - -Bugfixes --------- - -- Fix a bug where an appservice may not be forwarded events for a room it was recently invited to. Broke in v1.22.0. ([\#8676](https://github.com/matrix-org/synapse/issues/8676)) -- Fix `Object of type frozendict is not JSON serializable` exceptions when using third-party event rules. Broke in v1.22.0. ([\#8678](https://github.com/matrix-org/synapse/issues/8678)) - - -Synapse 1.22.0 (2020-10-27) -=========================== - -No significant changes. - - -Synapse 1.22.0rc2 (2020-10-26) -============================== - -Bugfixes --------- - -- Fix bugs where ephemeral events were not sent to appservices. Broke in v1.22.0rc1. ([\#8648](https://github.com/matrix-org/synapse/issues/8648), [\#8656](https://github.com/matrix-org/synapse/issues/8656)) -- Fix `user_daily_visits` table to not have duplicate rows per user/device due to multiple user agents. Broke in v1.22.0rc1. ([\#8654](https://github.com/matrix-org/synapse/issues/8654)) - -Synapse 1.22.0rc1 (2020-10-22) -============================== - -Features --------- - -- Add a configuration option for always using the "userinfo endpoint" for OpenID Connect. This fixes support for some identity providers, e.g. GitLab. Contributed by Benjamin Koch. ([\#7658](https://github.com/matrix-org/synapse/issues/7658)) -- Add ability for `ThirdPartyEventRules` modules to query and manipulate whether a room is in the public rooms directory. ([\#8292](https://github.com/matrix-org/synapse/issues/8292), [\#8467](https://github.com/matrix-org/synapse/issues/8467)) -- Add support for olm fallback keys ([MSC2732](https://github.com/matrix-org/matrix-doc/pull/2732)). ([\#8312](https://github.com/matrix-org/synapse/issues/8312), [\#8501](https://github.com/matrix-org/synapse/issues/8501)) -- Add support for running background tasks in a separate worker process. ([\#8369](https://github.com/matrix-org/synapse/issues/8369), [\#8458](https://github.com/matrix-org/synapse/issues/8458), [\#8489](https://github.com/matrix-org/synapse/issues/8489), [\#8513](https://github.com/matrix-org/synapse/issues/8513), [\#8544](https://github.com/matrix-org/synapse/issues/8544), [\#8599](https://github.com/matrix-org/synapse/issues/8599)) -- Add support for device dehydration ([MSC2697](https://github.com/matrix-org/matrix-doc/pull/2697)). ([\#8380](https://github.com/matrix-org/synapse/issues/8380)) -- Add support for [MSC2409](https://github.com/matrix-org/matrix-doc/pull/2409), which allows sending typing, read receipts, and presence events to appservices. ([\#8437](https://github.com/matrix-org/synapse/issues/8437), [\#8590](https://github.com/matrix-org/synapse/issues/8590)) -- Change default room version to "6", per [MSC2788](https://github.com/matrix-org/matrix-doc/pull/2788). ([\#8461](https://github.com/matrix-org/synapse/issues/8461)) -- Add the ability to send non-membership events into a room via the `ModuleApi`. ([\#8479](https://github.com/matrix-org/synapse/issues/8479)) -- Increase default upload size limit from 10M to 50M. Contributed by @Akkowicz. ([\#8502](https://github.com/matrix-org/synapse/issues/8502)) -- Add support for modifying event content in `ThirdPartyRules` modules. ([\#8535](https://github.com/matrix-org/synapse/issues/8535), [\#8564](https://github.com/matrix-org/synapse/issues/8564)) - - -Bugfixes --------- - -- Fix a longstanding bug where invalid ignored users in account data could break clients. ([\#8454](https://github.com/matrix-org/synapse/issues/8454)) -- Fix a bug where backfilling a room with an event that was missing the `redacts` field would break. ([\#8457](https://github.com/matrix-org/synapse/issues/8457)) -- Don't attempt to respond to some requests if the client has already disconnected. ([\#8465](https://github.com/matrix-org/synapse/issues/8465)) -- Fix message duplication if something goes wrong after persisting the event. ([\#8476](https://github.com/matrix-org/synapse/issues/8476)) -- Fix incremental sync returning an incorrect `prev_batch` token in timeline section, which when used to paginate returned events that were included in the incremental sync. Broken since v0.16.0. ([\#8486](https://github.com/matrix-org/synapse/issues/8486)) -- Expose the `uk.half-shot.msc2778.login.application_service` to clients from the login API. This feature was added in v1.21.0, but was not exposed as a potential login flow. ([\#8504](https://github.com/matrix-org/synapse/issues/8504)) -- Fix error code for `/profile/{userId}/displayname` to be `M_BAD_JSON`. ([\#8517](https://github.com/matrix-org/synapse/issues/8517)) -- Fix a bug introduced in v1.7.0 that could cause Synapse to insert values from non-state `m.room.retention` events into the `room_retention` database table. ([\#8527](https://github.com/matrix-org/synapse/issues/8527)) -- Fix not sending events over federation when using sharded event writers. ([\#8536](https://github.com/matrix-org/synapse/issues/8536)) -- Fix a long standing bug where email notifications for encrypted messages were blank. ([\#8545](https://github.com/matrix-org/synapse/issues/8545)) -- Fix increase in the number of `There was no active span...` errors logged when using OpenTracing. ([\#8567](https://github.com/matrix-org/synapse/issues/8567)) -- Fix a bug that prevented errors encountered during execution of the `synapse_port_db` from being correctly printed. ([\#8585](https://github.com/matrix-org/synapse/issues/8585)) -- Fix appservice transactions to only include a maximum of 100 persistent and 100 ephemeral events. ([\#8606](https://github.com/matrix-org/synapse/issues/8606)) - - -Updates to the Docker image ---------------------------- - -- Added multi-arch support (arm64,arm/v7) for the docker images. Contributed by @maquis196. ([\#7921](https://github.com/matrix-org/synapse/issues/7921)) -- Add support for passing commandline args to the synapse process. Contributed by @samuel-p. ([\#8390](https://github.com/matrix-org/synapse/issues/8390)) - - -Improved Documentation ----------------------- - -- Update the directions for using the manhole with coroutines. ([\#8462](https://github.com/matrix-org/synapse/issues/8462)) -- Improve readme by adding new shield.io badges. ([\#8493](https://github.com/matrix-org/synapse/issues/8493)) -- Added note about docker in manhole.md regarding which ip address to bind to. Contributed by @Maquis196. ([\#8526](https://github.com/matrix-org/synapse/issues/8526)) -- Document the new behaviour of the `allowed_lifetime_min` and `allowed_lifetime_max` settings in the room retention configuration. ([\#8529](https://github.com/matrix-org/synapse/issues/8529)) - - -Deprecations and Removals -------------------------- - -- Drop unused `device_max_stream_id` table. ([\#8589](https://github.com/matrix-org/synapse/issues/8589)) - - -Internal Changes ----------------- - -- Check for unreachable code with mypy. ([\#8432](https://github.com/matrix-org/synapse/issues/8432)) -- Add unit test for event persister sharding. ([\#8433](https://github.com/matrix-org/synapse/issues/8433)) -- Allow events to be sent to clients sooner when using sharded event persisters. ([\#8439](https://github.com/matrix-org/synapse/issues/8439), [\#8488](https://github.com/matrix-org/synapse/issues/8488), [\#8496](https://github.com/matrix-org/synapse/issues/8496), [\#8499](https://github.com/matrix-org/synapse/issues/8499)) -- Configure `public_baseurl` when using demo scripts. ([\#8443](https://github.com/matrix-org/synapse/issues/8443)) -- Add SQL logging on queries that happen during startup. ([\#8448](https://github.com/matrix-org/synapse/issues/8448)) -- Speed up unit tests when using PostgreSQL. ([\#8450](https://github.com/matrix-org/synapse/issues/8450)) -- Remove redundant database loads of stream_ordering for events we already have. ([\#8452](https://github.com/matrix-org/synapse/issues/8452)) -- Reduce inconsistencies between codepaths for membership and non-membership events. ([\#8463](https://github.com/matrix-org/synapse/issues/8463)) -- Combine `SpamCheckerApi` with the more generic `ModuleApi`. ([\#8464](https://github.com/matrix-org/synapse/issues/8464)) -- Additional testing for `ThirdPartyEventRules`. ([\#8468](https://github.com/matrix-org/synapse/issues/8468)) -- Add `-d` option to `./scripts-dev/lint.sh` to lint files that have changed since the last git commit. ([\#8472](https://github.com/matrix-org/synapse/issues/8472)) -- Unblacklist some sytests. ([\#8474](https://github.com/matrix-org/synapse/issues/8474)) -- Include the log level in the phone home stats. ([\#8477](https://github.com/matrix-org/synapse/issues/8477)) -- Remove outdated sphinx documentation, scripts and configuration. ([\#8480](https://github.com/matrix-org/synapse/issues/8480)) -- Clarify error message when plugin config parsers raise an error. ([\#8492](https://github.com/matrix-org/synapse/issues/8492)) -- Remove the deprecated `Handlers` object. ([\#8494](https://github.com/matrix-org/synapse/issues/8494)) -- Fix a threadsafety bug in unit tests. ([\#8497](https://github.com/matrix-org/synapse/issues/8497)) -- Add user agent to user_daily_visits table. ([\#8503](https://github.com/matrix-org/synapse/issues/8503)) -- Add type hints to various parts of the code base. ([\#8407](https://github.com/matrix-org/synapse/issues/8407), [\#8505](https://github.com/matrix-org/synapse/issues/8505), [\#8507](https://github.com/matrix-org/synapse/issues/8507), [\#8547](https://github.com/matrix-org/synapse/issues/8547), [\#8562](https://github.com/matrix-org/synapse/issues/8562), [\#8609](https://github.com/matrix-org/synapse/issues/8609)) -- Remove unused code from the test framework. ([\#8514](https://github.com/matrix-org/synapse/issues/8514)) -- Apply some internal fixes to the `HomeServer` class to make its code more idiomatic and statically-verifiable. ([\#8515](https://github.com/matrix-org/synapse/issues/8515)) -- Factor out common code between `RoomMemberHandler._locally_reject_invite` and `EventCreationHandler.create_event`. ([\#8537](https://github.com/matrix-org/synapse/issues/8537)) -- Improve database performance by executing more queries without starting transactions. ([\#8542](https://github.com/matrix-org/synapse/issues/8542)) -- Rename `Cache` to `DeferredCache`, to better reflect its purpose. ([\#8548](https://github.com/matrix-org/synapse/issues/8548)) -- Move metric registration code down into `LruCache`. ([\#8561](https://github.com/matrix-org/synapse/issues/8561), [\#8591](https://github.com/matrix-org/synapse/issues/8591)) -- Replace `DeferredCache` with the lighter-weight `LruCache` where possible. ([\#8563](https://github.com/matrix-org/synapse/issues/8563)) -- Add virtualenv-generated folders to `.gitignore`. ([\#8566](https://github.com/matrix-org/synapse/issues/8566)) -- Add `get_immediate` method to `DeferredCache`. ([\#8568](https://github.com/matrix-org/synapse/issues/8568)) -- Fix mypy not properly checking across the codebase, additionally, fix a typing assertion error in `handlers/auth.py`. ([\#8569](https://github.com/matrix-org/synapse/issues/8569)) -- Fix `synmark` benchmark runner. ([\#8571](https://github.com/matrix-org/synapse/issues/8571)) -- Modify `DeferredCache.get()` to return `Deferred`s instead of `ObservableDeferred`s. ([\#8572](https://github.com/matrix-org/synapse/issues/8572)) -- Adjust a protocol-type definition to fit `sqlite3` assertions. ([\#8577](https://github.com/matrix-org/synapse/issues/8577)) -- Support macOS on the `synmark` benchmark runner. ([\#8578](https://github.com/matrix-org/synapse/issues/8578)) -- Update `mypy` static type checker to 0.790. ([\#8583](https://github.com/matrix-org/synapse/issues/8583), [\#8600](https://github.com/matrix-org/synapse/issues/8600)) -- Re-organize the structured logging code to separate the TCP transport handling from the JSON formatting. ([\#8587](https://github.com/matrix-org/synapse/issues/8587)) -- Remove extraneous unittest logging decorators from unit tests. ([\#8592](https://github.com/matrix-org/synapse/issues/8592)) -- Minor optimisations in caching code. ([\#8593](https://github.com/matrix-org/synapse/issues/8593), [\#8594](https://github.com/matrix-org/synapse/issues/8594)) - - -Synapse 1.21.2 (2020-10-15) -=========================== - -Debian packages and Docker images have been rebuilt using the latest versions of dependency libraries, including authlib 0.15.1. Please see bugfixes below. - -Security advisory ------------------ - -* HTML pages served via Synapse were vulnerable to cross-site scripting (XSS) - attacks. All server administrators are encouraged to upgrade. - ([\#8444](https://github.com/matrix-org/synapse/pull/8444)) - ([CVE-2020-26891](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-26891)) - - This fix was originally included in v1.21.0 but was missing a security advisory. - - This was reported by [Denis Kasak](https://github.com/dkasak). - -Bugfixes --------- - -- Fix rare bug where sending an event would fail due to a racey assertion. ([\#8530](https://github.com/matrix-org/synapse/issues/8530)) -- An updated version of the authlib dependency is included in the Docker and Debian images to fix an issue using OpenID Connect. See [\#8534](https://github.com/matrix-org/synapse/issues/8534) for details. - - -Synapse 1.21.1 (2020-10-13) -=========================== - -This release fixes a regression in v1.21.0 that prevented debian packages from being built. -It is otherwise identical to v1.21.0. - -Synapse 1.21.0 (2020-10-12) -=========================== - -No significant changes since v1.21.0rc3. - -As [noted in -v1.20.0](https://github.com/matrix-org/synapse/blob/release-v1.21.0/CHANGES.md#synapse-1200-2020-09-22), -a future release will drop support for accessing Synapse's -[Admin API](https://github.com/matrix-org/synapse/tree/master/docs/admin_api) under the -`/_matrix/client/*` endpoint prefixes. At that point, the Admin API will only -be accessible under `/_synapse/admin`. - - -Synapse 1.21.0rc3 (2020-10-08) -============================== - -Bugfixes --------- - -- Fix duplication of events on high traffic servers, caused by PostgreSQL `could not serialize access due to concurrent update` errors. ([\#8456](https://github.com/matrix-org/synapse/issues/8456)) - - -Internal Changes ----------------- - -- Add Groovy Gorilla to the list of distributions we build `.deb`s for. ([\#8475](https://github.com/matrix-org/synapse/issues/8475)) - - -Synapse 1.21.0rc2 (2020-10-02) -============================== - -Features --------- - -- Convert additional templates from inline HTML to Jinja2 templates. ([\#8444](https://github.com/matrix-org/synapse/issues/8444)) - -Bugfixes --------- - -- Fix a regression in v1.21.0rc1 which broke thumbnails of remote media. ([\#8438](https://github.com/matrix-org/synapse/issues/8438)) -- Do not expose the experimental `uk.half-shot.msc2778.login.application_service` flow in the login API, which caused a compatibility problem with Element iOS. ([\#8440](https://github.com/matrix-org/synapse/issues/8440)) -- Fix malformed log line in new federation "catch up" logic. ([\#8442](https://github.com/matrix-org/synapse/issues/8442)) -- Fix DB query on startup for negative streams which caused long start up times. Introduced in [\#8374](https://github.com/matrix-org/synapse/issues/8374). ([\#8447](https://github.com/matrix-org/synapse/issues/8447)) - - -Synapse 1.21.0rc1 (2020-10-01) -============================== - -Features --------- - -- Require the user to confirm that their password should be reset after clicking the email confirmation link. ([\#8004](https://github.com/matrix-org/synapse/issues/8004)) -- Add an admin API `GET /_synapse/admin/v1/event_reports` to read entries of table `event_reports`. Contributed by @dklimpel. ([\#8217](https://github.com/matrix-org/synapse/issues/8217)) -- Consolidate the SSO error template across all configuration. ([\#8248](https://github.com/matrix-org/synapse/issues/8248), [\#8405](https://github.com/matrix-org/synapse/issues/8405)) -- Add a configuration option to specify a whitelist of domains that a user can be redirected to after validating their email or phone number. ([\#8275](https://github.com/matrix-org/synapse/issues/8275), [\#8417](https://github.com/matrix-org/synapse/issues/8417)) -- Add experimental support for sharding event persister. ([\#8294](https://github.com/matrix-org/synapse/issues/8294), [\#8387](https://github.com/matrix-org/synapse/issues/8387), [\#8396](https://github.com/matrix-org/synapse/issues/8396), [\#8419](https://github.com/matrix-org/synapse/issues/8419)) -- Add the room topic and avatar to the room details admin API. ([\#8305](https://github.com/matrix-org/synapse/issues/8305)) -- Add an admin API for querying rooms where a user is a member. Contributed by @dklimpel. ([\#8306](https://github.com/matrix-org/synapse/issues/8306)) -- Add `uk.half-shot.msc2778.login.application_service` login type to allow appservices to login. ([\#8320](https://github.com/matrix-org/synapse/issues/8320)) -- Add a configuration option that allows existing users to log in with OpenID Connect. Contributed by @BBBSnowball and @OmmyZhang. ([\#8345](https://github.com/matrix-org/synapse/issues/8345)) -- Add prometheus metrics for replication requests. ([\#8406](https://github.com/matrix-org/synapse/issues/8406)) -- Support passing additional single sign-on parameters to the client. ([\#8413](https://github.com/matrix-org/synapse/issues/8413)) -- Add experimental reporting of metrics on expensive rooms for state-resolution. ([\#8420](https://github.com/matrix-org/synapse/issues/8420)) -- Add experimental prometheus metric to track numbers of "large" rooms for state resolutiom. ([\#8425](https://github.com/matrix-org/synapse/issues/8425)) -- Add prometheus metrics to track federation delays. ([\#8430](https://github.com/matrix-org/synapse/issues/8430)) - - -Bugfixes --------- - -- Fix a bug in the media repository where remote thumbnails with the same size but different crop methods would overwrite each other. Contributed by @deepbluev7. ([\#7124](https://github.com/matrix-org/synapse/issues/7124)) -- Fix inconsistent handling of non-existent push rules, and stop tracking the `enabled` state of removed push rules. ([\#7796](https://github.com/matrix-org/synapse/issues/7796)) -- Fix a longstanding bug when storing a media file with an empty `upload_name`. ([\#7905](https://github.com/matrix-org/synapse/issues/7905)) -- Fix messages not being sent over federation until an event is sent into the same room. ([\#8230](https://github.com/matrix-org/synapse/issues/8230), [\#8247](https://github.com/matrix-org/synapse/issues/8247), [\#8258](https://github.com/matrix-org/synapse/issues/8258), [\#8272](https://github.com/matrix-org/synapse/issues/8272), [\#8322](https://github.com/matrix-org/synapse/issues/8322)) -- Fix a longstanding bug where files that could not be thumbnailed would result in an Internal Server Error. ([\#8236](https://github.com/matrix-org/synapse/issues/8236), [\#8435](https://github.com/matrix-org/synapse/issues/8435)) -- Upgrade minimum version of `canonicaljson` to version 1.4.0, to fix an unicode encoding issue. ([\#8262](https://github.com/matrix-org/synapse/issues/8262)) -- Fix longstanding bug which could lead to incomplete database upgrades on SQLite. ([\#8265](https://github.com/matrix-org/synapse/issues/8265)) -- Fix stack overflow when stderr is redirected to the logging system, and the logging system encounters an error. ([\#8268](https://github.com/matrix-org/synapse/issues/8268)) -- Fix a bug which cause the logging system to report errors, if `DEBUG` was enabled and no `context` filter was applied. ([\#8278](https://github.com/matrix-org/synapse/issues/8278)) -- Fix edge case where push could get delayed for a user until a later event was pushed. ([\#8287](https://github.com/matrix-org/synapse/issues/8287)) -- Fix fetching malformed events from remote servers. ([\#8324](https://github.com/matrix-org/synapse/issues/8324)) -- Fix `UnboundLocalError` from occuring when appservices send a malformed register request. ([\#8329](https://github.com/matrix-org/synapse/issues/8329)) -- Don't send push notifications to expired user accounts. ([\#8353](https://github.com/matrix-org/synapse/issues/8353)) -- Fix a regression in v1.19.0 with reactivating users through the admin API. ([\#8362](https://github.com/matrix-org/synapse/issues/8362)) -- Fix a bug where during device registration the length of the device name wasn't limited. ([\#8364](https://github.com/matrix-org/synapse/issues/8364)) -- Include `guest_access` in the fields that are checked for null bytes when updating `room_stats_state`. Broke in v1.7.2. ([\#8373](https://github.com/matrix-org/synapse/issues/8373)) -- Fix theoretical race condition where events are not sent down `/sync` if the synchrotron worker is restarted without restarting other workers. ([\#8374](https://github.com/matrix-org/synapse/issues/8374)) -- Fix a bug which could cause errors in rooms with malformed membership events, on servers using sqlite. ([\#8385](https://github.com/matrix-org/synapse/issues/8385)) -- Fix "Re-starting finished log context" warning when receiving an event we already had over federation. ([\#8398](https://github.com/matrix-org/synapse/issues/8398)) -- Fix incorrect handling of timeouts on outgoing HTTP requests. ([\#8400](https://github.com/matrix-org/synapse/issues/8400)) -- Fix a regression in v1.20.0 in the `synapse_port_db` script regarding the `ui_auth_sessions_ips` table. ([\#8410](https://github.com/matrix-org/synapse/issues/8410)) -- Remove unnecessary 3PID registration check when resetting password via an email address. Bug introduced in v0.34.0rc2. ([\#8414](https://github.com/matrix-org/synapse/issues/8414)) - - -Improved Documentation ----------------------- - -- Add `/_synapse/client` to the reverse proxy documentation. ([\#8227](https://github.com/matrix-org/synapse/issues/8227)) -- Add note to the reverse proxy settings documentation about disabling Apache's mod_security2. Contributed by Julian Fietkau (@jfietkau). ([\#8375](https://github.com/matrix-org/synapse/issues/8375)) -- Improve description of `server_name` config option in `homserver.yaml`. ([\#8415](https://github.com/matrix-org/synapse/issues/8415)) - - -Deprecations and Removals -------------------------- - -- Drop support for `prometheus_client` older than 0.4.0. ([\#8426](https://github.com/matrix-org/synapse/issues/8426)) - - -Internal Changes ----------------- - -- Fix tests on distros which disable TLSv1.0. Contributed by @danc86. ([\#8208](https://github.com/matrix-org/synapse/issues/8208)) -- Simplify the distributor code to avoid unnecessary work. ([\#8216](https://github.com/matrix-org/synapse/issues/8216)) -- Remove the `populate_stats_process_rooms_2` background job and restore functionality to `populate_stats_process_rooms`. ([\#8243](https://github.com/matrix-org/synapse/issues/8243)) -- Clean up type hints for `PaginationConfig`. ([\#8250](https://github.com/matrix-org/synapse/issues/8250), [\#8282](https://github.com/matrix-org/synapse/issues/8282)) -- Track the latest event for every destination and room for catch-up after federation outage. ([\#8256](https://github.com/matrix-org/synapse/issues/8256)) -- Fix non-user visible bug in implementation of `MultiWriterIdGenerator.get_current_token_for_writer`. ([\#8257](https://github.com/matrix-org/synapse/issues/8257)) -- Switch to the JSON implementation from the standard library. ([\#8259](https://github.com/matrix-org/synapse/issues/8259)) -- Add type hints to `synapse.util.async_helpers`. ([\#8260](https://github.com/matrix-org/synapse/issues/8260)) -- Simplify tests that mock asynchronous functions. ([\#8261](https://github.com/matrix-org/synapse/issues/8261)) -- Add type hints to `StreamToken` and `RoomStreamToken` classes. ([\#8279](https://github.com/matrix-org/synapse/issues/8279)) -- Change `StreamToken.room_key` to be a `RoomStreamToken` instance. ([\#8281](https://github.com/matrix-org/synapse/issues/8281)) -- Refactor notifier code to correctly use the max event stream position. ([\#8288](https://github.com/matrix-org/synapse/issues/8288)) -- Use slotted classes where possible. ([\#8296](https://github.com/matrix-org/synapse/issues/8296)) -- Support testing the local Synapse checkout against the [Complement homeserver test suite](https://github.com/matrix-org/complement/). ([\#8317](https://github.com/matrix-org/synapse/issues/8317)) -- Update outdated usages of `metaclass` to python 3 syntax. ([\#8326](https://github.com/matrix-org/synapse/issues/8326)) -- Move lint-related dependencies to package-extra field, update CONTRIBUTING.md to utilise this. ([\#8330](https://github.com/matrix-org/synapse/issues/8330), [\#8377](https://github.com/matrix-org/synapse/issues/8377)) -- Use the `admin_patterns` helper in additional locations. ([\#8331](https://github.com/matrix-org/synapse/issues/8331)) -- Fix test logging to allow braces in log output. ([\#8335](https://github.com/matrix-org/synapse/issues/8335)) -- Remove `__future__` imports related to Python 2 compatibility. ([\#8337](https://github.com/matrix-org/synapse/issues/8337)) -- Simplify `super()` calls to Python 3 syntax. ([\#8344](https://github.com/matrix-org/synapse/issues/8344)) -- Fix bad merge from `release-v1.20.0` branch to `develop`. ([\#8354](https://github.com/matrix-org/synapse/issues/8354)) -- Factor out a `_send_dummy_event_for_room` method. ([\#8370](https://github.com/matrix-org/synapse/issues/8370)) -- Improve logging of state resolution. ([\#8371](https://github.com/matrix-org/synapse/issues/8371)) -- Add type annotations to `SimpleHttpClient`. ([\#8372](https://github.com/matrix-org/synapse/issues/8372)) -- Refactor ID generators to use `async with` syntax. ([\#8383](https://github.com/matrix-org/synapse/issues/8383)) -- Add `EventStreamPosition` type. ([\#8388](https://github.com/matrix-org/synapse/issues/8388)) -- Create a mechanism for marking tests "logcontext clean". ([\#8399](https://github.com/matrix-org/synapse/issues/8399)) -- A pair of tiny cleanups in the federation request code. ([\#8401](https://github.com/matrix-org/synapse/issues/8401)) -- Add checks on startup that PostgreSQL sequences are consistent with their associated tables. ([\#8402](https://github.com/matrix-org/synapse/issues/8402)) -- Do not include appservice users when calculating the total MAU for a server. ([\#8404](https://github.com/matrix-org/synapse/issues/8404)) -- Typing fixes for `synapse.handlers.federation`. ([\#8422](https://github.com/matrix-org/synapse/issues/8422)) -- Various refactors to simplify stream token handling. ([\#8423](https://github.com/matrix-org/synapse/issues/8423)) -- Make stream token serializing/deserializing async. ([\#8427](https://github.com/matrix-org/synapse/issues/8427)) - - -Synapse 1.20.1 (2020-09-24) -=========================== - -Bugfixes --------- - -- Fix a bug introduced in v1.20.0 which caused the `synapse_port_db` script to fail. ([\#8386](https://github.com/matrix-org/synapse/issues/8386)) -- Fix a bug introduced in v1.20.0 which caused variables to be incorrectly escaped in Jinja2 templates. ([\#8394](https://github.com/matrix-org/synapse/issues/8394)) - - -Synapse 1.20.0 (2020-09-22) -=========================== - -No significant changes since v1.20.0rc5. - -Removal warning ---------------- - -Historically, the [Synapse Admin -API](https://github.com/matrix-org/synapse/tree/master/docs) has been -accessible under the `/_matrix/client/api/v1/admin`, -`/_matrix/client/unstable/admin`, `/_matrix/client/r0/admin` and -`/_synapse/admin` prefixes. In a future release, we will be dropping support -for accessing Synapse's Admin API using the `/_matrix/client/*` prefixes. - -From that point, the Admin API will only be accessible under `/_synapse/admin`. -This makes it easier for homeserver admins to lock down external access to the -Admin API endpoints. - -Synapse 1.20.0rc5 (2020-09-18) -============================== - -In addition to the below, Synapse 1.20.0rc5 also includes the bug fix that was included in 1.19.3. - -Features --------- - -- Add flags to the `/versions` endpoint for whether new rooms default to using E2EE. ([\#8343](https://github.com/matrix-org/synapse/issues/8343)) - - -Bugfixes --------- - -- Fix rate limiting of federation `/send` requests. ([\#8342](https://github.com/matrix-org/synapse/issues/8342)) -- Fix a longstanding bug where back pagination over federation could get stuck if it failed to handle a received event. ([\#8349](https://github.com/matrix-org/synapse/issues/8349)) - - -Internal Changes ----------------- - -- Blacklist [MSC2753](https://github.com/matrix-org/matrix-doc/pull/2753) SyTests until it is implemented. ([\#8285](https://github.com/matrix-org/synapse/issues/8285)) - - -Synapse 1.19.3 (2020-09-18) -=========================== - -Bugfixes --------- - -- Partially mitigate bug where newly joined servers couldn't get past events in a room when there is a malformed event. ([\#8350](https://github.com/matrix-org/synapse/issues/8350)) - - -Synapse 1.20.0rc4 (2020-09-16) -============================== - -Synapse 1.20.0rc4 is identical to 1.20.0rc3, with the addition of the security fix that was included in 1.19.2. - - -Synapse 1.19.2 (2020-09-16) -=========================== - -Due to the issue below server admins are encouraged to upgrade as soon as possible. - -Bugfixes --------- - -- Fix joining rooms over federation that include malformed events. ([\#8324](https://github.com/matrix-org/synapse/issues/8324)) - - -Synapse 1.20.0rc3 (2020-09-11) -============================== - -Bugfixes --------- - -- Fix a bug introduced in v1.20.0rc1 where the wrong exception was raised when invalid JSON data is encountered. ([\#8291](https://github.com/matrix-org/synapse/issues/8291)) - - -Synapse 1.20.0rc2 (2020-09-09) -============================== - -Bugfixes --------- - -- Fix a bug introduced in v1.20.0rc1 causing some features related to notifications to misbehave following the implementation of unread counts. ([\#8280](https://github.com/matrix-org/synapse/issues/8280)) - - -Synapse 1.20.0rc1 (2020-09-08) -============================== - -Removal warning ---------------- - -Some older clients used a [disallowed character](https://matrix.org/docs/spec/client_server/r0.6.1#post-matrix-client-r0-register-email-requesttoken) (`:`) in the `client_secret` parameter of various endpoints. The incorrect behaviour was allowed for backwards compatibility, but is now being removed from Synapse as most users have updated their client. Further context can be found at [\#6766](https://github.com/matrix-org/synapse/issues/6766). - -Features --------- - -- Add an endpoint to query your shared rooms with another user as an implementation of [MSC2666](https://github.com/matrix-org/matrix-doc/pull/2666). ([\#7785](https://github.com/matrix-org/synapse/issues/7785)) -- Iteratively encode JSON to avoid blocking the reactor. ([\#8013](https://github.com/matrix-org/synapse/issues/8013), [\#8116](https://github.com/matrix-org/synapse/issues/8116)) -- Add support for shadow-banning users (ignoring any message send requests). ([\#8034](https://github.com/matrix-org/synapse/issues/8034), [\#8092](https://github.com/matrix-org/synapse/issues/8092), [\#8095](https://github.com/matrix-org/synapse/issues/8095), [\#8142](https://github.com/matrix-org/synapse/issues/8142), [\#8152](https://github.com/matrix-org/synapse/issues/8152), [\#8157](https://github.com/matrix-org/synapse/issues/8157), [\#8158](https://github.com/matrix-org/synapse/issues/8158), [\#8176](https://github.com/matrix-org/synapse/issues/8176)) -- Use the default template file when its equivalent is not found in a custom template directory. ([\#8037](https://github.com/matrix-org/synapse/issues/8037), [\#8107](https://github.com/matrix-org/synapse/issues/8107), [\#8252](https://github.com/matrix-org/synapse/issues/8252)) -- Add unread messages count to sync responses, as specified in [MSC2654](https://github.com/matrix-org/matrix-doc/pull/2654). ([\#8059](https://github.com/matrix-org/synapse/issues/8059), [\#8254](https://github.com/matrix-org/synapse/issues/8254), [\#8270](https://github.com/matrix-org/synapse/issues/8270), [\#8274](https://github.com/matrix-org/synapse/issues/8274)) -- Optimise `/federation/v1/user/devices/` API by only returning devices with encryption keys. ([\#8198](https://github.com/matrix-org/synapse/issues/8198)) - - -Bugfixes --------- - -- Fix a memory leak by limiting the length of time that messages will be queued for a remote server that has been unreachable. ([\#7864](https://github.com/matrix-org/synapse/issues/7864)) -- Fix `Re-starting finished log context PUT-nnnn` warning when event persistence failed. ([\#8081](https://github.com/matrix-org/synapse/issues/8081)) -- Synapse now correctly enforces the valid characters in the `client_secret` parameter used in various endpoints. ([\#8101](https://github.com/matrix-org/synapse/issues/8101)) -- Fix a bug introduced in v1.7.2 impacting message retention policies that would allow federated homeservers to dictate a retention period that's lower than the configured minimum allowed duration in the configuration file. ([\#8104](https://github.com/matrix-org/synapse/issues/8104)) -- Fix a long-standing bug where invalid JSON would be accepted by Synapse. ([\#8106](https://github.com/matrix-org/synapse/issues/8106)) -- Fix a bug introduced in Synapse v1.12.0 which could cause `/sync` requests to fail with a 404 if you had a very old outstanding room invite. ([\#8110](https://github.com/matrix-org/synapse/issues/8110)) -- Return a proper error code when the rooms of an invalid group are requested. ([\#8129](https://github.com/matrix-org/synapse/issues/8129)) -- Fix a bug which could cause a leaked postgres connection if synapse was set to daemonize. ([\#8131](https://github.com/matrix-org/synapse/issues/8131)) -- Clarify the error code if a user tries to register with a numeric ID. This bug was introduced in v1.15.0. ([\#8135](https://github.com/matrix-org/synapse/issues/8135)) -- Fix a bug where appservices with ratelimiting disabled would still be ratelimited when joining rooms. This bug was introduced in v1.19.0. ([\#8139](https://github.com/matrix-org/synapse/issues/8139)) -- Fix logging in via OpenID Connect with a provider that uses integer user IDs. ([\#8190](https://github.com/matrix-org/synapse/issues/8190)) -- Fix a longstanding bug where user directory updates could break when unexpected profile data was included in events. ([\#8223](https://github.com/matrix-org/synapse/issues/8223)) -- Fix a longstanding bug where stats updates could break when unexpected profile data was included in events. ([\#8226](https://github.com/matrix-org/synapse/issues/8226)) -- Fix slow start times for large servers by removing a table scan of the `users` table from startup code. ([\#8271](https://github.com/matrix-org/synapse/issues/8271)) - - -Updates to the Docker image ---------------------------- - -- Fix builds of the Docker image on non-x86 platforms. ([\#8144](https://github.com/matrix-org/synapse/issues/8144)) -- Added curl for healthcheck support and readme updates for the change. Contributed by @maquis196. ([\#8147](https://github.com/matrix-org/synapse/issues/8147)) - - -Improved Documentation ----------------------- - -- Link to matrix-synapse-rest-password-provider in the password provider documentation. ([\#8111](https://github.com/matrix-org/synapse/issues/8111)) -- Updated documentation to note that Synapse does not follow `HTTP 308` redirects due to an upstream library not supporting them. Contributed by Ryan Cole. ([\#8120](https://github.com/matrix-org/synapse/issues/8120)) -- Explain better what GDPR-erased means when deactivating a user. ([\#8189](https://github.com/matrix-org/synapse/issues/8189)) - - -Internal Changes ----------------- - -- Add filter `name` to the `/users` admin API, which filters by user ID or displayname. Contributed by Awesome Technologies Innovationslabor GmbH. ([\#7377](https://github.com/matrix-org/synapse/issues/7377), [\#8163](https://github.com/matrix-org/synapse/issues/8163)) -- Reduce run times of some unit tests by advancing the reactor a fewer number of times. ([\#7757](https://github.com/matrix-org/synapse/issues/7757)) -- Don't fail `/submit_token` requests on incorrect session ID if `request_token_inhibit_3pid_errors` is turned on. ([\#7991](https://github.com/matrix-org/synapse/issues/7991)) -- Convert various parts of the codebase to async/await. ([\#8071](https://github.com/matrix-org/synapse/issues/8071), [\#8072](https://github.com/matrix-org/synapse/issues/8072), [\#8074](https://github.com/matrix-org/synapse/issues/8074), [\#8075](https://github.com/matrix-org/synapse/issues/8075), [\#8076](https://github.com/matrix-org/synapse/issues/8076), [\#8087](https://github.com/matrix-org/synapse/issues/8087), [\#8100](https://github.com/matrix-org/synapse/issues/8100), [\#8119](https://github.com/matrix-org/synapse/issues/8119), [\#8121](https://github.com/matrix-org/synapse/issues/8121), [\#8133](https://github.com/matrix-org/synapse/issues/8133), [\#8156](https://github.com/matrix-org/synapse/issues/8156), [\#8162](https://github.com/matrix-org/synapse/issues/8162), [\#8166](https://github.com/matrix-org/synapse/issues/8166), [\#8168](https://github.com/matrix-org/synapse/issues/8168), [\#8173](https://github.com/matrix-org/synapse/issues/8173), [\#8191](https://github.com/matrix-org/synapse/issues/8191), [\#8192](https://github.com/matrix-org/synapse/issues/8192), [\#8193](https://github.com/matrix-org/synapse/issues/8193), [\#8194](https://github.com/matrix-org/synapse/issues/8194), [\#8195](https://github.com/matrix-org/synapse/issues/8195), [\#8197](https://github.com/matrix-org/synapse/issues/8197), [\#8199](https://github.com/matrix-org/synapse/issues/8199), [\#8200](https://github.com/matrix-org/synapse/issues/8200), [\#8201](https://github.com/matrix-org/synapse/issues/8201), [\#8202](https://github.com/matrix-org/synapse/issues/8202), [\#8207](https://github.com/matrix-org/synapse/issues/8207), [\#8213](https://github.com/matrix-org/synapse/issues/8213), [\#8214](https://github.com/matrix-org/synapse/issues/8214)) -- Remove some unused database functions. ([\#8085](https://github.com/matrix-org/synapse/issues/8085)) -- Add type hints to various parts of the codebase. ([\#8090](https://github.com/matrix-org/synapse/issues/8090), [\#8127](https://github.com/matrix-org/synapse/issues/8127), [\#8187](https://github.com/matrix-org/synapse/issues/8187), [\#8241](https://github.com/matrix-org/synapse/issues/8241), [\#8140](https://github.com/matrix-org/synapse/issues/8140), [\#8183](https://github.com/matrix-org/synapse/issues/8183), [\#8232](https://github.com/matrix-org/synapse/issues/8232), [\#8235](https://github.com/matrix-org/synapse/issues/8235), [\#8237](https://github.com/matrix-org/synapse/issues/8237), [\#8244](https://github.com/matrix-org/synapse/issues/8244)) -- Return the previous stream token if a non-member event is a duplicate. ([\#8093](https://github.com/matrix-org/synapse/issues/8093), [\#8112](https://github.com/matrix-org/synapse/issues/8112)) -- Separate `get_current_token` into two since there are two different use cases for it. ([\#8113](https://github.com/matrix-org/synapse/issues/8113)) -- Remove `ChainedIdGenerator`. ([\#8123](https://github.com/matrix-org/synapse/issues/8123)) -- Reduce the amount of whitespace in JSON stored and sent in responses. ([\#8124](https://github.com/matrix-org/synapse/issues/8124)) -- Update the test federation client to handle streaming responses. ([\#8130](https://github.com/matrix-org/synapse/issues/8130)) -- Micro-optimisations to `get_auth_chain_ids`. ([\#8132](https://github.com/matrix-org/synapse/issues/8132)) -- Refactor `StreamIdGenerator` and `MultiWriterIdGenerator` to have the same interface. ([\#8161](https://github.com/matrix-org/synapse/issues/8161)) -- Add functions to `MultiWriterIdGen` used by events stream. ([\#8164](https://github.com/matrix-org/synapse/issues/8164), [\#8179](https://github.com/matrix-org/synapse/issues/8179)) -- Fix tests that were broken due to the merge of 1.19.1. ([\#8167](https://github.com/matrix-org/synapse/issues/8167)) -- Make `SlavedIdTracker.advance` have the same interface as `MultiWriterIDGenerator`. ([\#8171](https://github.com/matrix-org/synapse/issues/8171)) -- Remove unused `is_guest` parameter from, and add safeguard to, `MessageHandler.get_room_data`. ([\#8174](https://github.com/matrix-org/synapse/issues/8174), [\#8181](https://github.com/matrix-org/synapse/issues/8181)) -- Standardize the mypy configuration. ([\#8175](https://github.com/matrix-org/synapse/issues/8175)) -- Refactor some of `LoginRestServlet`'s helper methods, and move them to `AuthHandler` for easier reuse. ([\#8182](https://github.com/matrix-org/synapse/issues/8182)) -- Fix `wait_for_stream_position` to allow multiple waiters on same stream ID. ([\#8196](https://github.com/matrix-org/synapse/issues/8196)) -- Make `MultiWriterIDGenerator` work for streams that use negative values. ([\#8203](https://github.com/matrix-org/synapse/issues/8203)) -- Refactor queries for device keys and cross-signatures. ([\#8204](https://github.com/matrix-org/synapse/issues/8204), [\#8205](https://github.com/matrix-org/synapse/issues/8205), [\#8222](https://github.com/matrix-org/synapse/issues/8222), [\#8224](https://github.com/matrix-org/synapse/issues/8224), [\#8225](https://github.com/matrix-org/synapse/issues/8225), [\#8231](https://github.com/matrix-org/synapse/issues/8231), [\#8233](https://github.com/matrix-org/synapse/issues/8233), [\#8234](https://github.com/matrix-org/synapse/issues/8234)) -- Fix type hints for functions decorated with `@cached`. ([\#8240](https://github.com/matrix-org/synapse/issues/8240)) -- Remove obsolete `order` field from federation send queues. ([\#8245](https://github.com/matrix-org/synapse/issues/8245)) -- Stop sub-classing from object. ([\#8249](https://github.com/matrix-org/synapse/issues/8249)) -- Add more logging to debug slow startup. ([\#8264](https://github.com/matrix-org/synapse/issues/8264)) -- Do not attempt to upgrade database schema on worker processes. ([\#8266](https://github.com/matrix-org/synapse/issues/8266), [\#8276](https://github.com/matrix-org/synapse/issues/8276)) - - -Synapse 1.19.1 (2020-08-27) -=========================== - -No significant changes. - - -Synapse 1.19.1rc1 (2020-08-25) -============================== - -Bugfixes --------- - -- Fix a bug introduced in v1.19.0 where appservices with ratelimiting disabled would still be ratelimited when joining rooms. ([\#8139](https://github.com/matrix-org/synapse/issues/8139)) -- Fix a bug introduced in v1.19.0 that would cause e.g. profile updates to fail due to incorrect application of rate limits on join requests. ([\#8153](https://github.com/matrix-org/synapse/issues/8153)) - - -Synapse 1.19.0 (2020-08-17) -=========================== - -No significant changes since 1.19.0rc1. - -Removal warning ---------------- - -As outlined in the [previous release](https://github.com/matrix-org/synapse/releases/tag/v1.18.0), -we are no longer publishing Docker images with the `-py3` tag suffix. On top of that, we have also removed the -`latest-py3` tag. Please see -[the announcement in the upgrade notes for 1.18.0](https://github.com/matrix-org/synapse/blob/develop/docs/upgrade.md#upgrading-to-v1180). - - -Synapse 1.19.0rc1 (2020-08-13) -============================== - -Features --------- - -- Add option to allow server admins to join rooms which fail complexity checks. Contributed by @lugino-emeritus. ([\#7902](https://github.com/matrix-org/synapse/issues/7902)) -- Add an option to purge room or not with delete room admin endpoint (`POST /_synapse/admin/v1/rooms//delete`). Contributed by @dklimpel. ([\#7964](https://github.com/matrix-org/synapse/issues/7964)) -- Add rate limiting to users joining rooms. ([\#8008](https://github.com/matrix-org/synapse/issues/8008)) -- Add a `/health` endpoint to every configured HTTP listener that can be used as a health check endpoint by load balancers. ([\#8048](https://github.com/matrix-org/synapse/issues/8048)) -- Allow login to be blocked based on the values of SAML attributes. ([\#8052](https://github.com/matrix-org/synapse/issues/8052)) -- Allow guest access to the `GET /_matrix/client/r0/rooms/{room_id}/members` endpoint, according to MSC2689. Contributed by Awesome Technologies Innovationslabor GmbH. ([\#7314](https://github.com/matrix-org/synapse/issues/7314)) - - -Bugfixes --------- - -- Fix a bug introduced in Synapse v1.7.2 which caused inaccurate membership counts in the room directory. ([\#7977](https://github.com/matrix-org/synapse/issues/7977)) -- Fix a long standing bug: 'Duplicate key value violates unique constraint "event_relations_id"' when message retention is configured. ([\#7978](https://github.com/matrix-org/synapse/issues/7978)) -- Fix "no create event in auth events" when trying to reject invitation after inviter leaves. Bug introduced in Synapse v1.10.0. ([\#7980](https://github.com/matrix-org/synapse/issues/7980)) -- Fix various comments and minor discrepencies in server notices code. ([\#7996](https://github.com/matrix-org/synapse/issues/7996)) -- Fix a long standing bug where HTTP HEAD requests resulted in a 400 error. ([\#7999](https://github.com/matrix-org/synapse/issues/7999)) -- Fix a long-standing bug which caused two copies of some log lines to be written when synctl was used along with a MemoryHandler logger. ([\#8011](https://github.com/matrix-org/synapse/issues/8011), [\#8012](https://github.com/matrix-org/synapse/issues/8012)) - - -Updates to the Docker image ---------------------------- - -- We no longer publish Docker images with the `-py3` tag suffix, as [announced in the upgrade notes](https://github.com/matrix-org/synapse/blob/develop/docs/upgrade.md#upgrading-to-v1180). ([\#8056](https://github.com/matrix-org/synapse/issues/8056)) - - -Improved Documentation ----------------------- - -- Document how to set up a client .well-known file and fix several pieces of outdated documentation. ([\#7899](https://github.com/matrix-org/synapse/issues/7899)) -- Improve workers docs. ([\#7990](https://github.com/matrix-org/synapse/issues/7990), [\#8000](https://github.com/matrix-org/synapse/issues/8000)) -- Fix typo in `docs/workers.md`. ([\#7992](https://github.com/matrix-org/synapse/issues/7992)) -- Add documentation for how to undo a room shutdown. ([\#7998](https://github.com/matrix-org/synapse/issues/7998), [\#8010](https://github.com/matrix-org/synapse/issues/8010)) - - -Internal Changes ----------------- - -- Reduce the amount of whitespace in JSON stored and sent in responses. Contributed by David Vo. ([\#7372](https://github.com/matrix-org/synapse/issues/7372)) -- Switch to the JSON implementation from the standard library and bump the minimum version of the canonicaljson library to 1.2.0. ([\#7936](https://github.com/matrix-org/synapse/issues/7936), [\#7979](https://github.com/matrix-org/synapse/issues/7979)) -- Convert various parts of the codebase to async/await. ([\#7947](https://github.com/matrix-org/synapse/issues/7947), [\#7948](https://github.com/matrix-org/synapse/issues/7948), [\#7949](https://github.com/matrix-org/synapse/issues/7949), [\#7951](https://github.com/matrix-org/synapse/issues/7951), [\#7963](https://github.com/matrix-org/synapse/issues/7963), [\#7973](https://github.com/matrix-org/synapse/issues/7973), [\#7975](https://github.com/matrix-org/synapse/issues/7975), [\#7976](https://github.com/matrix-org/synapse/issues/7976), [\#7981](https://github.com/matrix-org/synapse/issues/7981), [\#7987](https://github.com/matrix-org/synapse/issues/7987), [\#7989](https://github.com/matrix-org/synapse/issues/7989), [\#8003](https://github.com/matrix-org/synapse/issues/8003), [\#8014](https://github.com/matrix-org/synapse/issues/8014), [\#8016](https://github.com/matrix-org/synapse/issues/8016), [\#8027](https://github.com/matrix-org/synapse/issues/8027), [\#8031](https://github.com/matrix-org/synapse/issues/8031), [\#8032](https://github.com/matrix-org/synapse/issues/8032), [\#8035](https://github.com/matrix-org/synapse/issues/8035), [\#8042](https://github.com/matrix-org/synapse/issues/8042), [\#8044](https://github.com/matrix-org/synapse/issues/8044), [\#8045](https://github.com/matrix-org/synapse/issues/8045), [\#8061](https://github.com/matrix-org/synapse/issues/8061), [\#8062](https://github.com/matrix-org/synapse/issues/8062), [\#8063](https://github.com/matrix-org/synapse/issues/8063), [\#8066](https://github.com/matrix-org/synapse/issues/8066), [\#8069](https://github.com/matrix-org/synapse/issues/8069), [\#8070](https://github.com/matrix-org/synapse/issues/8070)) -- Move some database-related log lines from the default logger to the database/transaction loggers. ([\#7952](https://github.com/matrix-org/synapse/issues/7952)) -- Add a script to detect source code files using non-unix line terminators. ([\#7965](https://github.com/matrix-org/synapse/issues/7965), [\#7970](https://github.com/matrix-org/synapse/issues/7970)) -- Log the SAML session ID during creation. ([\#7971](https://github.com/matrix-org/synapse/issues/7971)) -- Implement new experimental push rules for some users. ([\#7997](https://github.com/matrix-org/synapse/issues/7997)) -- Remove redundant and unreliable signature check for v1 Identity Service lookup responses. ([\#8001](https://github.com/matrix-org/synapse/issues/8001)) -- Improve the performance of the register endpoint. ([\#8009](https://github.com/matrix-org/synapse/issues/8009)) -- Reduce less useful output in the newsfragment CI step. Add a link to the changelog section of the contributing guide on error. ([\#8024](https://github.com/matrix-org/synapse/issues/8024)) -- Rename storage layer objects to be more sensible. ([\#8033](https://github.com/matrix-org/synapse/issues/8033)) -- Change the default log config to reduce disk I/O and storage for new servers. ([\#8040](https://github.com/matrix-org/synapse/issues/8040)) -- Add an assertion on `prev_events` in `create_new_client_event`. ([\#8041](https://github.com/matrix-org/synapse/issues/8041)) -- Add a comment to `ServerContextFactory` about the use of `SSLv23_METHOD`. ([\#8043](https://github.com/matrix-org/synapse/issues/8043)) -- Log `OPTIONS` requests at `DEBUG` rather than `INFO` level to reduce amount logged at `INFO`. ([\#8049](https://github.com/matrix-org/synapse/issues/8049)) -- Reduce amount of outbound request logging at `INFO` level. ([\#8050](https://github.com/matrix-org/synapse/issues/8050)) -- It is no longer necessary to explicitly define `filters` in the logging configuration. (Continuing to do so is redundant but harmless.) ([\#8051](https://github.com/matrix-org/synapse/issues/8051)) -- Add and improve type hints. ([\#8058](https://github.com/matrix-org/synapse/issues/8058), [\#8064](https://github.com/matrix-org/synapse/issues/8064), [\#8060](https://github.com/matrix-org/synapse/issues/8060), [\#8067](https://github.com/matrix-org/synapse/issues/8067)) - - -Synapse 1.18.0 (2020-07-30) -=========================== - -Deprecation Warnings --------------------- - -### Docker Tags with `-py3` Suffix - -From 10th August 2020, we will no longer publish Docker images with the `-py3` tag suffix. The images tagged with the `-py3` suffix have been identical to the non-suffixed tags since release 0.99.0, and the suffix is obsolete. - -On 10th August, we will remove the `latest-py3` tag. Existing per-release tags (such as `v1.18.0-py3`) will not be removed, but no new `-py3` tags will be added. - -Scripts relying on the `-py3` suffix will need to be updated. - - -### TCP-based Replication - -When setting up worker processes, we now recommend the use of a Redis server for replication. The old direct TCP connection method is deprecated and will be removed in a future release. See [docs/workers.md](https://github.com/matrix-org/synapse/blob/release-v1.18.0/docs/workers.md) for more details. - - -Improved Documentation ----------------------- - -- Update worker docs with latest enhancements. ([\#7969](https://github.com/matrix-org/synapse/issues/7969)) - - -Synapse 1.18.0rc2 (2020-07-28) -============================== - -Bugfixes --------- - -- Fix an `AssertionError` exception introduced in v1.18.0rc1. ([\#7876](https://github.com/matrix-org/synapse/issues/7876)) -- Fix experimental support for moving typing off master when worker is restarted, which is broken in v1.18.0rc1. ([\#7967](https://github.com/matrix-org/synapse/issues/7967)) - - -Internal Changes ----------------- - -- Further optimise queueing of inbound replication commands. ([\#7876](https://github.com/matrix-org/synapse/issues/7876)) - - -Synapse 1.18.0rc1 (2020-07-27) -============================== - -Features --------- - -- Include room states on invite events that are sent to application services. Contributed by @Sorunome. ([\#6455](https://github.com/matrix-org/synapse/issues/6455)) -- Add delete room admin endpoint (`POST /_synapse/admin/v1/rooms//delete`). Contributed by @dklimpel. ([\#7613](https://github.com/matrix-org/synapse/issues/7613), [\#7953](https://github.com/matrix-org/synapse/issues/7953)) -- Add experimental support for running multiple federation sender processes. ([\#7798](https://github.com/matrix-org/synapse/issues/7798)) -- Add the option to validate the `iss` and `aud` claims for JWT logins. ([\#7827](https://github.com/matrix-org/synapse/issues/7827)) -- Add support for handling registration requests across multiple client reader workers. ([\#7830](https://github.com/matrix-org/synapse/issues/7830)) -- Add an admin API to list the users in a room. Contributed by Awesome Technologies Innovationslabor GmbH. ([\#7842](https://github.com/matrix-org/synapse/issues/7842)) -- Allow email subjects to be customised through Synapse's configuration. ([\#7846](https://github.com/matrix-org/synapse/issues/7846)) -- Add the ability to re-activate an account from the admin API. ([\#7847](https://github.com/matrix-org/synapse/issues/7847), [\#7908](https://github.com/matrix-org/synapse/issues/7908)) -- Add experimental support for running multiple pusher workers. ([\#7855](https://github.com/matrix-org/synapse/issues/7855)) -- Add experimental support for moving typing off master. ([\#7869](https://github.com/matrix-org/synapse/issues/7869), [\#7959](https://github.com/matrix-org/synapse/issues/7959)) -- Report CPU metrics to prometheus for time spent processing replication commands. ([\#7879](https://github.com/matrix-org/synapse/issues/7879)) -- Support oEmbed for media previews. ([\#7920](https://github.com/matrix-org/synapse/issues/7920)) -- Abort federation requests where the client disconnects before the ratelimiter expires. ([\#7930](https://github.com/matrix-org/synapse/issues/7930)) -- Cache responses to `/_matrix/federation/v1/state_ids` to reduce duplicated work. ([\#7931](https://github.com/matrix-org/synapse/issues/7931)) - - -Bugfixes --------- - -- Fix detection of out of sync remote device lists when receiving events from remote users. ([\#7815](https://github.com/matrix-org/synapse/issues/7815)) -- Fix bug where Synapse fails to process an incoming event over federation if the server is missing too much of the event's auth chain. ([\#7817](https://github.com/matrix-org/synapse/issues/7817)) -- Fix a bug causing Synapse to misinterpret the value `off` for `encryption_enabled_by_default_for_room_type` in its configuration file(s) if that value isn't surrounded by quotes. This bug was introduced in v1.16.0. ([\#7822](https://github.com/matrix-org/synapse/issues/7822)) -- Fix bug where we did not always pass in `app_name` or `server_name` to email templates, including e.g. for registration emails. ([\#7829](https://github.com/matrix-org/synapse/issues/7829)) -- Errors which occur while using the non-standard JWT login now return the proper error: `403 Forbidden` with an error code of `M_FORBIDDEN`. ([\#7844](https://github.com/matrix-org/synapse/issues/7844)) -- Fix "AttributeError: 'str' object has no attribute 'get'" error message when applying per-room message retention policies. The bug was introduced in Synapse 1.7.0. ([\#7850](https://github.com/matrix-org/synapse/issues/7850)) -- Fix a bug introduced in Synapse 1.10.0 which could cause a "no create event in auth events" error during room creation. ([\#7854](https://github.com/matrix-org/synapse/issues/7854)) -- Fix a bug which allowed empty rooms to be rejoined over federation. ([\#7859](https://github.com/matrix-org/synapse/issues/7859)) -- Fix 'Unable to find a suitable guest user ID' error when using multiple client_reader workers. ([\#7866](https://github.com/matrix-org/synapse/issues/7866)) -- Fix a long standing bug where the tracing of async functions with opentracing was broken. ([\#7872](https://github.com/matrix-org/synapse/issues/7872), [\#7961](https://github.com/matrix-org/synapse/issues/7961)) -- Fix "TypeError in `synapse.notifier`" exceptions. ([\#7880](https://github.com/matrix-org/synapse/issues/7880)) -- Fix deprecation warning due to invalid escape sequences. ([\#7895](https://github.com/matrix-org/synapse/issues/7895)) - - -Updates to the Docker image ---------------------------- - -- Base docker image on Debian Buster rather than Alpine Linux. Contributed by @maquis196. ([\#7839](https://github.com/matrix-org/synapse/issues/7839)) - - -Improved Documentation ----------------------- - -- Provide instructions on using `register_new_matrix_user` via docker. ([\#7885](https://github.com/matrix-org/synapse/issues/7885)) -- Change the sample config postgres user section to use `synapse_user` instead of `synapse` to align with the documentation. ([\#7889](https://github.com/matrix-org/synapse/issues/7889)) -- Reorder database paragraphs to promote postgres over sqlite. ([\#7933](https://github.com/matrix-org/synapse/issues/7933)) -- Update the dates of ACME v1's end of life in [`ACME.md`](https://github.com/matrix-org/synapse/blob/master/docs/ACME.md). ([\#7934](https://github.com/matrix-org/synapse/issues/7934)) - - -Deprecations and Removals -------------------------- - -- Remove unused `synapse_replication_tcp_resource_invalidate_cache` prometheus metric. ([\#7878](https://github.com/matrix-org/synapse/issues/7878)) -- Remove Ubuntu Eoan from the list of `.deb` packages that we build as it is now end-of-life. Contributed by @gary-kim. ([\#7888](https://github.com/matrix-org/synapse/issues/7888)) - - -Internal Changes ----------------- - -- Switch parts of the codebase from `simplejson` to the standard library `json`. ([\#7802](https://github.com/matrix-org/synapse/issues/7802)) -- Add type hints to the http server code and remove an unused parameter. ([\#7813](https://github.com/matrix-org/synapse/issues/7813)) -- Add type hints to synapse.api.errors module. ([\#7820](https://github.com/matrix-org/synapse/issues/7820)) -- Ensure that calls to `json.dumps` are compatible with the standard library json. ([\#7836](https://github.com/matrix-org/synapse/issues/7836)) -- Remove redundant `retry_on_integrity_error` wrapper for event persistence code. ([\#7848](https://github.com/matrix-org/synapse/issues/7848)) -- Consistently use `db_to_json` to convert from database values to JSON objects. ([\#7849](https://github.com/matrix-org/synapse/issues/7849)) -- Convert various parts of the codebase to async/await. ([\#7851](https://github.com/matrix-org/synapse/issues/7851), [\#7860](https://github.com/matrix-org/synapse/issues/7860), [\#7868](https://github.com/matrix-org/synapse/issues/7868), [\#7871](https://github.com/matrix-org/synapse/issues/7871), [\#7873](https://github.com/matrix-org/synapse/issues/7873), [\#7874](https://github.com/matrix-org/synapse/issues/7874), [\#7884](https://github.com/matrix-org/synapse/issues/7884), [\#7912](https://github.com/matrix-org/synapse/issues/7912), [\#7935](https://github.com/matrix-org/synapse/issues/7935), [\#7939](https://github.com/matrix-org/synapse/issues/7939), [\#7942](https://github.com/matrix-org/synapse/issues/7942), [\#7944](https://github.com/matrix-org/synapse/issues/7944)) -- Add support for handling registration requests across multiple client reader workers. ([\#7853](https://github.com/matrix-org/synapse/issues/7853)) -- Small performance improvement in typing processing. ([\#7856](https://github.com/matrix-org/synapse/issues/7856)) -- The default value of `filter_timeline_limit` was changed from -1 (no limit) to 100. ([\#7858](https://github.com/matrix-org/synapse/issues/7858)) -- Optimise queueing of inbound replication commands. ([\#7861](https://github.com/matrix-org/synapse/issues/7861)) -- Add some type annotations to `HomeServer` and `BaseHandler`. ([\#7870](https://github.com/matrix-org/synapse/issues/7870)) -- Clean up `PreserveLoggingContext`. ([\#7877](https://github.com/matrix-org/synapse/issues/7877)) -- Change "unknown room version" logging from 'error' to 'warning'. ([\#7881](https://github.com/matrix-org/synapse/issues/7881)) -- Stop using `device_max_stream_id` table and just use `device_inbox.stream_id`. ([\#7882](https://github.com/matrix-org/synapse/issues/7882)) -- Return an empty body for OPTIONS requests. ([\#7886](https://github.com/matrix-org/synapse/issues/7886)) -- Fix typo in generated config file. Contributed by @ThiefMaster. ([\#7890](https://github.com/matrix-org/synapse/issues/7890)) -- Import ABC from `collections.abc` for Python 3.10 compatibility. ([\#7892](https://github.com/matrix-org/synapse/issues/7892)) -- Remove unused functions `time_function`, `trace_function`, `get_previous_frames` - and `get_previous_frame` from `synapse.logging.utils` module. ([\#7897](https://github.com/matrix-org/synapse/issues/7897)) -- Lint the `contrib/` directory in CI and linting scripts, add `synctl` to the linting script for consistency with CI. ([\#7914](https://github.com/matrix-org/synapse/issues/7914)) -- Use Element CSS and logo in notification emails when app name is Element. ([\#7919](https://github.com/matrix-org/synapse/issues/7919)) -- Optimisation to /sync handling: skip serializing the response if the client has already disconnected. ([\#7927](https://github.com/matrix-org/synapse/issues/7927)) -- When a client disconnects, don't log it as 'Error processing request'. ([\#7928](https://github.com/matrix-org/synapse/issues/7928)) -- Add debugging to `/sync` response generation (disabled by default). ([\#7929](https://github.com/matrix-org/synapse/issues/7929)) -- Update comments that refer to Deferreds for async functions. ([\#7945](https://github.com/matrix-org/synapse/issues/7945)) -- Simplify error handling in federation handler. ([\#7950](https://github.com/matrix-org/synapse/issues/7950)) - - -Synapse 1.17.0 (2020-07-13) -=========================== - -Synapse 1.17.0 is identical to 1.17.0rc1, with the addition of the fix that was included in 1.16.1. - - -Synapse 1.16.1 (2020-07-10) -=========================== - -In some distributions of Synapse 1.16.0, we incorrectly included a database migration which added a new, unused table. This release removes the redundant table. - -Bugfixes --------- - -- Drop table `local_rejections_stream` which was incorrectly added in Synapse 1.16.0. ([\#7816](https://github.com/matrix-org/synapse/issues/7816), [b1beb3ff5](https://github.com/matrix-org/synapse/commit/b1beb3ff5)) - - -Synapse 1.17.0rc1 (2020-07-09) -============================== - -Bugfixes --------- - -- Fix inconsistent handling of upper and lower case in email addresses when used as identifiers for login, etc. Contributed by @dklimpel. ([\#7021](https://github.com/matrix-org/synapse/issues/7021)) -- Fix "Tried to close a non-active scope!" error messages when opentracing is enabled. ([\#7732](https://github.com/matrix-org/synapse/issues/7732)) -- Fix incorrect error message when database CTYPE was set incorrectly. ([\#7760](https://github.com/matrix-org/synapse/issues/7760)) -- Fix to not ignore `set_tweak` actions in Push Rules that have no `value`, as permitted by the specification. ([\#7766](https://github.com/matrix-org/synapse/issues/7766)) -- Fix synctl to handle empty config files correctly. Contributed by @kotovalexarian. ([\#7779](https://github.com/matrix-org/synapse/issues/7779)) -- Fixes a long standing bug in worker mode where worker information was saved in the devices table instead of the original IP address and user agent. ([\#7797](https://github.com/matrix-org/synapse/issues/7797)) -- Fix 'stuck invites' which happen when we are unable to reject a room invite received over federation. ([\#7804](https://github.com/matrix-org/synapse/issues/7804), [\#7809](https://github.com/matrix-org/synapse/issues/7809), [\#7810](https://github.com/matrix-org/synapse/issues/7810)) - - -Updates to the Docker image ---------------------------- - -- Include libwebp in the Docker file to properly handle webp image uploads. ([\#7791](https://github.com/matrix-org/synapse/issues/7791)) - - -Improved Documentation ----------------------- - -- Improve the documentation of the non-standard JSON web token login type. ([\#7776](https://github.com/matrix-org/synapse/issues/7776)) -- Update doc links for caddy. Contributed by Nicolai Søborg. ([\#7789](https://github.com/matrix-org/synapse/issues/7789)) - - -Internal Changes ----------------- - -- Refactor getting replication updates from database. ([\#7740](https://github.com/matrix-org/synapse/issues/7740)) -- Send push notifications with a high or low priority depending upon whether they may generate user-observable effects. ([\#7765](https://github.com/matrix-org/synapse/issues/7765)) -- Use symbolic names for replication stream names. ([\#7768](https://github.com/matrix-org/synapse/issues/7768)) -- Add early returns to `_check_for_soft_fail`. ([\#7769](https://github.com/matrix-org/synapse/issues/7769)) -- Fix up `synapse.handlers.federation` to pass mypy. ([\#7770](https://github.com/matrix-org/synapse/issues/7770)) -- Convert the appserver handler to async/await. ([\#7775](https://github.com/matrix-org/synapse/issues/7775)) -- Allow to use higher versions of prometheus_client <0.9.0 which are expected to introduce no breaking changes. Contributed by Oliver Kurz. ([\#7780](https://github.com/matrix-org/synapse/issues/7780)) -- Update linting scripts and codebase to be compatible with `isort` v5. ([\#7786](https://github.com/matrix-org/synapse/issues/7786)) -- Stop populating unused table `local_invites`. ([\#7793](https://github.com/matrix-org/synapse/issues/7793)) -- Ensure that strings (not bytes) are passed into JSON serialization. ([\#7799](https://github.com/matrix-org/synapse/issues/7799)) -- Switch from simplejson to the standard library json. ([\#7800](https://github.com/matrix-org/synapse/issues/7800)) -- Add `signing_key` property to `HomeServer` to save code duplication. ([\#7805](https://github.com/matrix-org/synapse/issues/7805)) -- Improve stacktraces from exceptions in background processes. ([\#7808](https://github.com/matrix-org/synapse/issues/7808)) -- Fix various spelling errors in comments and log lines. ([\#7811](https://github.com/matrix-org/synapse/issues/7811)) - - -Synapse 1.16.0 (2020-07-08) -=========================== - -No significant changes since 1.16.0rc2. - -Note that this release deprecates the `m.login.jwt` login method, renaming it -to `org.matrix.login.jwt`, as `m.login.jwt` is not part of the Matrix spec. -Otherwise the behaviour is identical. Synapse will accept both names for now, -but this may change in a future release. - -Synapse 1.16.0rc2 (2020-07-02) -============================== - -Synapse 1.16.0rc2 includes the security fixes released with Synapse 1.15.2. -Please see [below](#synapse-1152-2020-07-02) for more details. - -Improved Documentation ----------------------- - -- Update postgres image in example `docker-compose.yaml` to tag `12-alpine`. ([\#7696](https://github.com/matrix-org/synapse/issues/7696)) - - -Internal Changes ----------------- - -- Add some metrics for inbound and outbound federation latencies: `synapse_federation_server_pdu_process_time` and `synapse_event_processing_lag_by_event`. ([\#7771](https://github.com/matrix-org/synapse/issues/7771)) - - -Synapse 1.15.2 (2020-07-02) -=========================== - -Due to the two security issues highlighted below, server administrators are -encouraged to update Synapse. We are not aware of these vulnerabilities being -exploited in the wild. - -Security advisory ------------------ - -* A malicious homeserver could force Synapse to reset the state in a room to a - small subset of the correct state. This affects all Synapse deployments which - federate with untrusted servers. ([96e9afe6](https://github.com/matrix-org/synapse/commit/96e9afe62500310977dc3cbc99a8d16d3d2fa15c)) -* HTML pages served via Synapse were vulnerable to clickjacking attacks. This - predominantly affects homeservers with single-sign-on enabled, but all server - administrators are encouraged to upgrade. ([ea26e9a9](https://github.com/matrix-org/synapse/commit/ea26e9a98b0541fc886a1cb826a38352b7599dbe)) - - This was reported by [Quentin Gliech](https://sandhose.fr/). - - -Synapse 1.16.0rc1 (2020-07-01) -============================== - -Features --------- - -- Add an option to enable encryption by default for new rooms. ([\#7639](https://github.com/matrix-org/synapse/issues/7639)) -- Add support for running multiple media repository workers. See [docs/workers.md](https://github.com/matrix-org/synapse/blob/release-v1.16.0/docs/workers.md) for instructions. ([\#7706](https://github.com/matrix-org/synapse/issues/7706)) -- Media can now be marked as safe from quarantined. ([\#7718](https://github.com/matrix-org/synapse/issues/7718)) -- Expand the configuration options for auto-join rooms. ([\#7763](https://github.com/matrix-org/synapse/issues/7763)) - - -Bugfixes --------- - -- Remove `user_id` from the response to `GET /_matrix/client/r0/presence/{userId}/status` to match the specification. ([\#7606](https://github.com/matrix-org/synapse/issues/7606)) -- In worker mode, ensure that replicated data has not already been received. ([\#7648](https://github.com/matrix-org/synapse/issues/7648)) -- Fix intermittent exception during startup, introduced in Synapse 1.14.0. ([\#7663](https://github.com/matrix-org/synapse/issues/7663)) -- Include a user-agent for federation and well-known requests. ([\#7677](https://github.com/matrix-org/synapse/issues/7677)) -- Accept the proper field (`phone`) for the `m.id.phone` identifier type. The legacy field of `number` is still accepted as a fallback. Bug introduced in v0.20.0. ([\#7687](https://github.com/matrix-org/synapse/issues/7687)) -- Fix "Starting db txn 'get_completed_ui_auth_stages' from sentinel context" warning. The bug was introduced in 1.13.0. ([\#7688](https://github.com/matrix-org/synapse/issues/7688)) -- Compare the URI and method during user interactive authentication (instead of the URI twice). Bug introduced in 1.13.0. ([\#7689](https://github.com/matrix-org/synapse/issues/7689)) -- Fix a long standing bug where the response to the `GET room_keys/version` endpoint had the incorrect type for the `etag` field. ([\#7691](https://github.com/matrix-org/synapse/issues/7691)) -- Fix logged error during device resync in opentracing. Broke in v1.14.0. ([\#7698](https://github.com/matrix-org/synapse/issues/7698)) -- Do not break push rule evaluation when receiving an event with a non-string body. This is a long-standing bug. ([\#7701](https://github.com/matrix-org/synapse/issues/7701)) -- Fixs a long standing bug which resulted in an exception: "TypeError: argument of type 'ObservableDeferred' is not iterable". ([\#7708](https://github.com/matrix-org/synapse/issues/7708)) -- The `synapse_port_db` script no longer fails when the `ui_auth_sessions` table is non-empty. This bug has existed since v1.13.0. ([\#7711](https://github.com/matrix-org/synapse/issues/7711)) -- Synapse will now fetch media from the proper specified URL (using the r0 prefix instead of the unspecified v1). ([\#7714](https://github.com/matrix-org/synapse/issues/7714)) -- Fix the tables ignored by `synapse_port_db` to be in sync the current database schema. ([\#7717](https://github.com/matrix-org/synapse/issues/7717)) -- Fix missing `Content-Length` on HTTP responses from the metrics handler. ([\#7730](https://github.com/matrix-org/synapse/issues/7730)) -- Fix large state resolutions from stalling Synapse for seconds at a time. ([\#7735](https://github.com/matrix-org/synapse/issues/7735), [\#7746](https://github.com/matrix-org/synapse/issues/7746)) - - -Improved Documentation ----------------------- - -- Spelling correction in sample_config.yaml. ([\#7652](https://github.com/matrix-org/synapse/issues/7652)) -- Added instructions for how to use Keycloak via OpenID Connect to authenticate with Synapse. ([\#7659](https://github.com/matrix-org/synapse/issues/7659)) -- Corrected misspelling of PostgreSQL. ([\#7724](https://github.com/matrix-org/synapse/issues/7724)) - - -Deprecations and Removals -------------------------- - -- Deprecate `m.login.jwt` login method in favour of `org.matrix.login.jwt`, as `m.login.jwt` is not part of the Matrix spec. ([\#7675](https://github.com/matrix-org/synapse/issues/7675)) - - -Internal Changes ----------------- - -- Refactor getting replication updates from database. ([\#7636](https://github.com/matrix-org/synapse/issues/7636)) -- Clean-up the login fallback code. ([\#7657](https://github.com/matrix-org/synapse/issues/7657)) -- Increase the default SAML session expiry time to 15 minutes. ([\#7664](https://github.com/matrix-org/synapse/issues/7664)) -- Convert the device message and pagination handlers to async/await. ([\#7678](https://github.com/matrix-org/synapse/issues/7678)) -- Convert typing handler to async/await. ([\#7679](https://github.com/matrix-org/synapse/issues/7679)) -- Require `parameterized` package version to be at least 0.7.0. ([\#7680](https://github.com/matrix-org/synapse/issues/7680)) -- Refactor handling of `listeners` configuration settings. ([\#7681](https://github.com/matrix-org/synapse/issues/7681)) -- Replace uses of `six.iterkeys`/`iteritems`/`itervalues` with `keys()`/`items()`/`values()`. ([\#7692](https://github.com/matrix-org/synapse/issues/7692)) -- Add support for using `rust-python-jaeger-reporter` library to reduce jaeger tracing overhead. ([\#7697](https://github.com/matrix-org/synapse/issues/7697)) -- Make Tox actions work on Debian 10. ([\#7703](https://github.com/matrix-org/synapse/issues/7703)) -- Replace all remaining uses of `six` with native Python 3 equivalents. Contributed by @ilmari. ([\#7704](https://github.com/matrix-org/synapse/issues/7704)) -- Fix broken link in sample config. ([\#7712](https://github.com/matrix-org/synapse/issues/7712)) -- Speed up state res v2 across large state differences. ([\#7725](https://github.com/matrix-org/synapse/issues/7725)) -- Convert directory handler to async/await. ([\#7727](https://github.com/matrix-org/synapse/issues/7727)) -- Move `flake8` to the end of `scripts-dev/lint.sh` as it takes the longest and could cause the script to exit early. ([\#7738](https://github.com/matrix-org/synapse/issues/7738)) -- Explain the "test" conditional requirement for dependencies is not all of the modules necessary to run the unit tests. ([\#7751](https://github.com/matrix-org/synapse/issues/7751)) -- Add some metrics for inbound and outbound federation latencies: `synapse_federation_server_pdu_process_time` and `synapse_event_processing_lag_by_event`. ([\#7755](https://github.com/matrix-org/synapse/issues/7755)) - - -Synapse 1.15.1 (2020-06-16) -=========================== - -Bugfixes --------- - -- Fix a bug introduced in v1.15.0 that would crash Synapse on start when using certain password auth providers. ([\#7684](https://github.com/matrix-org/synapse/issues/7684)) -- Fix a bug introduced in v1.15.0 which meant that some 3PID management endpoints were not accessible on the correct URL. ([\#7685](https://github.com/matrix-org/synapse/issues/7685)) - - -Synapse 1.15.0 (2020-06-11) -=========================== - -No significant changes. - - -Synapse 1.15.0rc1 (2020-06-09) -============================== - -Features --------- - -- Advertise support for Client-Server API r0.6.0 and remove related unstable feature flags. ([\#6585](https://github.com/matrix-org/synapse/issues/6585)) -- Add an option to disable autojoining rooms for guest accounts. ([\#6637](https://github.com/matrix-org/synapse/issues/6637)) -- For SAML authentication, add the ability to pass email addresses to be added to new users' accounts via SAML attributes. Contributed by Christopher Cooper. ([\#7385](https://github.com/matrix-org/synapse/issues/7385)) -- Add admin APIs to allow server admins to manage users' devices. Contributed by @dklimpel. ([\#7481](https://github.com/matrix-org/synapse/issues/7481)) -- Add support for generating thumbnails for WebP images. Previously, users would see an empty box instead of preview image. Contributed by @WGH-. ([\#7586](https://github.com/matrix-org/synapse/issues/7586)) -- Support the standardized `m.login.sso` user-interactive authentication flow. ([\#7630](https://github.com/matrix-org/synapse/issues/7630)) - - -Bugfixes --------- - -- Allow new users to be registered via the admin API even if the monthly active user limit has been reached. Contributed by @dklimpel. ([\#7263](https://github.com/matrix-org/synapse/issues/7263)) -- Fix email notifications not being enabled for new users when created via the Admin API. ([\#7267](https://github.com/matrix-org/synapse/issues/7267)) -- Fix str placeholders in an instance of `PrepareDatabaseException`. Introduced in Synapse v1.8.0. ([\#7575](https://github.com/matrix-org/synapse/issues/7575)) -- Fix a bug in automatic user creation during first time login with `m.login.jwt`. Regression in v1.6.0. Contributed by @olof. ([\#7585](https://github.com/matrix-org/synapse/issues/7585)) -- Fix a bug causing the cross-signing keys to be ignored when resyncing a device list. ([\#7594](https://github.com/matrix-org/synapse/issues/7594)) -- Fix metrics failing when there is a large number of active background processes. ([\#7597](https://github.com/matrix-org/synapse/issues/7597)) -- Fix bug where returning rooms for a group would fail if it included a room that the server was not in. ([\#7599](https://github.com/matrix-org/synapse/issues/7599)) -- Fix duplicate key violation when persisting read markers. ([\#7607](https://github.com/matrix-org/synapse/issues/7607)) -- Prevent an entire iteration of the device list resync loop from failing if one server responds with a malformed result. ([\#7609](https://github.com/matrix-org/synapse/issues/7609)) -- Fix exceptions when fetching events from a remote host fails. ([\#7622](https://github.com/matrix-org/synapse/issues/7622)) -- Make `synctl restart` start synapse if it wasn't running. ([\#7624](https://github.com/matrix-org/synapse/issues/7624)) -- Pass device information through to the login endpoint when using the login fallback. ([\#7629](https://github.com/matrix-org/synapse/issues/7629)) -- Advertise the `m.login.token` login flow when OpenID Connect is enabled. ([\#7631](https://github.com/matrix-org/synapse/issues/7631)) -- Fix bug in account data replication stream. ([\#7656](https://github.com/matrix-org/synapse/issues/7656)) - - -Improved Documentation ----------------------- - -- Update the OpenBSD installation instructions. ([\#7587](https://github.com/matrix-org/synapse/issues/7587)) -- Advertise Python 3.8 support in `setup.py`. ([\#7602](https://github.com/matrix-org/synapse/issues/7602)) -- Add a link to `#synapse:matrix.org` in the troubleshooting section of the README. ([\#7603](https://github.com/matrix-org/synapse/issues/7603)) -- Clarifications to the admin api documentation. ([\#7647](https://github.com/matrix-org/synapse/issues/7647)) - - -Internal Changes ----------------- - -- Convert the identity handler to async/await. ([\#7561](https://github.com/matrix-org/synapse/issues/7561)) -- Improve query performance for fetching state from a PostgreSQL database. Contributed by @ilmari. ([\#7567](https://github.com/matrix-org/synapse/issues/7567)) -- Speed up processing of federation stream RDATA rows. ([\#7584](https://github.com/matrix-org/synapse/issues/7584)) -- Add comment to systemd example to show postgresql dependency. ([\#7591](https://github.com/matrix-org/synapse/issues/7591)) -- Refactor `Ratelimiter` to limit the amount of expensive config value accesses. ([\#7595](https://github.com/matrix-org/synapse/issues/7595)) -- Convert groups handlers to async/await. ([\#7600](https://github.com/matrix-org/synapse/issues/7600)) -- Clean up exception handling in `SAML2ResponseResource`. ([\#7614](https://github.com/matrix-org/synapse/issues/7614)) -- Check that all asynchronous tasks succeed and general cleanup of `MonthlyActiveUsersTestCase` and `TestMauLimit`. ([\#7619](https://github.com/matrix-org/synapse/issues/7619)) -- Convert `get_user_id_by_threepid` to async/await. ([\#7620](https://github.com/matrix-org/synapse/issues/7620)) -- Switch to upstream `dh-virtualenv` rather than our fork for Debian package builds. ([\#7621](https://github.com/matrix-org/synapse/issues/7621)) -- Update CI scripts to check the number in the newsfile fragment. ([\#7623](https://github.com/matrix-org/synapse/issues/7623)) -- Check if the localpart of a Matrix ID is reserved for guest users earlier in the registration flow, as well as when responding to requests to `/register/available`. ([\#7625](https://github.com/matrix-org/synapse/issues/7625)) -- Minor cleanups to OpenID Connect integration. ([\#7628](https://github.com/matrix-org/synapse/issues/7628)) -- Attempt to fix flaky test: `PhoneHomeStatsTestCase.test_performance_100`. ([\#7634](https://github.com/matrix-org/synapse/issues/7634)) -- Fix typos of `m.olm.curve25519-aes-sha2` and `m.megolm.v1.aes-sha2` in comments, test files. ([\#7637](https://github.com/matrix-org/synapse/issues/7637)) -- Convert user directory, state deltas, and stats handlers to async/await. ([\#7640](https://github.com/matrix-org/synapse/issues/7640)) -- Remove some unused constants. ([\#7644](https://github.com/matrix-org/synapse/issues/7644)) -- Fix type information on `assert_*_is_admin` methods. ([\#7645](https://github.com/matrix-org/synapse/issues/7645)) -- Convert registration handler to async/await. ([\#7649](https://github.com/matrix-org/synapse/issues/7649)) - - -Synapse 1.14.0 (2020-05-28) -=========================== - -No significant changes. - - -Synapse 1.14.0rc2 (2020-05-27) -============================== - -Bugfixes --------- - -- Fix cache config to not apply cache factor to event cache. Regression in v1.14.0rc1. ([\#7578](https://github.com/matrix-org/synapse/issues/7578)) -- Fix bug where `ReplicationStreamer` was not always started when replication was enabled. Bug introduced in v1.14.0rc1. ([\#7579](https://github.com/matrix-org/synapse/issues/7579)) -- Fix specifying individual cache factors for caches with special characters in their name. Regression in v1.14.0rc1. ([\#7580](https://github.com/matrix-org/synapse/issues/7580)) - - -Improved Documentation ----------------------- - -- Fix the OIDC `client_auth_method` value in the sample config. ([\#7581](https://github.com/matrix-org/synapse/issues/7581)) - - -Synapse 1.14.0rc1 (2020-05-26) -============================== - -Features --------- - -- Synapse's cache factor can now be configured in `homeserver.yaml` by the `caches.global_factor` setting. Additionally, `caches.per_cache_factors` controls the cache factors for individual caches. ([\#6391](https://github.com/matrix-org/synapse/issues/6391)) -- Add OpenID Connect login/registration support. Contributed by Quentin Gliech, on behalf of [les Connecteurs](https://connecteu.rs). ([\#7256](https://github.com/matrix-org/synapse/issues/7256), [\#7457](https://github.com/matrix-org/synapse/issues/7457)) -- Add room details admin endpoint. Contributed by Awesome Technologies Innovationslabor GmbH. ([\#7317](https://github.com/matrix-org/synapse/issues/7317)) -- Allow for using more than one spam checker module at once. ([\#7435](https://github.com/matrix-org/synapse/issues/7435)) -- Add additional authentication checks for `m.room.power_levels` event per [MSC2209](https://github.com/matrix-org/matrix-doc/pull/2209). ([\#7502](https://github.com/matrix-org/synapse/issues/7502)) -- Implement room version 6 per [MSC2240](https://github.com/matrix-org/matrix-doc/pull/2240). ([\#7506](https://github.com/matrix-org/synapse/issues/7506)) -- Add highly experimental option to move event persistence off master. ([\#7281](https://github.com/matrix-org/synapse/issues/7281), [\#7374](https://github.com/matrix-org/synapse/issues/7374), [\#7436](https://github.com/matrix-org/synapse/issues/7436), [\#7440](https://github.com/matrix-org/synapse/issues/7440), [\#7475](https://github.com/matrix-org/synapse/issues/7475), [\#7490](https://github.com/matrix-org/synapse/issues/7490), [\#7491](https://github.com/matrix-org/synapse/issues/7491), [\#7492](https://github.com/matrix-org/synapse/issues/7492), [\#7493](https://github.com/matrix-org/synapse/issues/7493), [\#7495](https://github.com/matrix-org/synapse/issues/7495), [\#7515](https://github.com/matrix-org/synapse/issues/7515), [\#7516](https://github.com/matrix-org/synapse/issues/7516), [\#7517](https://github.com/matrix-org/synapse/issues/7517), [\#7542](https://github.com/matrix-org/synapse/issues/7542)) - - -Bugfixes --------- - -- Fix a bug where event updates might not be sent over replication to worker processes after the stream falls behind. ([\#7384](https://github.com/matrix-org/synapse/issues/7384)) -- Allow expired user accounts to log out their device sessions. ([\#7443](https://github.com/matrix-org/synapse/issues/7443)) -- Fix a bug that would cause Synapse not to resync out-of-sync device lists. ([\#7453](https://github.com/matrix-org/synapse/issues/7453)) -- Prevent rooms with 0 members or with invalid version strings from breaking group queries. ([\#7465](https://github.com/matrix-org/synapse/issues/7465)) -- Workaround for an upstream Twisted bug that caused Synapse to become unresponsive after startup. ([\#7473](https://github.com/matrix-org/synapse/issues/7473)) -- Fix Redis reconnection logic that can result in missed updates over replication if master reconnects to Redis without restarting. ([\#7482](https://github.com/matrix-org/synapse/issues/7482)) -- When sending `m.room.member` events, omit `displayname` and `avatar_url` if they aren't set instead of setting them to `null`. Contributed by Aaron Raimist. ([\#7497](https://github.com/matrix-org/synapse/issues/7497)) -- Fix incorrect `method` label on `synapse_http_matrixfederationclient_{requests,responses}` prometheus metrics. ([\#7503](https://github.com/matrix-org/synapse/issues/7503)) -- Ignore incoming presence events from other homeservers if presence is disabled locally. ([\#7508](https://github.com/matrix-org/synapse/issues/7508)) -- Fix a long-standing bug that broke the update remote profile background process. ([\#7511](https://github.com/matrix-org/synapse/issues/7511)) -- Hash passwords as early as possible during password reset. ([\#7538](https://github.com/matrix-org/synapse/issues/7538)) -- Fix bug where a local user leaving a room could fail under rare circumstances. ([\#7548](https://github.com/matrix-org/synapse/issues/7548)) -- Fix "Missing RelayState parameter" error when using user interactive authentication with SAML for some SAML providers. ([\#7552](https://github.com/matrix-org/synapse/issues/7552)) -- Fix exception `'GenericWorkerReplicationHandler' object has no attribute 'send_federation_ack'`, introduced in v1.13.0. ([\#7564](https://github.com/matrix-org/synapse/issues/7564)) -- `synctl` now warns if it was unable to stop Synapse and will not attempt to start Synapse if nothing was stopped. Contributed by Romain Bouyé. ([\#6598](https://github.com/matrix-org/synapse/issues/6598)) - - -Updates to the Docker image ---------------------------- - -- Update docker runtime image to Alpine v3.11. Contributed by @Starbix. ([\#7398](https://github.com/matrix-org/synapse/issues/7398)) - - -Improved Documentation ----------------------- - -- Update information about mapping providers for SAML and OpenID. ([\#7458](https://github.com/matrix-org/synapse/issues/7458)) -- Add additional reverse proxy example for Caddy v2. Contributed by Jeff Peeler. ([\#7463](https://github.com/matrix-org/synapse/issues/7463)) -- Fix copy-paste error in `ServerNoticesConfig` docstring. Contributed by @ptman. ([\#7477](https://github.com/matrix-org/synapse/issues/7477)) -- Improve the formatting of `reverse_proxy.md`. ([\#7514](https://github.com/matrix-org/synapse/issues/7514)) -- Change the systemd worker service to check that the worker config file exists instead of silently failing. Contributed by David Vo. ([\#7528](https://github.com/matrix-org/synapse/issues/7528)) -- Minor clarifications to the TURN docs. ([\#7533](https://github.com/matrix-org/synapse/issues/7533)) - - -Internal Changes ----------------- - -- Add typing annotations in `synapse.federation`. ([\#7382](https://github.com/matrix-org/synapse/issues/7382)) -- Convert the room handler to async/await. ([\#7396](https://github.com/matrix-org/synapse/issues/7396)) -- Improve performance of `get_e2e_cross_signing_key`. ([\#7428](https://github.com/matrix-org/synapse/issues/7428)) -- Improve performance of `mark_as_sent_devices_by_remote`. ([\#7429](https://github.com/matrix-org/synapse/issues/7429), [\#7562](https://github.com/matrix-org/synapse/issues/7562)) -- Add type hints to the SAML handler. ([\#7445](https://github.com/matrix-org/synapse/issues/7445)) -- Remove storage method `get_hosts_in_room` that is no longer called anywhere. ([\#7448](https://github.com/matrix-org/synapse/issues/7448)) -- Fix some typos in the `notice_expiry` templates. ([\#7449](https://github.com/matrix-org/synapse/issues/7449)) -- Convert the federation handler to async/await. ([\#7459](https://github.com/matrix-org/synapse/issues/7459)) -- Convert the search handler to async/await. ([\#7460](https://github.com/matrix-org/synapse/issues/7460)) -- Add type hints to `synapse.event_auth`. ([\#7505](https://github.com/matrix-org/synapse/issues/7505)) -- Convert the room member handler to async/await. ([\#7507](https://github.com/matrix-org/synapse/issues/7507)) -- Add type hints to room member handler. ([\#7513](https://github.com/matrix-org/synapse/issues/7513)) -- Fix typing annotations in `tests.replication`. ([\#7518](https://github.com/matrix-org/synapse/issues/7518)) -- Remove some redundant Python 2 support code. ([\#7519](https://github.com/matrix-org/synapse/issues/7519)) -- All endpoints now respond with a 200 OK for `OPTIONS` requests. ([\#7534](https://github.com/matrix-org/synapse/issues/7534), [\#7560](https://github.com/matrix-org/synapse/issues/7560)) -- Synapse now exports [detailed allocator statistics](https://doc.pypy.org/en/latest/gc_info.html#gc-get-stats) and basic GC timings as Prometheus metrics (`pypy_gc_time_seconds_total` and `pypy_memory_bytes`) when run under PyPy. Contributed by Ivan Shapovalov. ([\#7536](https://github.com/matrix-org/synapse/issues/7536)) -- Remove Ubuntu Cosmic and Disco from the list of distributions which we provide `.deb`s for, due to end-of-life. ([\#7539](https://github.com/matrix-org/synapse/issues/7539)) -- Make worker processes return a stubbed-out response to `GET /presence` requests. ([\#7545](https://github.com/matrix-org/synapse/issues/7545)) -- Optimise some references to `hs.config`. ([\#7546](https://github.com/matrix-org/synapse/issues/7546)) -- On upgrade room only send canonical alias once. ([\#7547](https://github.com/matrix-org/synapse/issues/7547)) -- Fix some indentation inconsistencies in the sample config. ([\#7550](https://github.com/matrix-org/synapse/issues/7550)) -- Include `synapse.http.site` in type checking. ([\#7553](https://github.com/matrix-org/synapse/issues/7553)) -- Fix some test code to not mangle stacktraces, to make it easier to debug errors. ([\#7554](https://github.com/matrix-org/synapse/issues/7554)) -- Refresh apt cache when building `dh_virtualenv` docker image. ([\#7555](https://github.com/matrix-org/synapse/issues/7555)) -- Stop logging some expected HTTP request errors as exceptions. ([\#7556](https://github.com/matrix-org/synapse/issues/7556), [\#7563](https://github.com/matrix-org/synapse/issues/7563)) -- Convert sending mail to async/await. ([\#7557](https://github.com/matrix-org/synapse/issues/7557)) -- Simplify `reap_monthly_active_users`. ([\#7558](https://github.com/matrix-org/synapse/issues/7558)) - - -Synapse 1.13.0 (2020-05-19) -=========================== - -This release brings some potential changes necessary for certain -configurations of Synapse: - -* If your Synapse is configured to use SSO and have a custom - `sso_redirect_confirm_template_dir` configuration option set, you will need - to duplicate the new `sso_auth_confirm.html`, `sso_auth_success.html` and - `sso_account_deactivated.html` templates into that directory. -* Synapse plugins using the `complete_sso_login` method of - `synapse.module_api.ModuleApi` should instead switch to the async/await - version, `complete_sso_login_async`, which includes additional checks. The - former version is now deprecated. -* A bug was introduced in Synapse 1.4.0 which could cause the room directory - to be incomplete or empty if Synapse was upgraded directly from v1.2.1 or - earlier, to versions between v1.4.0 and v1.12.x. - -Please review the [upgrade notes](docs/upgrade.md) for more details on these changes -and for general upgrade guidance. - - -Notice of change to the default `git` branch for Synapse --------------------------------------------------------- - -With the release of Synapse 1.13.0, the default `git` branch for Synapse has -changed to `develop`, which is the development tip. This is more consistent with -common practice and modern `git` usage. - -The `master` branch, which tracks the latest release, is still available. It is -recommended that developers and distributors who have scripts which run builds -using the default branch of Synapse should therefore consider pinning their -scripts to `master`. - - -Internal Changes ----------------- - -- Update the version of dh-virtualenv we use to build debs, and add focal to the list of target distributions. ([\#7526](https://github.com/matrix-org/synapse/issues/7526)) - - -Synapse 1.13.0rc3 (2020-05-18) -============================== - -Bugfixes --------- - -- Hash passwords as early as possible during registration. ([\#7523](https://github.com/matrix-org/synapse/issues/7523)) - - -Synapse 1.13.0rc2 (2020-05-14) -============================== - -Bugfixes --------- - -- Fix a long-standing bug which could cause messages not to be sent over federation, when state events with state keys matching user IDs (such as custom user statuses) were received. ([\#7376](https://github.com/matrix-org/synapse/issues/7376)) -- Restore compatibility with non-compliant clients during the user interactive authentication process, fixing a problem introduced in v1.13.0rc1. ([\#7483](https://github.com/matrix-org/synapse/issues/7483)) - -Internal Changes ----------------- - -- Fix linting errors in new version of Flake8. ([\#7470](https://github.com/matrix-org/synapse/issues/7470)) - - -Synapse 1.13.0rc1 (2020-05-11) -============================== - -Features --------- - -- Extend the `web_client_location` option to accept an absolute URL to use as a redirect. Adds a warning when running the web client on the same hostname as homeserver. Contributed by Martin Milata. ([\#7006](https://github.com/matrix-org/synapse/issues/7006)) -- Set `Referrer-Policy` header to `no-referrer` on media downloads. ([\#7009](https://github.com/matrix-org/synapse/issues/7009)) -- Add support for running replication over Redis when using workers. ([\#7040](https://github.com/matrix-org/synapse/issues/7040), [\#7325](https://github.com/matrix-org/synapse/issues/7325), [\#7352](https://github.com/matrix-org/synapse/issues/7352), [\#7401](https://github.com/matrix-org/synapse/issues/7401), [\#7427](https://github.com/matrix-org/synapse/issues/7427), [\#7439](https://github.com/matrix-org/synapse/issues/7439), [\#7446](https://github.com/matrix-org/synapse/issues/7446), [\#7450](https://github.com/matrix-org/synapse/issues/7450), [\#7454](https://github.com/matrix-org/synapse/issues/7454)) -- Admin API `POST /_synapse/admin/v1/join/` to join users to a room like `auto_join_rooms` for creation of users. ([\#7051](https://github.com/matrix-org/synapse/issues/7051)) -- Add options to prevent users from changing their profile or associated 3PIDs. ([\#7096](https://github.com/matrix-org/synapse/issues/7096)) -- Support SSO in the user interactive authentication workflow. ([\#7102](https://github.com/matrix-org/synapse/issues/7102), [\#7186](https://github.com/matrix-org/synapse/issues/7186), [\#7279](https://github.com/matrix-org/synapse/issues/7279), [\#7343](https://github.com/matrix-org/synapse/issues/7343)) -- Allow server admins to define and enforce a password policy ([MSC2000](https://github.com/matrix-org/matrix-doc/issues/2000)). ([\#7118](https://github.com/matrix-org/synapse/issues/7118)) -- Improve the support for SSO authentication on the login fallback page. ([\#7152](https://github.com/matrix-org/synapse/issues/7152), [\#7235](https://github.com/matrix-org/synapse/issues/7235)) -- Always whitelist the login fallback in the SSO configuration if `public_baseurl` is set. ([\#7153](https://github.com/matrix-org/synapse/issues/7153)) -- Admin users are no longer required to be in a room to create an alias for it. ([\#7191](https://github.com/matrix-org/synapse/issues/7191)) -- Require admin privileges to enable room encryption by default. This does not affect existing rooms. ([\#7230](https://github.com/matrix-org/synapse/issues/7230)) -- Add a config option for specifying the value of the Accept-Language HTTP header when generating URL previews. ([\#7265](https://github.com/matrix-org/synapse/issues/7265)) -- Allow `/requestToken` endpoints to hide the existence (or lack thereof) of 3PID associations on the homeserver. ([\#7315](https://github.com/matrix-org/synapse/issues/7315)) -- Add a configuration setting to tweak the threshold for dummy events. ([\#7422](https://github.com/matrix-org/synapse/issues/7422)) - - -Bugfixes --------- - -- Don't attempt to use an invalid sqlite config if no database configuration is provided. Contributed by @nekatak. ([\#6573](https://github.com/matrix-org/synapse/issues/6573)) -- Fix single-sign on with CAS systems: pass the same service URL when requesting the CAS ticket and when calling the `proxyValidate` URL. Contributed by @Naugrimm. ([\#6634](https://github.com/matrix-org/synapse/issues/6634)) -- Fix missing field `default` when fetching user-defined push rules. ([\#6639](https://github.com/matrix-org/synapse/issues/6639)) -- Improve error responses when accessing remote public room lists. ([\#6899](https://github.com/matrix-org/synapse/issues/6899), [\#7368](https://github.com/matrix-org/synapse/issues/7368)) -- Transfer alias mappings on room upgrade. ([\#6946](https://github.com/matrix-org/synapse/issues/6946)) -- Ensure that a user interactive authentication session is tied to a single request. ([\#7068](https://github.com/matrix-org/synapse/issues/7068), [\#7455](https://github.com/matrix-org/synapse/issues/7455)) -- Fix a bug in the federation API which could cause occasional "Failed to get PDU" errors. ([\#7089](https://github.com/matrix-org/synapse/issues/7089)) -- Return the proper error (`M_BAD_ALIAS`) when a non-existant canonical alias is provided. ([\#7109](https://github.com/matrix-org/synapse/issues/7109)) -- Fix a bug which meant that groups updates were not correctly replicated between workers. ([\#7117](https://github.com/matrix-org/synapse/issues/7117)) -- Fix starting workers when federation sending not split out. ([\#7133](https://github.com/matrix-org/synapse/issues/7133)) -- Ensure `is_verified` is a boolean in responses to `GET /_matrix/client/r0/room_keys/keys`. Also warn the user if they forgot the `version` query param. ([\#7150](https://github.com/matrix-org/synapse/issues/7150)) -- Fix error page being shown when a custom SAML handler attempted to redirect when processing an auth response. ([\#7151](https://github.com/matrix-org/synapse/issues/7151)) -- Avoid importing `sqlite3` when using the postgres backend. Contributed by David Vo. ([\#7155](https://github.com/matrix-org/synapse/issues/7155)) -- Fix excessive CPU usage by `prune_old_outbound_device_pokes` job. ([\#7159](https://github.com/matrix-org/synapse/issues/7159)) -- Fix a bug which could cause outbound federation traffic to stop working if a client uploaded an incorrect e2e device signature. ([\#7177](https://github.com/matrix-org/synapse/issues/7177)) -- Fix a bug which could cause incorrect 'cyclic dependency' error. ([\#7178](https://github.com/matrix-org/synapse/issues/7178)) -- Fix a bug that could cause a user to be invited to a server notices (aka System Alerts) room without any notice being sent. ([\#7199](https://github.com/matrix-org/synapse/issues/7199)) -- Fix some worker-mode replication handling not being correctly recorded in CPU usage stats. ([\#7203](https://github.com/matrix-org/synapse/issues/7203)) -- Do not allow a deactivated user to login via SSO. ([\#7240](https://github.com/matrix-org/synapse/issues/7240), [\#7259](https://github.com/matrix-org/synapse/issues/7259)) -- Fix --help command-line argument. ([\#7249](https://github.com/matrix-org/synapse/issues/7249)) -- Fix room publish permissions not being checked on room creation. ([\#7260](https://github.com/matrix-org/synapse/issues/7260)) -- Reject unknown session IDs during user interactive authentication instead of silently creating a new session. ([\#7268](https://github.com/matrix-org/synapse/issues/7268)) -- Fix a SQL query introduced in Synapse 1.12.0 which could cause large amounts of logging to the postgres slow-query log. ([\#7274](https://github.com/matrix-org/synapse/issues/7274)) -- Persist user interactive authentication sessions across workers and Synapse restarts. ([\#7302](https://github.com/matrix-org/synapse/issues/7302)) -- Fixed backwards compatibility logic of the first value of `trusted_third_party_id_servers` being used for `account_threepid_delegates.email`, which occurs when the former, deprecated option is set and the latter is not. ([\#7316](https://github.com/matrix-org/synapse/issues/7316)) -- Fix a bug where event updates might not be sent over replication to worker processes after the stream falls behind. ([\#7337](https://github.com/matrix-org/synapse/issues/7337), [\#7358](https://github.com/matrix-org/synapse/issues/7358)) -- Fix bad error handling that would cause Synapse to crash if it's provided with a YAML configuration file that's either empty or doesn't parse into a key-value map. ([\#7341](https://github.com/matrix-org/synapse/issues/7341)) -- Fix incorrect metrics reporting for `renew_attestations` background task. ([\#7344](https://github.com/matrix-org/synapse/issues/7344)) -- Prevent non-federating rooms from appearing in responses to federated `POST /publicRoom` requests when a filter was included. ([\#7367](https://github.com/matrix-org/synapse/issues/7367)) -- Fix a bug which would cause the room durectory to be incorrectly populated if Synapse was upgraded directly from v1.2.1 or earlier to v1.4.0 or later. Note that this fix does not apply retrospectively; see the [upgrade notes](docs/upgrade.md#upgrading-to-v1130) for more information. ([\#7387](https://github.com/matrix-org/synapse/issues/7387)) -- Fix bug in `EventContext.deserialize`. ([\#7393](https://github.com/matrix-org/synapse/issues/7393)) - - -Improved Documentation ----------------------- - -- Update Debian installation instructions to recommend installing the `virtualenv` package instead of `python3-virtualenv`. ([\#6892](https://github.com/matrix-org/synapse/issues/6892)) -- Improve the documentation for database configuration. ([\#6988](https://github.com/matrix-org/synapse/issues/6988)) -- Improve the documentation of application service configuration files. ([\#7091](https://github.com/matrix-org/synapse/issues/7091)) -- Update pre-built package name for FreeBSD. ([\#7107](https://github.com/matrix-org/synapse/issues/7107)) -- Update postgres docs with login troubleshooting information. ([\#7119](https://github.com/matrix-org/synapse/issues/7119)) -- Clean up INSTALL.md a bit. ([\#7141](https://github.com/matrix-org/synapse/issues/7141)) -- Add documentation for running a local CAS server for testing. ([\#7147](https://github.com/matrix-org/synapse/issues/7147)) -- Improve README.md by being explicit about public IP recommendation for TURN relaying. ([\#7167](https://github.com/matrix-org/synapse/issues/7167)) -- Fix a small typo in the `metrics_flags` config option. ([\#7171](https://github.com/matrix-org/synapse/issues/7171)) -- Update the contributed documentation on managing synapse workers with systemd, and bring it into the core distribution. ([\#7234](https://github.com/matrix-org/synapse/issues/7234)) -- Add documentation to the `password_providers` config option. Add known password provider implementations to docs. ([\#7238](https://github.com/matrix-org/synapse/issues/7238), [\#7248](https://github.com/matrix-org/synapse/issues/7248)) -- Modify suggested nginx reverse proxy configuration to match Synapse's default file upload size. Contributed by @ProCycleDev. ([\#7251](https://github.com/matrix-org/synapse/issues/7251)) -- Documentation of media_storage_providers options updated to avoid misunderstandings. Contributed by Tristan Lins. ([\#7272](https://github.com/matrix-org/synapse/issues/7272)) -- Add documentation on monitoring workers with Prometheus. ([\#7357](https://github.com/matrix-org/synapse/issues/7357)) -- Clarify endpoint usage in the users admin api documentation. ([\#7361](https://github.com/matrix-org/synapse/issues/7361)) - - -Deprecations and Removals -------------------------- - -- Remove nonfunctional `captcha_bypass_secret` option from `homeserver.yaml`. ([\#7137](https://github.com/matrix-org/synapse/issues/7137)) - - -Internal Changes ----------------- - -- Add benchmarks for LruCache. ([\#6446](https://github.com/matrix-org/synapse/issues/6446)) -- Return total number of users and profile attributes in admin users endpoint. Contributed by Awesome Technologies Innovationslabor GmbH. ([\#6881](https://github.com/matrix-org/synapse/issues/6881)) -- Change device list streams to have one row per ID. ([\#7010](https://github.com/matrix-org/synapse/issues/7010)) -- Remove concept of a non-limited stream. ([\#7011](https://github.com/matrix-org/synapse/issues/7011)) -- Move catchup of replication streams logic to worker. ([\#7024](https://github.com/matrix-org/synapse/issues/7024), [\#7195](https://github.com/matrix-org/synapse/issues/7195), [\#7226](https://github.com/matrix-org/synapse/issues/7226), [\#7239](https://github.com/matrix-org/synapse/issues/7239), [\#7286](https://github.com/matrix-org/synapse/issues/7286), [\#7290](https://github.com/matrix-org/synapse/issues/7290), [\#7318](https://github.com/matrix-org/synapse/issues/7318), [\#7326](https://github.com/matrix-org/synapse/issues/7326), [\#7378](https://github.com/matrix-org/synapse/issues/7378), [\#7421](https://github.com/matrix-org/synapse/issues/7421)) -- Convert some of synapse.rest.media to async/await. ([\#7110](https://github.com/matrix-org/synapse/issues/7110), [\#7184](https://github.com/matrix-org/synapse/issues/7184), [\#7241](https://github.com/matrix-org/synapse/issues/7241)) -- De-duplicate / remove unused REST code for login and auth. ([\#7115](https://github.com/matrix-org/synapse/issues/7115)) -- Convert `*StreamRow` classes to inner classes. ([\#7116](https://github.com/matrix-org/synapse/issues/7116)) -- Clean up some LoggingContext code. ([\#7120](https://github.com/matrix-org/synapse/issues/7120), [\#7181](https://github.com/matrix-org/synapse/issues/7181), [\#7183](https://github.com/matrix-org/synapse/issues/7183), [\#7408](https://github.com/matrix-org/synapse/issues/7408), [\#7426](https://github.com/matrix-org/synapse/issues/7426)) -- Add explicit `instance_id` for USER_SYNC commands and remove implicit `conn_id` usage. ([\#7128](https://github.com/matrix-org/synapse/issues/7128)) -- Refactored the CAS authentication logic to a separate class. ([\#7136](https://github.com/matrix-org/synapse/issues/7136)) -- Run replication streamers on workers. ([\#7146](https://github.com/matrix-org/synapse/issues/7146)) -- Add tests for outbound device pokes. ([\#7157](https://github.com/matrix-org/synapse/issues/7157)) -- Fix device list update stream ids going backward. ([\#7158](https://github.com/matrix-org/synapse/issues/7158)) -- Use `stream.current_token()` and remove `stream_positions()`. ([\#7172](https://github.com/matrix-org/synapse/issues/7172)) -- Move client command handling out of TCP protocol. ([\#7185](https://github.com/matrix-org/synapse/issues/7185)) -- Move server command handling out of TCP protocol. ([\#7187](https://github.com/matrix-org/synapse/issues/7187)) -- Fix consistency of HTTP status codes reported in log lines. ([\#7188](https://github.com/matrix-org/synapse/issues/7188)) -- Only run one background database update at a time. ([\#7190](https://github.com/matrix-org/synapse/issues/7190)) -- Remove sent outbound device list pokes from the database. ([\#7192](https://github.com/matrix-org/synapse/issues/7192)) -- Add a background database update job to clear out duplicate `device_lists_outbound_pokes`. ([\#7193](https://github.com/matrix-org/synapse/issues/7193)) -- Remove some extraneous debugging log lines. ([\#7207](https://github.com/matrix-org/synapse/issues/7207)) -- Add explicit Python build tooling as dependencies for the snapcraft build. ([\#7213](https://github.com/matrix-org/synapse/issues/7213)) -- Add typing information to federation server code. ([\#7219](https://github.com/matrix-org/synapse/issues/7219)) -- Extend room admin api (`GET /_synapse/admin/v1/rooms`) with additional attributes. ([\#7225](https://github.com/matrix-org/synapse/issues/7225)) -- Unblacklist '/upgrade creates a new room' sytest for workers. ([\#7228](https://github.com/matrix-org/synapse/issues/7228)) -- Remove redundant checks on `daemonize` from synctl. ([\#7233](https://github.com/matrix-org/synapse/issues/7233)) -- Upgrade jQuery to v3.4.1 on fallback login/registration pages. ([\#7236](https://github.com/matrix-org/synapse/issues/7236)) -- Change log line that told user to implement onLogin/onRegister fallback js functions to a warning, instead of an info, so it's more visible. ([\#7237](https://github.com/matrix-org/synapse/issues/7237)) -- Correct the parameters of a test fixture. Contributed by Isaiah Singletary. ([\#7243](https://github.com/matrix-org/synapse/issues/7243)) -- Convert auth handler to async/await. ([\#7261](https://github.com/matrix-org/synapse/issues/7261)) -- Add some unit tests for replication. ([\#7278](https://github.com/matrix-org/synapse/issues/7278)) -- Improve typing annotations in `synapse.replication.tcp.streams.Stream`. ([\#7291](https://github.com/matrix-org/synapse/issues/7291)) -- Reduce log verbosity of url cache cleanup tasks. ([\#7295](https://github.com/matrix-org/synapse/issues/7295)) -- Fix sample SAML Service Provider configuration. Contributed by @frcl. ([\#7300](https://github.com/matrix-org/synapse/issues/7300)) -- Fix StreamChangeCache to work with multiple entities changing on the same stream id. ([\#7303](https://github.com/matrix-org/synapse/issues/7303)) -- Fix an incorrect import in IdentityHandler. ([\#7319](https://github.com/matrix-org/synapse/issues/7319)) -- Reduce logging verbosity for successful federation requests. ([\#7321](https://github.com/matrix-org/synapse/issues/7321)) -- Convert some federation handler code to async/await. ([\#7338](https://github.com/matrix-org/synapse/issues/7338)) -- Fix collation for postgres for unit tests. ([\#7359](https://github.com/matrix-org/synapse/issues/7359)) -- Convert RegistrationWorkerStore.is_server_admin and dependent code to async/await. ([\#7363](https://github.com/matrix-org/synapse/issues/7363)) -- Add an `instance_name` to `RDATA` and `POSITION` replication commands. ([\#7364](https://github.com/matrix-org/synapse/issues/7364)) -- Thread through instance name to replication client. ([\#7369](https://github.com/matrix-org/synapse/issues/7369)) -- Convert synapse.server_notices to async/await. ([\#7394](https://github.com/matrix-org/synapse/issues/7394)) -- Convert synapse.notifier to async/await. ([\#7395](https://github.com/matrix-org/synapse/issues/7395)) -- Fix issues with the Python package manifest. ([\#7404](https://github.com/matrix-org/synapse/issues/7404)) -- Prevent methods in `synapse.handlers.auth` from polling the homeserver config every request. ([\#7420](https://github.com/matrix-org/synapse/issues/7420)) -- Speed up fetching device lists changes when handling `/sync` requests. ([\#7423](https://github.com/matrix-org/synapse/issues/7423)) -- Run group attestation renewal in series rather than parallel for performance. ([\#7442](https://github.com/matrix-org/synapse/issues/7442)) - - -Synapse 1.12.4 (2020-04-23) -=========================== - -No significant changes. - - -Synapse 1.12.4rc1 (2020-04-22) -============================== - -Features --------- - -- Always send users their own device updates. ([\#7160](https://github.com/matrix-org/synapse/issues/7160)) -- Add support for handling GET requests for `account_data` on a worker. ([\#7311](https://github.com/matrix-org/synapse/issues/7311)) - - -Bugfixes --------- - -- Fix a bug that prevented cross-signing with users on worker-mode synapses. ([\#7255](https://github.com/matrix-org/synapse/issues/7255)) -- Do not treat display names as globs in push rules. ([\#7271](https://github.com/matrix-org/synapse/issues/7271)) -- Fix a bug with cross-signing devices belonging to remote users who did not share a room with any user on the local homeserver. ([\#7289](https://github.com/matrix-org/synapse/issues/7289)) - -Synapse 1.12.3 (2020-04-03) -=========================== - -- Remove the the pin to Pillow 7.0 which was introduced in Synapse 1.12.2, and -correctly fix the issue with building the Debian packages. ([\#7212](https://github.com/matrix-org/synapse/issues/7212)) - -Synapse 1.12.2 (2020-04-02) -=========================== - -This release works around [an issue](https://github.com/matrix-org/synapse/issues/7208) with building the debian packages. - -No other significant changes since 1.12.1. - -Synapse 1.12.1 (2020-04-02) -=========================== - -No significant changes since 1.12.1rc1. - - -Synapse 1.12.1rc1 (2020-03-31) -============================== - -Bugfixes --------- - -- Fix starting workers when federation sending not split out. ([\#7133](https://github.com/matrix-org/synapse/issues/7133)). Introduced in v1.12.0. -- Avoid importing `sqlite3` when using the postgres backend. Contributed by David Vo. ([\#7155](https://github.com/matrix-org/synapse/issues/7155)). Introduced in v1.12.0rc1. -- Fix a bug which could cause outbound federation traffic to stop working if a client uploaded an incorrect e2e device signature. ([\#7177](https://github.com/matrix-org/synapse/issues/7177)). Introduced in v1.11.0. - -Synapse 1.12.0 (2020-03-23) -=========================== - -Debian packages and Docker images are rebuilt using the latest versions of -dependency libraries, including Twisted 20.3.0. **Please see security advisory -below**. - -Potential slow database update during upgrade ---------------------------------------------- - -Synapse 1.12.0 includes a database update which is run as part of the upgrade, -and which may take some time (several hours in the case of a large -server). Synapse will not respond to HTTP requests while this update is taking -place. For imformation on seeing if you are affected, and workaround if you -are, see the [upgrade notes](docs/upgrade.md#upgrading-to-v1120). - -Security advisory ------------------ - -Synapse may be vulnerable to request-smuggling attacks when it is used with a -reverse-proxy. The vulnerabilties are fixed in Twisted 20.3.0, and are -described in -[CVE-2020-10108](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-10108) -and -[CVE-2020-10109](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-10109). -For a good introduction to this class of request-smuggling attacks, see -https://portswigger.net/research/http-desync-attacks-request-smuggling-reborn. - -We are not aware of these vulnerabilities being exploited in the wild, and -do not believe that they are exploitable with current versions of any reverse -proxies. Nevertheless, we recommend that all Synapse administrators ensure that -they have the latest versions of the Twisted library to ensure that their -installation remains secure. - -* Administrators using the [`matrix.org` Docker - image](https://hub.docker.com/r/matrixdotorg/synapse/) or the [Debian/Ubuntu - packages from - `matrix.org`](https://matrix-org.github.io/synapse/latest/setup/installation.html#matrixorg-packages) - should ensure that they have version 1.12.0 installed: these images include - Twisted 20.3.0. -* Administrators who have [installed Synapse from - source](https://matrix-org.github.io/synapse/latest/setup/installation.html#installing-from-source) - should upgrade Twisted within their virtualenv by running: - ```sh - /bin/pip install 'Twisted>=20.3.0' - ``` -* Administrators who have installed Synapse from distribution packages should - consult the information from their distributions. - -The `matrix.org` Synapse instance was not vulnerable to these vulnerabilities. - -Advance notice of change to the default `git` branch for Synapse ----------------------------------------------------------------- - -Currently, the default `git` branch for Synapse is `master`, which tracks the -latest release. - -After the release of Synapse 1.13.0, we intend to change this default to -`develop`, which is the development tip. This is more consistent with common -practice and modern `git` usage. - -Although we try to keep `develop` in a stable state, there may be occasions -where regressions creep in. Developers and distributors who have scripts which -run builds using the default branch of `Synapse` should therefore consider -pinning their scripts to `master`. - - -Synapse 1.12.0rc1 (2020-03-19) -============================== - -Features --------- - -- Changes related to room alias management ([MSC2432](https://github.com/matrix-org/matrix-doc/pull/2432)): - - Publishing/removing a room from the room directory now requires the user to have a power level capable of modifying the canonical alias, instead of the room aliases. ([\#6965](https://github.com/matrix-org/synapse/issues/6965)) - - Validate the `alt_aliases` property of canonical alias events. ([\#6971](https://github.com/matrix-org/synapse/issues/6971)) - - Users with a power level sufficient to modify the canonical alias of a room can now delete room aliases. ([\#6986](https://github.com/matrix-org/synapse/issues/6986)) - - Implement updated authorization rules and redaction rules for aliases events, from [MSC2261](https://github.com/matrix-org/matrix-doc/pull/2261) and [MSC2432](https://github.com/matrix-org/matrix-doc/pull/2432). ([\#7037](https://github.com/matrix-org/synapse/issues/7037)) - - Stop sending m.room.aliases events during room creation and upgrade. ([\#6941](https://github.com/matrix-org/synapse/issues/6941)) - - Synapse no longer uses room alias events to calculate room names for push notifications. ([\#6966](https://github.com/matrix-org/synapse/issues/6966)) - - The room list endpoint no longer returns a list of aliases. ([\#6970](https://github.com/matrix-org/synapse/issues/6970)) - - Remove special handling of aliases events from [MSC2260](https://github.com/matrix-org/matrix-doc/pull/2260) added in v1.10.0rc1. ([\#7034](https://github.com/matrix-org/synapse/issues/7034)) -- Expose the `synctl`, `hash_password` and `generate_config` commands in the snapcraft package. Contributed by @devec0. ([\#6315](https://github.com/matrix-org/synapse/issues/6315)) -- Check that server_name is correctly set before running database updates. ([\#6982](https://github.com/matrix-org/synapse/issues/6982)) -- Break down monthly active users by `appservice_id` and emit via Prometheus. ([\#7030](https://github.com/matrix-org/synapse/issues/7030)) -- Render a configurable and comprehensible error page if something goes wrong during the SAML2 authentication process. ([\#7058](https://github.com/matrix-org/synapse/issues/7058), [\#7067](https://github.com/matrix-org/synapse/issues/7067)) -- Add an optional parameter to control whether other sessions are logged out when a user's password is modified. ([\#7085](https://github.com/matrix-org/synapse/issues/7085)) -- Add prometheus metrics for the number of active pushers. ([\#7103](https://github.com/matrix-org/synapse/issues/7103), [\#7106](https://github.com/matrix-org/synapse/issues/7106)) -- Improve performance when making HTTPS requests to sygnal, sydent, etc, by sharing the SSL context object between connections. ([\#7094](https://github.com/matrix-org/synapse/issues/7094)) - - -Bugfixes --------- - -- When a user's profile is updated via the admin API, also generate a displayname/avatar update for that user in each room. ([\#6572](https://github.com/matrix-org/synapse/issues/6572)) -- Fix a couple of bugs in email configuration handling. ([\#6962](https://github.com/matrix-org/synapse/issues/6962)) -- Fix an issue affecting worker-based deployments where replication would stop working, necessitating a full restart, after joining a large room. ([\#6967](https://github.com/matrix-org/synapse/issues/6967)) -- Fix `duplicate key` error which was logged when rejoining a room over federation. ([\#6968](https://github.com/matrix-org/synapse/issues/6968)) -- Prevent user from setting 'deactivated' to anything other than a bool on the v2 PUT /users Admin API. ([\#6990](https://github.com/matrix-org/synapse/issues/6990)) -- Fix py35-old CI by using native tox package. ([\#7018](https://github.com/matrix-org/synapse/issues/7018)) -- Fix a bug causing `org.matrix.dummy_event` to be included in responses from `/sync`. ([\#7035](https://github.com/matrix-org/synapse/issues/7035)) -- Fix a bug that renders UTF-8 text files incorrectly when loaded from media. Contributed by @TheStranjer. ([\#7044](https://github.com/matrix-org/synapse/issues/7044)) -- Fix a bug that would cause Synapse to respond with an error about event visibility if a client tried to request the state of a room at a given token. ([\#7066](https://github.com/matrix-org/synapse/issues/7066)) -- Repair a data-corruption issue which was introduced in Synapse 1.10, and fixed in Synapse 1.11, and which could cause `/sync` to return with 404 errors about missing events and unknown rooms. ([\#7070](https://github.com/matrix-org/synapse/issues/7070)) -- Fix a bug causing account validity renewal emails to be sent even if the feature is turned off in some cases. ([\#7074](https://github.com/matrix-org/synapse/issues/7074)) - - -Improved Documentation ----------------------- - -- Updated CentOS8 install instructions. Contributed by Richard Kellner. ([\#6925](https://github.com/matrix-org/synapse/issues/6925)) -- Fix `POSTGRES_INITDB_ARGS` in the `contrib/docker/docker-compose.yml` example docker-compose configuration. ([\#6984](https://github.com/matrix-org/synapse/issues/6984)) -- Change date in [INSTALL.md](./INSTALL.md#tls-certificates) for last date of getting TLS certificates to November 2019. ([\#7015](https://github.com/matrix-org/synapse/issues/7015)) -- Document that the fallback auth endpoints must be routed to the same worker node as the register endpoints. ([\#7048](https://github.com/matrix-org/synapse/issues/7048)) - - -Deprecations and Removals -------------------------- - -- Remove the unused query_auth federation endpoint per [MSC2451](https://github.com/matrix-org/matrix-doc/pull/2451). ([\#7026](https://github.com/matrix-org/synapse/issues/7026)) - - -Internal Changes ----------------- - -- Add type hints to `logging/context.py`. ([\#6309](https://github.com/matrix-org/synapse/issues/6309)) -- Add some clarifications to `README.md` in the database schema directory. ([\#6615](https://github.com/matrix-org/synapse/issues/6615)) -- Refactoring work in preparation for changing the event redaction algorithm. ([\#6874](https://github.com/matrix-org/synapse/issues/6874), [\#6875](https://github.com/matrix-org/synapse/issues/6875), [\#6983](https://github.com/matrix-org/synapse/issues/6983), [\#7003](https://github.com/matrix-org/synapse/issues/7003)) -- Improve performance of v2 state resolution for large rooms. ([\#6952](https://github.com/matrix-org/synapse/issues/6952), [\#7095](https://github.com/matrix-org/synapse/issues/7095)) -- Reduce time spent doing GC, by freezing objects on startup. ([\#6953](https://github.com/matrix-org/synapse/issues/6953)) -- Minor perfermance fixes to `get_auth_chain_ids`. ([\#6954](https://github.com/matrix-org/synapse/issues/6954)) -- Don't record remote cross-signing keys in the `devices` table. ([\#6956](https://github.com/matrix-org/synapse/issues/6956)) -- Use flake8-comprehensions to enforce good hygiene of list/set/dict comprehensions. ([\#6957](https://github.com/matrix-org/synapse/issues/6957)) -- Merge worker apps together. ([\#6964](https://github.com/matrix-org/synapse/issues/6964), [\#7002](https://github.com/matrix-org/synapse/issues/7002), [\#7055](https://github.com/matrix-org/synapse/issues/7055), [\#7104](https://github.com/matrix-org/synapse/issues/7104)) -- Remove redundant `store_room` call from `FederationHandler._process_received_pdu`. ([\#6979](https://github.com/matrix-org/synapse/issues/6979)) -- Update warning for incorrect database collation/ctype to include link to documentation. ([\#6985](https://github.com/matrix-org/synapse/issues/6985)) -- Add some type annotations to the database storage classes. ([\#6987](https://github.com/matrix-org/synapse/issues/6987)) -- Port `synapse.handlers.presence` to async/await. ([\#6991](https://github.com/matrix-org/synapse/issues/6991), [\#7019](https://github.com/matrix-org/synapse/issues/7019)) -- Add some type annotations to the federation base & client classes. ([\#6995](https://github.com/matrix-org/synapse/issues/6995)) -- Port `synapse.rest.keys` to async/await. ([\#7020](https://github.com/matrix-org/synapse/issues/7020)) -- Add a type check to `is_verified` when processing room keys. ([\#7045](https://github.com/matrix-org/synapse/issues/7045)) -- Add type annotations and comments to the auth handler. ([\#7063](https://github.com/matrix-org/synapse/issues/7063)) - - -Synapse 1.11.1 (2020-03-03) -=========================== - -This release includes a security fix impacting installations using Single Sign-On (i.e. SAML2 or CAS) for authentication. Administrators of such installations are encouraged to upgrade as soon as possible. - -The release also includes fixes for a couple of other bugs. - -Bugfixes --------- - -- Add a confirmation step to the SSO login flow before redirecting users to the redirect URL. ([b2bd54a2](https://github.com/matrix-org/synapse/commit/b2bd54a2e31d9a248f73fadb184ae9b4cbdb49f9), [65c73cdf](https://github.com/matrix-org/synapse/commit/65c73cdfec1876a9fec2fd2c3a74923cd146fe0b), [a0178df1](https://github.com/matrix-org/synapse/commit/a0178df10422a76fd403b82d2b2a4ed28a9a9d1e)) -- Fixed set a user as an admin with the admin API `PUT /_synapse/admin/v2/users/`. Contributed by @dklimpel. ([\#6910](https://github.com/matrix-org/synapse/issues/6910)) -- Fix bug introduced in Synapse 1.11.0 which sometimes caused errors when joining rooms over federation, with `'coroutine' object has no attribute 'event_id'`. ([\#6996](https://github.com/matrix-org/synapse/issues/6996)) - - -Synapse 1.11.0 (2020-02-21) -=========================== - -Improved Documentation ----------------------- - -- Small grammatical fixes to the ACME v1 deprecation notice. ([\#6944](https://github.com/matrix-org/synapse/issues/6944)) - - -Synapse 1.11.0rc1 (2020-02-19) -============================== - -Features --------- - -- Admin API to add or modify threepids of user accounts. ([\#6769](https://github.com/matrix-org/synapse/issues/6769)) -- Limit the number of events that can be requested by the backfill federation API to 100. ([\#6864](https://github.com/matrix-org/synapse/issues/6864)) -- Add ability to run some group APIs on workers. ([\#6866](https://github.com/matrix-org/synapse/issues/6866)) -- Reject device display names over 100 characters in length to prevent abuse. ([\#6882](https://github.com/matrix-org/synapse/issues/6882)) -- Add ability to route federation user device queries to workers. ([\#6873](https://github.com/matrix-org/synapse/issues/6873)) -- The result of a user directory search can now be filtered via the spam checker. ([\#6888](https://github.com/matrix-org/synapse/issues/6888)) -- Implement new `GET /_matrix/client/unstable/org.matrix.msc2432/rooms/{roomId}/aliases` endpoint as per [MSC2432](https://github.com/matrix-org/matrix-doc/pull/2432). ([\#6939](https://github.com/matrix-org/synapse/issues/6939), [\#6948](https://github.com/matrix-org/synapse/issues/6948), [\#6949](https://github.com/matrix-org/synapse/issues/6949)) -- Stop sending `m.room.alias` events wheng adding / removing aliases. Check `alt_aliases` in the latest `m.room.canonical_alias` event when deleting an alias. ([\#6904](https://github.com/matrix-org/synapse/issues/6904)) -- Change the default power levels of invites, tombstones and server ACLs for new rooms. ([\#6834](https://github.com/matrix-org/synapse/issues/6834)) - -Bugfixes --------- - -- Fixed third party event rules function `on_create_room`'s return value being ignored. ([\#6781](https://github.com/matrix-org/synapse/issues/6781)) -- Allow URL-encoded User IDs on `/_synapse/admin/v2/users/[/admin]` endpoints. Thanks to @NHAS for reporting. ([\#6825](https://github.com/matrix-org/synapse/issues/6825)) -- Fix Synapse refusing to start if `federation_certificate_verification_whitelist` option is blank. ([\#6849](https://github.com/matrix-org/synapse/issues/6849)) -- Fix errors from logging in the purge jobs related to the message retention policies support. ([\#6945](https://github.com/matrix-org/synapse/issues/6945)) -- Return a 404 instead of 200 for querying information of a non-existant user through the admin API. ([\#6901](https://github.com/matrix-org/synapse/issues/6901)) - - -Updates to the Docker image ---------------------------- - -- The deprecated "generate-config-on-the-fly" mode is no longer supported. ([\#6918](https://github.com/matrix-org/synapse/issues/6918)) - - -Improved Documentation ----------------------- - -- Add details of PR merge strategy to contributing docs. ([\#6846](https://github.com/matrix-org/synapse/issues/6846)) -- Spell out that the last event sent to a room won't be deleted by a purge. ([\#6891](https://github.com/matrix-org/synapse/issues/6891)) -- Update Synapse's documentation to warn about the deprecation of ACME v1. ([\#6905](https://github.com/matrix-org/synapse/issues/6905), [\#6907](https://github.com/matrix-org/synapse/issues/6907), [\#6909](https://github.com/matrix-org/synapse/issues/6909)) -- Add documentation for the spam checker. ([\#6906](https://github.com/matrix-org/synapse/issues/6906)) -- Fix worker docs to point `/publicised_groups` API correctly. ([\#6938](https://github.com/matrix-org/synapse/issues/6938)) -- Clean up and update docs on setting up federation. ([\#6940](https://github.com/matrix-org/synapse/issues/6940)) -- Add a warning about indentation to generated configuration files. ([\#6920](https://github.com/matrix-org/synapse/issues/6920)) -- Databases created using the compose file in contrib/docker will now always have correct encoding and locale settings. Contributed by Fridtjof Mund. ([\#6921](https://github.com/matrix-org/synapse/issues/6921)) -- Update pip install directions in readme to avoid error when using zsh. ([\#6855](https://github.com/matrix-org/synapse/issues/6855)) - - -Deprecations and Removals -------------------------- - -- Remove `m.lazy_load_members` from `unstable_features` since lazy loading is in the stable Client-Server API version r0.5.0. ([\#6877](https://github.com/matrix-org/synapse/issues/6877)) - - -Internal Changes ----------------- - -- Add type hints to `SyncHandler`. ([\#6821](https://github.com/matrix-org/synapse/issues/6821)) -- Refactoring work in preparation for changing the event redaction algorithm. ([\#6823](https://github.com/matrix-org/synapse/issues/6823), [\#6827](https://github.com/matrix-org/synapse/issues/6827), [\#6854](https://github.com/matrix-org/synapse/issues/6854), [\#6856](https://github.com/matrix-org/synapse/issues/6856), [\#6857](https://github.com/matrix-org/synapse/issues/6857), [\#6858](https://github.com/matrix-org/synapse/issues/6858)) -- Fix stacktraces when using `ObservableDeferred` and async/await. ([\#6836](https://github.com/matrix-org/synapse/issues/6836)) -- Port much of `synapse.handlers.federation` to async/await. ([\#6837](https://github.com/matrix-org/synapse/issues/6837), [\#6840](https://github.com/matrix-org/synapse/issues/6840)) -- Populate `rooms.room_version` database column at startup, rather than in a background update. ([\#6847](https://github.com/matrix-org/synapse/issues/6847)) -- Reduce amount we log at `INFO` level. ([\#6833](https://github.com/matrix-org/synapse/issues/6833), [\#6862](https://github.com/matrix-org/synapse/issues/6862)) -- Remove unused `get_room_stats_state` method. ([\#6869](https://github.com/matrix-org/synapse/issues/6869)) -- Add typing to `synapse.federation.sender` and port to async/await. ([\#6871](https://github.com/matrix-org/synapse/issues/6871)) -- Refactor `_EventInternalMetadata` object to improve type safety. ([\#6872](https://github.com/matrix-org/synapse/issues/6872)) -- Add an additional entry to the SyTest blacklist for worker mode. ([\#6883](https://github.com/matrix-org/synapse/issues/6883)) -- Fix the use of sed in the linting scripts when using BSD sed. ([\#6887](https://github.com/matrix-org/synapse/issues/6887)) -- Add type hints to the spam checker module. ([\#6915](https://github.com/matrix-org/synapse/issues/6915)) -- Convert the directory handler tests to use HomeserverTestCase. ([\#6919](https://github.com/matrix-org/synapse/issues/6919)) -- Increase DB/CPU perf of `_is_server_still_joined` check. ([\#6936](https://github.com/matrix-org/synapse/issues/6936)) -- Tiny optimisation for incoming HTTP request dispatch. ([\#6950](https://github.com/matrix-org/synapse/issues/6950)) - - -Synapse 1.10.1 (2020-02-17) -=========================== - -Bugfixes --------- - -- Fix a bug introduced in Synapse 1.10.0 which would cause room state to be cleared in the database if Synapse was upgraded direct from 1.2.1 or earlier to 1.10.0. ([\#6924](https://github.com/matrix-org/synapse/issues/6924)) - - -Synapse 1.10.0 (2020-02-12) -=========================== - -**WARNING to client developers**: As of this release Synapse validates `client_secret` parameters in the Client-Server API as per the spec. See [\#6766](https://github.com/matrix-org/synapse/issues/6766) for details. - -Updates to the Docker image ---------------------------- - -- Update the docker images to Alpine Linux 3.11. ([\#6897](https://github.com/matrix-org/synapse/issues/6897)) - - -Synapse 1.10.0rc5 (2020-02-11) -============================== - -Bugfixes --------- - -- Fix the filtering introduced in 1.10.0rc3 to also apply to the state blocks returned by `/sync`. ([\#6884](https://github.com/matrix-org/synapse/issues/6884)) - -Synapse 1.10.0rc4 (2020-02-11) -============================== - -This release candidate was built incorrectly and is superceded by 1.10.0rc5. - -Synapse 1.10.0rc3 (2020-02-10) -============================== - -Features --------- - -- Filter out `m.room.aliases` from the CS API to mitigate abuse while a better solution is specced. ([\#6878](https://github.com/matrix-org/synapse/issues/6878)) - - -Internal Changes ----------------- - -- Fix continuous integration failures with old versions of `pip`, which were introduced by a release of the `zipp` library. ([\#6880](https://github.com/matrix-org/synapse/issues/6880)) - - -Synapse 1.10.0rc2 (2020-02-06) -============================== - -Bugfixes --------- - -- Fix an issue with cross-signing where device signatures were not sent to remote servers. ([\#6844](https://github.com/matrix-org/synapse/issues/6844)) -- Fix to the unknown remote device detection which was introduced in 1.10.rc1. ([\#6848](https://github.com/matrix-org/synapse/issues/6848)) - - -Internal Changes ----------------- - -- Detect unexpected sender keys on remote encrypted events and resync device lists. ([\#6850](https://github.com/matrix-org/synapse/issues/6850)) - - -Synapse 1.10.0rc1 (2020-01-31) -============================== - -Features --------- - -- Add experimental support for updated authorization rules for aliases events, from [MSC2260](https://github.com/matrix-org/matrix-doc/pull/2260). ([\#6787](https://github.com/matrix-org/synapse/issues/6787), [\#6790](https://github.com/matrix-org/synapse/issues/6790), [\#6794](https://github.com/matrix-org/synapse/issues/6794)) - - -Bugfixes --------- - -- Warn if postgres database has a non-C locale, as that can cause issues when upgrading locales (e.g. due to upgrading OS). ([\#6734](https://github.com/matrix-org/synapse/issues/6734)) -- Minor fixes to `PUT /_synapse/admin/v2/users` admin api. ([\#6761](https://github.com/matrix-org/synapse/issues/6761)) -- Validate `client_secret` parameter using the regex provided by the Client-Server API, temporarily allowing `:` characters for older clients. The `:` character will be removed in a future release. ([\#6767](https://github.com/matrix-org/synapse/issues/6767)) -- Fix persisting redaction events that have been redacted (or otherwise don't have a redacts key). ([\#6771](https://github.com/matrix-org/synapse/issues/6771)) -- Fix outbound federation request metrics. ([\#6795](https://github.com/matrix-org/synapse/issues/6795)) -- Fix bug where querying a remote user's device keys that weren't cached resulted in only returning a single device. ([\#6796](https://github.com/matrix-org/synapse/issues/6796)) -- Fix race in federation sender worker that delayed sending of device updates. ([\#6799](https://github.com/matrix-org/synapse/issues/6799), [\#6800](https://github.com/matrix-org/synapse/issues/6800)) -- Fix bug where Synapse didn't invalidate cache of remote users' devices when Synapse left a room. ([\#6801](https://github.com/matrix-org/synapse/issues/6801)) -- Fix waking up other workers when remote server is detected to have come back online. ([\#6811](https://github.com/matrix-org/synapse/issues/6811)) - - -Improved Documentation ----------------------- - -- Clarify documentation related to `user_dir` and `federation_reader` workers. ([\#6775](https://github.com/matrix-org/synapse/issues/6775)) - - -Internal Changes ----------------- - -- Record room versions in the `rooms` table. ([\#6729](https://github.com/matrix-org/synapse/issues/6729), [\#6788](https://github.com/matrix-org/synapse/issues/6788), [\#6810](https://github.com/matrix-org/synapse/issues/6810)) -- Propagate cache invalidates from workers to other workers. ([\#6748](https://github.com/matrix-org/synapse/issues/6748)) -- Remove some unnecessary admin handler abstraction methods. ([\#6751](https://github.com/matrix-org/synapse/issues/6751)) -- Add some debugging for media storage providers. ([\#6757](https://github.com/matrix-org/synapse/issues/6757)) -- Detect unknown remote devices and mark cache as stale. ([\#6776](https://github.com/matrix-org/synapse/issues/6776), [\#6819](https://github.com/matrix-org/synapse/issues/6819)) -- Attempt to resync remote users' devices when detected as stale. ([\#6786](https://github.com/matrix-org/synapse/issues/6786)) -- Delete current state from the database when server leaves a room. ([\#6792](https://github.com/matrix-org/synapse/issues/6792)) -- When a client asks for a remote user's device keys check if the local cache for that user has been marked as potentially stale. ([\#6797](https://github.com/matrix-org/synapse/issues/6797)) -- Add background update to clean out left rooms from current state. ([\#6802](https://github.com/matrix-org/synapse/issues/6802), [\#6816](https://github.com/matrix-org/synapse/issues/6816)) -- Refactoring work in preparation for changing the event redaction algorithm. ([\#6803](https://github.com/matrix-org/synapse/issues/6803), [\#6805](https://github.com/matrix-org/synapse/issues/6805), [\#6806](https://github.com/matrix-org/synapse/issues/6806), [\#6807](https://github.com/matrix-org/synapse/issues/6807), [\#6820](https://github.com/matrix-org/synapse/issues/6820)) - - -Synapse 1.9.1 (2020-01-28) -========================== - -Bugfixes --------- - -- Fix bug where setting `mau_limit_reserved_threepids` config would cause Synapse to refuse to start. ([\#6793](https://github.com/matrix-org/synapse/issues/6793)) - - -Synapse 1.9.0 (2020-01-23) -========================== - -**WARNING**: As of this release, Synapse no longer supports versions of SQLite before 3.11, and will refuse to start when configured to use an older version. Administrators are recommended to migrate their database to Postgres (see instructions [here](docs/postgres.md)). - -If your Synapse deployment uses workers, note that the reverse-proxy configurations for the `synapse.app.media_repository`, `synapse.app.federation_reader` and `synapse.app.event_creator` workers have changed, with the addition of a few paths (see the updated configurations [here](docs/workers.md#available-worker-applications)). Existing configurations will continue to work. - - -Improved Documentation ----------------------- - -- Fix endpoint documentation for the List Rooms admin API. ([\#6770](https://github.com/matrix-org/synapse/issues/6770)) - - -Synapse 1.9.0rc1 (2020-01-22) -============================= - -Features --------- - -- Allow admin to create or modify a user. Contributed by Awesome Technologies Innovationslabor GmbH. ([\#5742](https://github.com/matrix-org/synapse/issues/5742)) -- Add new quarantine media admin APIs to quarantine by media ID or by user who uploaded the media. ([\#6681](https://github.com/matrix-org/synapse/issues/6681), [\#6756](https://github.com/matrix-org/synapse/issues/6756)) -- Add `org.matrix.e2e_cross_signing` to `unstable_features` in `/versions` as per [MSC1756](https://github.com/matrix-org/matrix-doc/pull/1756). ([\#6712](https://github.com/matrix-org/synapse/issues/6712)) -- Add a new admin API to list and filter rooms on the server. ([\#6720](https://github.com/matrix-org/synapse/issues/6720)) - - -Bugfixes --------- - -- Correctly proxy HTTP errors due to API calls to remote group servers. ([\#6654](https://github.com/matrix-org/synapse/issues/6654)) -- Fix media repo admin APIs when using a media worker. ([\#6664](https://github.com/matrix-org/synapse/issues/6664)) -- Fix "CRITICAL" errors being logged when a request is received for a uri containing non-ascii characters. ([\#6682](https://github.com/matrix-org/synapse/issues/6682)) -- Fix a bug where we would assign a numeric user ID if somebody tried registering with an empty username. ([\#6690](https://github.com/matrix-org/synapse/issues/6690)) -- Fix `purge_room` admin API. ([\#6711](https://github.com/matrix-org/synapse/issues/6711)) -- Fix a bug causing Synapse to not always purge quiet rooms with a low `max_lifetime` in their message retention policies when running the automated purge jobs. ([\#6714](https://github.com/matrix-org/synapse/issues/6714)) -- Fix the `synapse_port_db` not correctly running background updates. Thanks @tadzik for reporting. ([\#6718](https://github.com/matrix-org/synapse/issues/6718)) -- Fix changing password via user admin API. ([\#6730](https://github.com/matrix-org/synapse/issues/6730)) -- Fix `/events/:event_id` deprecated API. ([\#6731](https://github.com/matrix-org/synapse/issues/6731)) -- Fix monthly active user limiting support for worker mode, fixes [#4639](https://github.com/matrix-org/synapse/issues/4639). ([\#6742](https://github.com/matrix-org/synapse/issues/6742)) -- Fix bug when setting `account_validity` to an empty block in the config. Thanks to @Sorunome for reporting. ([\#6747](https://github.com/matrix-org/synapse/issues/6747)) -- Fix `AttributeError: 'NoneType' object has no attribute 'get'` in `hash_password` when configuration has an empty `password_config`. Contributed by @ivilata. ([\#6753](https://github.com/matrix-org/synapse/issues/6753)) -- Fix the `docker-compose.yaml` overriding the entire `/etc` folder of the container. Contributed by Fabian Meyer. ([\#6656](https://github.com/matrix-org/synapse/issues/6656)) - - -Improved Documentation ----------------------- - -- Fix a typo in the configuration example for purge jobs in the sample configuration file. ([\#6621](https://github.com/matrix-org/synapse/issues/6621)) -- Add complete documentation of the message retention policies support. ([\#6624](https://github.com/matrix-org/synapse/issues/6624), [\#6665](https://github.com/matrix-org/synapse/issues/6665)) -- Add some helpful tips about changelog entries to the GitHub pull request template. ([\#6663](https://github.com/matrix-org/synapse/issues/6663)) -- Clarify the `account_validity` and `email` sections of the sample configuration. ([\#6685](https://github.com/matrix-org/synapse/issues/6685)) -- Add more endpoints to the documentation for Synapse workers. ([\#6698](https://github.com/matrix-org/synapse/issues/6698)) - - -Deprecations and Removals -------------------------- - -- Synapse no longer supports versions of SQLite before 3.11, and will refuse to start when configured to use an older version. Administrators are recommended to migrate their database to Postgres (see instructions [here](docs/postgres.md)). ([\#6675](https://github.com/matrix-org/synapse/issues/6675)) - - -Internal Changes ----------------- - -- Add `local_current_membership` table for tracking local user membership state in rooms. ([\#6655](https://github.com/matrix-org/synapse/issues/6655), [\#6728](https://github.com/matrix-org/synapse/issues/6728)) -- Port `synapse.replication.tcp` to async/await. ([\#6666](https://github.com/matrix-org/synapse/issues/6666)) -- Fixup `synapse.replication` to pass mypy checks. ([\#6667](https://github.com/matrix-org/synapse/issues/6667)) -- Allow `additional_resources` to implement `IResource` directly. ([\#6686](https://github.com/matrix-org/synapse/issues/6686)) -- Allow REST endpoint implementations to raise a `RedirectException`, which will redirect the user's browser to a given location. ([\#6687](https://github.com/matrix-org/synapse/issues/6687)) -- Updates and extensions to the module API. ([\#6688](https://github.com/matrix-org/synapse/issues/6688)) -- Updates to the SAML mapping provider API. ([\#6689](https://github.com/matrix-org/synapse/issues/6689), [\#6723](https://github.com/matrix-org/synapse/issues/6723)) -- Remove redundant `RegistrationError` class. ([\#6691](https://github.com/matrix-org/synapse/issues/6691)) -- Don't block processing of incoming EDUs behind processing PDUs in the same transaction. ([\#6697](https://github.com/matrix-org/synapse/issues/6697)) -- Remove duplicate check for the `session` query parameter on the `/auth/xxx/fallback/web` Client-Server endpoint. ([\#6702](https://github.com/matrix-org/synapse/issues/6702)) -- Attempt to retry sending a transaction when we detect a remote server has come back online, rather than waiting for a transaction to be triggered by new data. ([\#6706](https://github.com/matrix-org/synapse/issues/6706)) -- Add `StateMap` type alias to simplify types. ([\#6715](https://github.com/matrix-org/synapse/issues/6715)) -- Add a `DeltaState` to track changes to be made to current state during event persistence. ([\#6716](https://github.com/matrix-org/synapse/issues/6716)) -- Add more logging around message retention policies support. ([\#6717](https://github.com/matrix-org/synapse/issues/6717)) -- When processing a SAML response, log the assertions for easier configuration. ([\#6724](https://github.com/matrix-org/synapse/issues/6724)) -- Fixup `synapse.rest` to pass mypy. ([\#6732](https://github.com/matrix-org/synapse/issues/6732), [\#6764](https://github.com/matrix-org/synapse/issues/6764)) -- Fixup `synapse.api` to pass mypy. ([\#6733](https://github.com/matrix-org/synapse/issues/6733)) -- Allow streaming cache 'invalidate all' to workers. ([\#6749](https://github.com/matrix-org/synapse/issues/6749)) -- Remove unused CI docker compose files. ([\#6754](https://github.com/matrix-org/synapse/issues/6754)) - - -Synapse 1.8.0 (2020-01-09) -========================== - -**WARNING**: As of this release Synapse will refuse to start if the `log_file` config option is specified. Support for the option was removed in v1.3.0. - - -Bugfixes --------- - -- Fix `GET` request on `/_synapse/admin/v2/users` endpoint. Contributed by Awesome Technologies Innovationslabor GmbH. ([\#6563](https://github.com/matrix-org/synapse/issues/6563)) -- Fix incorrect signing of responses from the key server implementation. ([\#6657](https://github.com/matrix-org/synapse/issues/6657)) - - -Synapse 1.8.0rc1 (2020-01-07) -============================= - -Features --------- - -- Add v2 APIs for the `send_join` and `send_leave` federation endpoints (as described in [MSC1802](https://github.com/matrix-org/matrix-doc/pull/1802)). ([\#6349](https://github.com/matrix-org/synapse/issues/6349)) -- Add a develop script to generate full SQL schemas. ([\#6394](https://github.com/matrix-org/synapse/issues/6394)) -- Add custom SAML username mapping functionality through an external provider plugin. ([\#6411](https://github.com/matrix-org/synapse/issues/6411)) -- Automatically delete empty groups/communities. ([\#6453](https://github.com/matrix-org/synapse/issues/6453)) -- Add option `limit_profile_requests_to_users_who_share_rooms` to prevent requirement of a local user sharing a room with another user to query their profile information. ([\#6523](https://github.com/matrix-org/synapse/issues/6523)) -- Add an `export_signing_key` script to extract the public part of signing keys when rotating them. ([\#6546](https://github.com/matrix-org/synapse/issues/6546)) -- Add experimental config option to specify multiple databases. ([\#6580](https://github.com/matrix-org/synapse/issues/6580)) -- Raise an error if someone tries to use the `log_file` config option. ([\#6626](https://github.com/matrix-org/synapse/issues/6626)) - - -Bugfixes --------- - -- Prevent redacted events from being returned during message search. ([\#6377](https://github.com/matrix-org/synapse/issues/6377), [\#6522](https://github.com/matrix-org/synapse/issues/6522)) -- Prevent error on trying to search a upgraded room when the server is not in the predecessor room. ([\#6385](https://github.com/matrix-org/synapse/issues/6385)) -- Improve performance of looking up cross-signing keys. ([\#6486](https://github.com/matrix-org/synapse/issues/6486)) -- Fix race which occasionally caused deleted devices to reappear. ([\#6514](https://github.com/matrix-org/synapse/issues/6514)) -- Fix missing row in `device_max_stream_id` that could cause unable to decrypt errors after server restart. ([\#6555](https://github.com/matrix-org/synapse/issues/6555)) -- Fix a bug which meant that we did not send systemd notifications on startup if acme was enabled. ([\#6571](https://github.com/matrix-org/synapse/issues/6571)) -- Fix exception when fetching the `matrix.org:ed25519:auto` key. ([\#6625](https://github.com/matrix-org/synapse/issues/6625)) -- Fix bug where a moderator upgraded a room and became an admin in the new room. ([\#6633](https://github.com/matrix-org/synapse/issues/6633)) -- Fix an error which was thrown by the `PresenceHandler` `_on_shutdown` handler. ([\#6640](https://github.com/matrix-org/synapse/issues/6640)) -- Fix exceptions in the synchrotron worker log when events are rejected. ([\#6645](https://github.com/matrix-org/synapse/issues/6645)) -- Ensure that upgraded rooms are removed from the directory. ([\#6648](https://github.com/matrix-org/synapse/issues/6648)) -- Fix a bug causing Synapse not to fetch missing events when it believes it has every event in the room. ([\#6652](https://github.com/matrix-org/synapse/issues/6652)) - - -Improved Documentation ----------------------- - -- Document the Room Shutdown Admin API. ([\#6541](https://github.com/matrix-org/synapse/issues/6541)) -- Reword sections of [docs/federate.md](docs/federate.md) that explained delegation at time of Synapse 1.0 transition. ([\#6601](https://github.com/matrix-org/synapse/issues/6601)) -- Added the section 'Configuration' in [docs/turn-howto.md](docs/turn-howto.md). ([\#6614](https://github.com/matrix-org/synapse/issues/6614)) - - -Deprecations and Removals -------------------------- - -- Remove redundant code from event authorisation implementation. ([\#6502](https://github.com/matrix-org/synapse/issues/6502)) -- Remove unused, undocumented `/_matrix/content` API. ([\#6628](https://github.com/matrix-org/synapse/issues/6628)) - - -Internal Changes ----------------- - -- Add *experimental* support for multiple physical databases and split out state storage to separate data store. ([\#6245](https://github.com/matrix-org/synapse/issues/6245), [\#6510](https://github.com/matrix-org/synapse/issues/6510), [\#6511](https://github.com/matrix-org/synapse/issues/6511), [\#6513](https://github.com/matrix-org/synapse/issues/6513), [\#6564](https://github.com/matrix-org/synapse/issues/6564), [\#6565](https://github.com/matrix-org/synapse/issues/6565)) -- Port sections of code base to async/await. ([\#6496](https://github.com/matrix-org/synapse/issues/6496), [\#6504](https://github.com/matrix-org/synapse/issues/6504), [\#6505](https://github.com/matrix-org/synapse/issues/6505), [\#6517](https://github.com/matrix-org/synapse/issues/6517), [\#6559](https://github.com/matrix-org/synapse/issues/6559), [\#6647](https://github.com/matrix-org/synapse/issues/6647), [\#6653](https://github.com/matrix-org/synapse/issues/6653)) -- Remove `SnapshotCache` in favour of `ResponseCache`. ([\#6506](https://github.com/matrix-org/synapse/issues/6506)) -- Silence mypy errors for files outside those specified. ([\#6512](https://github.com/matrix-org/synapse/issues/6512)) -- Clean up some logging when handling incoming events over federation. ([\#6515](https://github.com/matrix-org/synapse/issues/6515)) -- Test more folders against mypy. ([\#6534](https://github.com/matrix-org/synapse/issues/6534)) -- Update `mypy` to new version. ([\#6537](https://github.com/matrix-org/synapse/issues/6537)) -- Adjust the sytest blacklist for worker mode. ([\#6538](https://github.com/matrix-org/synapse/issues/6538)) -- Remove unused `get_pagination_rows` methods from `EventSource` classes. ([\#6557](https://github.com/matrix-org/synapse/issues/6557)) -- Clean up logs from the push notifier at startup. ([\#6558](https://github.com/matrix-org/synapse/issues/6558)) -- Improve diagnostics on database upgrade failure. ([\#6570](https://github.com/matrix-org/synapse/issues/6570)) -- Reduce the reconnect time when worker replication fails, to make it easier to catch up. ([\#6617](https://github.com/matrix-org/synapse/issues/6617)) -- Simplify http handling by removing redundant `SynapseRequestFactory`. ([\#6619](https://github.com/matrix-org/synapse/issues/6619)) -- Add a workaround for synapse raising exceptions when fetching the notary's own key from the notary. ([\#6620](https://github.com/matrix-org/synapse/issues/6620)) -- Automate generation of the sample log config. ([\#6627](https://github.com/matrix-org/synapse/issues/6627)) -- Simplify event creation code by removing redundant queries on the `event_reference_hashes` table. ([\#6629](https://github.com/matrix-org/synapse/issues/6629)) -- Fix errors when `frozen_dicts` are enabled. ([\#6642](https://github.com/matrix-org/synapse/issues/6642)) - - -Synapse 1.7.3 (2019-12-31) -========================== - -This release fixes a long-standing bug in the state resolution algorithm. - -Bugfixes --------- - -- Fix exceptions caused by state resolution choking on malformed events. ([\#6608](https://github.com/matrix-org/synapse/issues/6608)) - - -Synapse 1.7.2 (2019-12-20) -========================== - -This release fixes some regressions introduced in Synapse 1.7.0 and 1.7.1. - -Bugfixes --------- - -- Fix a regression introduced in Synapse 1.7.1 which caused errors when attempting to backfill rooms over federation. ([\#6576](https://github.com/matrix-org/synapse/issues/6576)) -- Fix a bug introduced in Synapse 1.7.0 which caused an error on startup when upgrading from versions before 1.3.0. ([\#6578](https://github.com/matrix-org/synapse/issues/6578)) - - -Synapse 1.7.1 (2019-12-18) -========================== - -This release includes several security fixes as well as a fix to a bug exposed by the security fixes. Administrators are encouraged to upgrade as soon as possible. - -Security updates ----------------- - -- Fix a bug which could cause room events to be incorrectly authorized using events from a different room. ([\#6501](https://github.com/matrix-org/synapse/issues/6501), [\#6503](https://github.com/matrix-org/synapse/issues/6503), [\#6521](https://github.com/matrix-org/synapse/issues/6521), [\#6524](https://github.com/matrix-org/synapse/issues/6524), [\#6530](https://github.com/matrix-org/synapse/issues/6530), [\#6531](https://github.com/matrix-org/synapse/issues/6531)) -- Fix a bug causing responses to the `/context` client endpoint to not use the pruned version of the event. ([\#6553](https://github.com/matrix-org/synapse/issues/6553)) -- Fix a cause of state resets in room versions 2 onwards. ([\#6556](https://github.com/matrix-org/synapse/issues/6556), [\#6560](https://github.com/matrix-org/synapse/issues/6560)) - -Bugfixes --------- - -- Fix a bug which could cause the federation server to incorrectly return errors when handling certain obscure event graphs. ([\#6526](https://github.com/matrix-org/synapse/issues/6526), [\#6527](https://github.com/matrix-org/synapse/issues/6527)) - -Synapse 1.7.0 (2019-12-13) -========================== - -This release changes the default settings so that only local authenticated users can query the server's room directory. See the [upgrade notes](docs/upgrade.md#upgrading-to-v170) for details. - -Support for SQLite versions before 3.11 is now deprecated. A future release will refuse to start if used with an SQLite version before 3.11. - -Administrators are reminded that SQLite should not be used for production instances. Instructions for migrating to Postgres are available [here](docs/postgres.md). A future release of synapse will, by default, disable federation for servers using SQLite. - -No significant changes since 1.7.0rc2. - - -Synapse 1.7.0rc2 (2019-12-11) -============================= - -Bugfixes --------- - -- Fix incorrect error message for invalid requests when setting user's avatar URL. ([\#6497](https://github.com/matrix-org/synapse/issues/6497)) -- Fix support for SQLite 3.7. ([\#6499](https://github.com/matrix-org/synapse/issues/6499)) -- Fix regression where sending email push would not work when using a pusher worker. ([\#6507](https://github.com/matrix-org/synapse/issues/6507), [\#6509](https://github.com/matrix-org/synapse/issues/6509)) - - -Synapse 1.7.0rc1 (2019-12-09) -============================= - -Features --------- - -- Implement per-room message retention policies. ([\#5815](https://github.com/matrix-org/synapse/issues/5815), [\#6436](https://github.com/matrix-org/synapse/issues/6436)) -- Add etag and count fields to key backup endpoints to help clients guess if there are new keys. ([\#5858](https://github.com/matrix-org/synapse/issues/5858)) -- Add `/admin/v2/users` endpoint with pagination. Contributed by Awesome Technologies Innovationslabor GmbH. ([\#5925](https://github.com/matrix-org/synapse/issues/5925)) -- Require User-Interactive Authentication for `/account/3pid/add`, meaning the user's password will be required to add a third-party ID to their account. ([\#6119](https://github.com/matrix-org/synapse/issues/6119)) -- Implement the `/_matrix/federation/unstable/net.atleastfornow/state/` API as drafted in MSC2314. ([\#6176](https://github.com/matrix-org/synapse/issues/6176)) -- Configure privacy-preserving settings by default for the room directory. ([\#6355](https://github.com/matrix-org/synapse/issues/6355)) -- Add ephemeral messages support by partially implementing [MSC2228](https://github.com/matrix-org/matrix-doc/pull/2228). ([\#6409](https://github.com/matrix-org/synapse/issues/6409)) -- Add support for [MSC 2367](https://github.com/matrix-org/matrix-doc/pull/2367), which allows specifying a reason on all membership events. ([\#6434](https://github.com/matrix-org/synapse/issues/6434)) - - -Bugfixes --------- - -- Transfer non-standard power levels on room upgrade. ([\#6237](https://github.com/matrix-org/synapse/issues/6237)) -- Fix error from the Pillow library when uploading RGBA images. ([\#6241](https://github.com/matrix-org/synapse/issues/6241)) -- Correctly apply the event filter to the `state`, `events_before` and `events_after` fields in the response to `/context` requests. ([\#6329](https://github.com/matrix-org/synapse/issues/6329)) -- Fix caching devices for remote users when using workers, so that we don't attempt to refetch (and potentially fail) each time a user requests devices. ([\#6332](https://github.com/matrix-org/synapse/issues/6332)) -- Prevent account data syncs getting lost across TCP replication. ([\#6333](https://github.com/matrix-org/synapse/issues/6333)) -- Fix bug: TypeError in `register_user()` while using LDAP auth module. ([\#6406](https://github.com/matrix-org/synapse/issues/6406)) -- Fix an intermittent exception when handling read-receipts. ([\#6408](https://github.com/matrix-org/synapse/issues/6408)) -- Fix broken guest registration when there are existing blocks of numeric user IDs. ([\#6420](https://github.com/matrix-org/synapse/issues/6420)) -- Fix startup error when http proxy is defined. ([\#6421](https://github.com/matrix-org/synapse/issues/6421)) -- Fix error when using synapse_port_db on a vanilla synapse db. ([\#6449](https://github.com/matrix-org/synapse/issues/6449)) -- Fix uploading multiple cross signing signatures for the same user. ([\#6451](https://github.com/matrix-org/synapse/issues/6451)) -- Fix bug which lead to exceptions being thrown in a loop when a cross-signed device is deleted. ([\#6462](https://github.com/matrix-org/synapse/issues/6462)) -- Fix `synapse_port_db` not exiting with a 0 code if something went wrong during the port process. ([\#6470](https://github.com/matrix-org/synapse/issues/6470)) -- Improve sanity-checking when receiving events over federation. ([\#6472](https://github.com/matrix-org/synapse/issues/6472)) -- Fix inaccurate per-block Prometheus metrics. ([\#6491](https://github.com/matrix-org/synapse/issues/6491)) -- Fix small performance regression for sending invites. ([\#6493](https://github.com/matrix-org/synapse/issues/6493)) -- Back out cross-signing code added in Synapse 1.5.0, which caused a performance regression. ([\#6494](https://github.com/matrix-org/synapse/issues/6494)) - - -Improved Documentation ----------------------- - -- Update documentation and variables in user contributed systemd reference file. ([\#6369](https://github.com/matrix-org/synapse/issues/6369), [\#6490](https://github.com/matrix-org/synapse/issues/6490)) -- Fix link in the user directory documentation. ([\#6388](https://github.com/matrix-org/synapse/issues/6388)) -- Add build instructions to the docker readme. ([\#6390](https://github.com/matrix-org/synapse/issues/6390)) -- Switch Ubuntu package install recommendation to use python3 packages in INSTALL.md. ([\#6443](https://github.com/matrix-org/synapse/issues/6443)) -- Write some docs for the quarantine_media api. ([\#6458](https://github.com/matrix-org/synapse/issues/6458)) -- Convert CONTRIBUTING.rst to markdown (among other small fixes). ([\#6461](https://github.com/matrix-org/synapse/issues/6461)) - - -Deprecations and Removals -------------------------- - -- Remove admin/v1/users_paginate endpoint. Contributed by Awesome Technologies Innovationslabor GmbH. ([\#5925](https://github.com/matrix-org/synapse/issues/5925)) -- Remove fallback for federation with old servers which lack the /federation/v1/state_ids API. ([\#6488](https://github.com/matrix-org/synapse/issues/6488)) - - -Internal Changes ----------------- - -- Add benchmarks for structured logging and improve output performance. ([\#6266](https://github.com/matrix-org/synapse/issues/6266)) -- Improve the performance of outputting structured logging. ([\#6322](https://github.com/matrix-org/synapse/issues/6322)) -- Refactor some code in the event authentication path for clarity. ([\#6343](https://github.com/matrix-org/synapse/issues/6343), [\#6468](https://github.com/matrix-org/synapse/issues/6468), [\#6480](https://github.com/matrix-org/synapse/issues/6480)) -- Clean up some unnecessary quotation marks around the codebase. ([\#6362](https://github.com/matrix-org/synapse/issues/6362)) -- Complain on startup instead of 500'ing during runtime when `public_baseurl` isn't set when necessary. ([\#6379](https://github.com/matrix-org/synapse/issues/6379)) -- Add a test scenario to make sure room history purges don't break `/messages` in the future. ([\#6392](https://github.com/matrix-org/synapse/issues/6392)) -- Clarifications for the email configuration settings. ([\#6423](https://github.com/matrix-org/synapse/issues/6423)) -- Add more tests to the blacklist when running in worker mode. ([\#6429](https://github.com/matrix-org/synapse/issues/6429)) -- Refactor data store layer to support multiple databases in the future. ([\#6454](https://github.com/matrix-org/synapse/issues/6454), [\#6464](https://github.com/matrix-org/synapse/issues/6464), [\#6469](https://github.com/matrix-org/synapse/issues/6469), [\#6487](https://github.com/matrix-org/synapse/issues/6487)) -- Port synapse.rest.client.v1 to async/await. ([\#6482](https://github.com/matrix-org/synapse/issues/6482)) -- Port synapse.rest.client.v2_alpha to async/await. ([\#6483](https://github.com/matrix-org/synapse/issues/6483)) -- Port SyncHandler to async/await. ([\#6484](https://github.com/matrix-org/synapse/issues/6484)) - -Synapse 1.6.1 (2019-11-28) -========================== - -Security updates ----------------- - -This release includes a security fix ([\#6426](https://github.com/matrix-org/synapse/issues/6426), below). Administrators are encouraged to upgrade as soon as possible. - -Bugfixes --------- - -- Clean up local threepids from user on account deactivation. ([\#6426](https://github.com/matrix-org/synapse/issues/6426)) -- Fix startup error when http proxy is defined. ([\#6421](https://github.com/matrix-org/synapse/issues/6421)) - - -Synapse 1.6.0 (2019-11-26) -========================== - -Bugfixes --------- - -- Fix phone home stats reporting. ([\#6418](https://github.com/matrix-org/synapse/issues/6418)) - - -Synapse 1.6.0rc2 (2019-11-25) -============================= - -Bugfixes --------- - -- Fix a bug which could cause the background database update hander for event labels to get stuck in a loop raising exceptions. ([\#6407](https://github.com/matrix-org/synapse/issues/6407)) - - -Synapse 1.6.0rc1 (2019-11-20) -============================= - -Features --------- - -- Add federation support for cross-signing. ([\#5727](https://github.com/matrix-org/synapse/issues/5727)) -- Increase default room version from 4 to 5, thereby enforcing server key validity period checks. ([\#6220](https://github.com/matrix-org/synapse/issues/6220)) -- Add support for outbound http proxying via http_proxy/HTTPS_PROXY env vars. ([\#6238](https://github.com/matrix-org/synapse/issues/6238)) -- Implement label-based filtering on `/sync` and `/messages` ([MSC2326](https://github.com/matrix-org/matrix-doc/pull/2326)). ([\#6301](https://github.com/matrix-org/synapse/issues/6301), [\#6310](https://github.com/matrix-org/synapse/issues/6310), [\#6340](https://github.com/matrix-org/synapse/issues/6340)) - - -Bugfixes --------- - -- Fix LruCache callback deduplication for Python 3.8. Contributed by @V02460. ([\#6213](https://github.com/matrix-org/synapse/issues/6213)) -- Remove a room from a server's public rooms list on room upgrade. ([\#6232](https://github.com/matrix-org/synapse/issues/6232), [\#6235](https://github.com/matrix-org/synapse/issues/6235)) -- Delete keys from key backup when deleting backup versions. ([\#6253](https://github.com/matrix-org/synapse/issues/6253)) -- Make notification of cross-signing signatures work with workers. ([\#6254](https://github.com/matrix-org/synapse/issues/6254)) -- Fix exception when remote servers attempt to join a room that they're not allowed to join. ([\#6278](https://github.com/matrix-org/synapse/issues/6278)) -- Prevent errors from appearing on Synapse startup if `git` is not installed. ([\#6284](https://github.com/matrix-org/synapse/issues/6284)) -- Appservice requests will no longer contain a double slash prefix when the appservice url provided ends in a slash. ([\#6306](https://github.com/matrix-org/synapse/issues/6306)) -- Fix `/purge_room` admin API. ([\#6307](https://github.com/matrix-org/synapse/issues/6307)) -- Fix the `hidden` field in the `devices` table for SQLite versions prior to 3.23.0. ([\#6313](https://github.com/matrix-org/synapse/issues/6313)) -- Fix bug which casued rejected events to be persisted with the wrong room state. ([\#6320](https://github.com/matrix-org/synapse/issues/6320)) -- Fix bug where `rc_login` ratelimiting would prematurely kick in. ([\#6335](https://github.com/matrix-org/synapse/issues/6335)) -- Prevent the server taking a long time to start up when guest registration is enabled. ([\#6338](https://github.com/matrix-org/synapse/issues/6338)) -- Fix bug where upgrading a guest account to a full user would fail when account validity is enabled. ([\#6359](https://github.com/matrix-org/synapse/issues/6359)) -- Fix `to_device` stream ID getting reset every time Synapse restarts, which had the potential to cause unable to decrypt errors. ([\#6363](https://github.com/matrix-org/synapse/issues/6363)) -- Fix permission denied error when trying to generate a config file with the docker image. ([\#6389](https://github.com/matrix-org/synapse/issues/6389)) - - -Improved Documentation ----------------------- - -- Contributor documentation now mentions script to run linters. ([\#6164](https://github.com/matrix-org/synapse/issues/6164)) -- Modify CAPTCHA_SETUP.md to update the terms `private key` and `public key` to `secret key` and `site key` respectively. Contributed by Yash Jipkate. ([\#6257](https://github.com/matrix-org/synapse/issues/6257)) -- Update `INSTALL.md` Email section to talk about `account_threepid_delegates`. ([\#6272](https://github.com/matrix-org/synapse/issues/6272)) -- Fix a small typo in `account_threepid_delegates` configuration option. ([\#6273](https://github.com/matrix-org/synapse/issues/6273)) - - -Internal Changes ----------------- - -- Add a CI job to test the `synapse_port_db` script. ([\#6140](https://github.com/matrix-org/synapse/issues/6140), [\#6276](https://github.com/matrix-org/synapse/issues/6276)) -- Convert EventContext to an attrs. ([\#6218](https://github.com/matrix-org/synapse/issues/6218)) -- Move `persist_events` out from main data store. ([\#6240](https://github.com/matrix-org/synapse/issues/6240), [\#6300](https://github.com/matrix-org/synapse/issues/6300)) -- Reduce verbosity of user/room stats. ([\#6250](https://github.com/matrix-org/synapse/issues/6250)) -- Reduce impact of debug logging. ([\#6251](https://github.com/matrix-org/synapse/issues/6251)) -- Expose some homeserver functionality to spam checkers. ([\#6259](https://github.com/matrix-org/synapse/issues/6259)) -- Change cache descriptors to always return deferreds. ([\#6263](https://github.com/matrix-org/synapse/issues/6263), [\#6291](https://github.com/matrix-org/synapse/issues/6291)) -- Fix incorrect comment regarding the functionality of an `if` statement. ([\#6269](https://github.com/matrix-org/synapse/issues/6269)) -- Update CI to run `isort` over the `scripts` and `scripts-dev` directories. ([\#6270](https://github.com/matrix-org/synapse/issues/6270)) -- Replace every instance of `logger.warn` method with `logger.warning` as the former is deprecated. ([\#6271](https://github.com/matrix-org/synapse/issues/6271), [\#6314](https://github.com/matrix-org/synapse/issues/6314)) -- Port replication http server endpoints to async/await. ([\#6274](https://github.com/matrix-org/synapse/issues/6274)) -- Port room rest handlers to async/await. ([\#6275](https://github.com/matrix-org/synapse/issues/6275)) -- Remove redundant CLI parameters on CI's `flake8` step. ([\#6277](https://github.com/matrix-org/synapse/issues/6277)) -- Port `federation_server.py` to async/await. ([\#6279](https://github.com/matrix-org/synapse/issues/6279)) -- Port receipt and read markers to async/wait. ([\#6280](https://github.com/matrix-org/synapse/issues/6280)) -- Split out state storage into separate data store. ([\#6294](https://github.com/matrix-org/synapse/issues/6294), [\#6295](https://github.com/matrix-org/synapse/issues/6295)) -- Refactor EventContext for clarity. ([\#6298](https://github.com/matrix-org/synapse/issues/6298)) -- Update the version of black used to 19.10b0. ([\#6304](https://github.com/matrix-org/synapse/issues/6304)) -- Add some documentation about worker replication. ([\#6305](https://github.com/matrix-org/synapse/issues/6305)) -- Move admin endpoints into separate files. Contributed by Awesome Technologies Innovationslabor GmbH. ([\#6308](https://github.com/matrix-org/synapse/issues/6308)) -- Document the use of `lint.sh` for code style enforcement & extend it to run on specified paths only. ([\#6312](https://github.com/matrix-org/synapse/issues/6312)) -- Add optional python dependencies and dependant binary libraries to snapcraft packaging. ([\#6317](https://github.com/matrix-org/synapse/issues/6317)) -- Remove the dependency on psutil and replace functionality with the stdlib `resource` module. ([\#6318](https://github.com/matrix-org/synapse/issues/6318), [\#6336](https://github.com/matrix-org/synapse/issues/6336)) -- Improve documentation for EventContext fields. ([\#6319](https://github.com/matrix-org/synapse/issues/6319)) -- Add some checks that we aren't using state from rejected events. ([\#6330](https://github.com/matrix-org/synapse/issues/6330)) -- Add continuous integration for python 3.8. ([\#6341](https://github.com/matrix-org/synapse/issues/6341)) -- Correct spacing/case of various instances of the word "homeserver". ([\#6357](https://github.com/matrix-org/synapse/issues/6357)) -- Temporarily blacklist the failing unit test PurgeRoomTestCase.test_purge_room. ([\#6361](https://github.com/matrix-org/synapse/issues/6361)) - - -Synapse 1.5.1 (2019-11-06) -========================== - -Features --------- - -- Limit the length of data returned by url previews, to prevent DoS attacks. ([\#6331](https://github.com/matrix-org/synapse/issues/6331), [\#6334](https://github.com/matrix-org/synapse/issues/6334)) - - -Synapse 1.5.0 (2019-10-29) -========================== - -Security updates ----------------- - -This release includes a security fix ([\#6262](https://github.com/matrix-org/synapse/issues/6262), below). Administrators are encouraged to upgrade as soon as possible. - -Bugfixes --------- - -- Fix bug where room directory search was case sensitive. ([\#6268](https://github.com/matrix-org/synapse/issues/6268)) - - -Synapse 1.5.0rc2 (2019-10-28) -============================= - -Bugfixes --------- - -- Update list of boolean columns in `synapse_port_db`. ([\#6247](https://github.com/matrix-org/synapse/issues/6247)) -- Fix /keys/query API on workers. ([\#6256](https://github.com/matrix-org/synapse/issues/6256)) -- Improve signature checking on some federation APIs. ([\#6262](https://github.com/matrix-org/synapse/issues/6262)) - - -Internal Changes ----------------- - -- Move schema delta files to the correct data store. ([\#6248](https://github.com/matrix-org/synapse/issues/6248)) -- Small performance improvement by removing repeated config lookups in room stats calculation. ([\#6255](https://github.com/matrix-org/synapse/issues/6255)) - - -Synapse 1.5.0rc1 (2019-10-24) -========================== - -Features --------- - -- Improve quality of thumbnails for 1-bit/8-bit color palette images. ([\#2142](https://github.com/matrix-org/synapse/issues/2142)) -- Add ability to upload cross-signing signatures. ([\#5726](https://github.com/matrix-org/synapse/issues/5726)) -- Allow uploading of cross-signing keys. ([\#5769](https://github.com/matrix-org/synapse/issues/5769)) -- CAS login now provides a default display name for users if a `displayname_attribute` is set in the configuration file. ([\#6114](https://github.com/matrix-org/synapse/issues/6114)) -- Reject all pending invites for a user during deactivation. ([\#6125](https://github.com/matrix-org/synapse/issues/6125)) -- Add config option to suppress client side resource limit alerting. ([\#6173](https://github.com/matrix-org/synapse/issues/6173)) - - -Bugfixes --------- - -- Return an HTTP 404 instead of 400 when requesting a filter by ID that is unknown to the server. Thanks to @krombel for contributing this! ([\#2380](https://github.com/matrix-org/synapse/issues/2380)) -- Fix a bug where users could be invited twice to the same group. ([\#3436](https://github.com/matrix-org/synapse/issues/3436)) -- Fix `/createRoom` failing with badly-formatted MXIDs in the invitee list. Thanks to @wener291! ([\#4088](https://github.com/matrix-org/synapse/issues/4088)) -- Make the `synapse_port_db` script create the right indexes on a new PostgreSQL database. ([\#6102](https://github.com/matrix-org/synapse/issues/6102), [\#6178](https://github.com/matrix-org/synapse/issues/6178), [\#6243](https://github.com/matrix-org/synapse/issues/6243)) -- Fix bug when uploading a large file: Synapse responds with `M_UNKNOWN` while it should be `M_TOO_LARGE` according to spec. Contributed by Anshul Angaria. ([\#6109](https://github.com/matrix-org/synapse/issues/6109)) -- Fix user push rules being deleted from a room when it is upgraded. ([\#6144](https://github.com/matrix-org/synapse/issues/6144)) -- Don't 500 when trying to exchange a revoked 3PID invite. ([\#6147](https://github.com/matrix-org/synapse/issues/6147)) -- Fix transferring notifications and tags when joining an upgraded room that is new to your server. ([\#6155](https://github.com/matrix-org/synapse/issues/6155)) -- Fix bug where guest account registration can wedge after restart. ([\#6161](https://github.com/matrix-org/synapse/issues/6161)) -- Fix monthly active user reaping when reserved users are specified. ([\#6168](https://github.com/matrix-org/synapse/issues/6168)) -- Fix `/federation/v1/state` endpoint not supporting newer room versions. ([\#6170](https://github.com/matrix-org/synapse/issues/6170)) -- Fix bug where we were updating censored events as bytes rather than text, occaisonally causing invalid JSON being inserted breaking APIs that attempted to fetch such events. ([\#6186](https://github.com/matrix-org/synapse/issues/6186)) -- Fix occasional missed updates in the room and user directories. ([\#6187](https://github.com/matrix-org/synapse/issues/6187)) -- Fix tracing of non-JSON APIs, `/media`, `/key` etc. ([\#6195](https://github.com/matrix-org/synapse/issues/6195)) -- Fix bug where presence would not get timed out correctly if a synchrotron worker is used and restarted. ([\#6212](https://github.com/matrix-org/synapse/issues/6212)) -- synapse_port_db: Add 2 additional BOOLEAN_COLUMNS to be able to convert from database schema v56. ([\#6216](https://github.com/matrix-org/synapse/issues/6216)) -- Fix a bug where the Synapse demo script blacklisted `::1` (ipv6 localhost) from receiving federation traffic. ([\#6229](https://github.com/matrix-org/synapse/issues/6229)) - - -Updates to the Docker image ---------------------------- - -- Fix logging getting lost for the docker image. ([\#6197](https://github.com/matrix-org/synapse/issues/6197)) - - -Internal Changes ----------------- - -- Update `user_filters` table to have a unique index, and non-null columns. Thanks to @pik for contributing this. ([\#1172](https://github.com/matrix-org/synapse/issues/1172), [\#6175](https://github.com/matrix-org/synapse/issues/6175), [\#6184](https://github.com/matrix-org/synapse/issues/6184)) -- Allow devices to be marked as hidden, for use by features such as cross-signing. - This adds a new field with a default value to the devices field in the database, - and so the database upgrade may take a long time depending on how many devices - are in the database. ([\#5759](https://github.com/matrix-org/synapse/issues/5759)) -- Move lookup-related functions from RoomMemberHandler to IdentityHandler. ([\#5978](https://github.com/matrix-org/synapse/issues/5978)) -- Improve performance of the public room list directory. ([\#6019](https://github.com/matrix-org/synapse/issues/6019), [\#6152](https://github.com/matrix-org/synapse/issues/6152), [\#6153](https://github.com/matrix-org/synapse/issues/6153), [\#6154](https://github.com/matrix-org/synapse/issues/6154)) -- Edit header dicts docstrings in `SimpleHttpClient` to note that `str` or `bytes` can be passed as header keys. ([\#6077](https://github.com/matrix-org/synapse/issues/6077)) -- Add snapcraft packaging information. Contributed by @devec0. ([\#6084](https://github.com/matrix-org/synapse/issues/6084), [\#6191](https://github.com/matrix-org/synapse/issues/6191)) -- Kill off half-implemented password-reset via sms. ([\#6101](https://github.com/matrix-org/synapse/issues/6101)) -- Remove `get_user_by_req` opentracing span and add some tags. ([\#6108](https://github.com/matrix-org/synapse/issues/6108)) -- Drop some unused database tables. ([\#6115](https://github.com/matrix-org/synapse/issues/6115)) -- Add env var to turn on tracking of log context changes. ([\#6127](https://github.com/matrix-org/synapse/issues/6127)) -- Refactor configuration loading to allow better typechecking. ([\#6137](https://github.com/matrix-org/synapse/issues/6137)) -- Log responder when responding to media request. ([\#6139](https://github.com/matrix-org/synapse/issues/6139)) -- Improve performance of `find_next_generated_user_id` DB query. ([\#6148](https://github.com/matrix-org/synapse/issues/6148)) -- Expand type-checking on modules imported by `synapse.config`. ([\#6150](https://github.com/matrix-org/synapse/issues/6150)) -- Use Postgres ANY for selecting many values. ([\#6156](https://github.com/matrix-org/synapse/issues/6156)) -- Add more caching to `_get_joined_users_from_context` DB query. ([\#6159](https://github.com/matrix-org/synapse/issues/6159)) -- Add some metrics on the federation sender. ([\#6160](https://github.com/matrix-org/synapse/issues/6160)) -- Add some logging to the rooms stats updates, to try to track down a flaky test. ([\#6167](https://github.com/matrix-org/synapse/issues/6167)) -- Remove unused `timeout` parameter from `_get_public_room_list`. ([\#6179](https://github.com/matrix-org/synapse/issues/6179)) -- Reject (accidental) attempts to insert bytes into postgres tables. ([\#6186](https://github.com/matrix-org/synapse/issues/6186)) -- Make `version` optional in body of `PUT /room_keys/version/{version}`, since it's redundant. ([\#6189](https://github.com/matrix-org/synapse/issues/6189)) -- Make storage layer responsible for adding device names to key, rather than the handler. ([\#6193](https://github.com/matrix-org/synapse/issues/6193)) -- Port `synapse.rest.admin` module to use async/await. ([\#6196](https://github.com/matrix-org/synapse/issues/6196)) -- Enforce that all boolean configuration values are lowercase in CI. ([\#6203](https://github.com/matrix-org/synapse/issues/6203)) -- Remove some unused event-auth code. ([\#6214](https://github.com/matrix-org/synapse/issues/6214)) -- Remove `Auth.check` method. ([\#6217](https://github.com/matrix-org/synapse/issues/6217)) -- Remove `format_tap.py` script in favour of a perl reimplementation in Sytest's repo. ([\#6219](https://github.com/matrix-org/synapse/issues/6219)) -- Refactor storage layer in preparation to support having multiple databases. ([\#6231](https://github.com/matrix-org/synapse/issues/6231)) -- Remove some extra quotation marks across the codebase. ([\#6236](https://github.com/matrix-org/synapse/issues/6236)) - - -Synapse 1.4.1 (2019-10-18) -========================== - -No changes since 1.4.1rc1. - - -Synapse 1.4.1rc1 (2019-10-17) -============================= - -Bugfixes --------- - -- Fix bug where redacted events were sometimes incorrectly censored in the database, breaking APIs that attempted to fetch such events. ([\#6185](https://github.com/matrix-org/synapse/issues/6185), [5b0e9948](https://github.com/matrix-org/synapse/commit/5b0e9948eaae801643e594b5abc8ee4b10bd194e)) - -Synapse 1.4.0 (2019-10-03) -========================== - -Bugfixes --------- - -- Redact `client_secret` in server logs. ([\#6158](https://github.com/matrix-org/synapse/issues/6158)) - - -Synapse 1.4.0rc2 (2019-10-02) -============================= - -Bugfixes --------- - -- Fix bug in background update that adds last seen information to the `devices` table, and improve its performance on Postgres. ([\#6135](https://github.com/matrix-org/synapse/issues/6135)) -- Fix bad performance of censoring redactions background task. ([\#6141](https://github.com/matrix-org/synapse/issues/6141)) -- Fix fetching censored redactions from DB, which caused APIs like initial sync to fail if it tried to include the censored redaction. ([\#6145](https://github.com/matrix-org/synapse/issues/6145)) -- Fix exceptions when storing large retry intervals for down remote servers. ([\#6146](https://github.com/matrix-org/synapse/issues/6146)) - - -Internal Changes ----------------- - -- Fix up sample config entry for `redaction_retention_period` option. ([\#6117](https://github.com/matrix-org/synapse/issues/6117)) - - -Synapse 1.4.0rc1 (2019-09-26) -============================= - -Note that this release includes significant changes around 3pid -verification. Administrators are reminded to review the [upgrade notes](docs/upgrade.md#upgrading-to-v140). - -Features --------- - -- Changes to 3pid verification: - - Add the ability to send registration emails from the homeserver rather than delegating to an identity server. ([\#5835](https://github.com/matrix-org/synapse/issues/5835), [\#5940](https://github.com/matrix-org/synapse/issues/5940), [\#5993](https://github.com/matrix-org/synapse/issues/5993), [\#5994](https://github.com/matrix-org/synapse/issues/5994), [\#5868](https://github.com/matrix-org/synapse/issues/5868)) - - Replace `trust_identity_server_for_password_resets` config option with `account_threepid_delegates`, and make the `id_server` parameteter optional on `*/requestToken` endpoints, as per [MSC2263](https://github.com/matrix-org/matrix-doc/pull/2263). ([\#5876](https://github.com/matrix-org/synapse/issues/5876), [\#5969](https://github.com/matrix-org/synapse/issues/5969), [\#6028](https://github.com/matrix-org/synapse/issues/6028)) - - Switch to using the v2 Identity Service `/lookup` API where available, with fallback to v1. (Implements [MSC2134](https://github.com/matrix-org/matrix-doc/pull/2134) plus `id_access_token authentication` for v2 Identity Service APIs from [MSC2140](https://github.com/matrix-org/matrix-doc/pull/2140)). ([\#5897](https://github.com/matrix-org/synapse/issues/5897)) - - Remove `bind_email` and `bind_msisdn` parameters from `/register` ala [MSC2140](https://github.com/matrix-org/matrix-doc/pull/2140). ([\#5964](https://github.com/matrix-org/synapse/issues/5964)) - - Add `m.id_access_token` to `unstable_features` in `/versions` as per [MSC2264](https://github.com/matrix-org/matrix-doc/pull/2264). ([\#5974](https://github.com/matrix-org/synapse/issues/5974)) - - Use the v2 Identity Service API for 3PID invites. ([\#5979](https://github.com/matrix-org/synapse/issues/5979)) - - Add `POST /_matrix/client/unstable/account/3pid/unbind` endpoint from [MSC2140](https://github.com/matrix-org/matrix-doc/pull/2140) for unbinding a 3PID from an identity server without removing it from the homeserver user account. ([\#5980](https://github.com/matrix-org/synapse/issues/5980), [\#6062](https://github.com/matrix-org/synapse/issues/6062)) - - Use `account_threepid_delegate.email` and `account_threepid_delegate.msisdn` for validating threepid sessions. ([\#6011](https://github.com/matrix-org/synapse/issues/6011)) - - Allow homeserver to handle or delegate email validation when adding an email to a user's account. ([\#6042](https://github.com/matrix-org/synapse/issues/6042)) - - Implement new Client Server API endpoints `/account/3pid/add` and `/account/3pid/bind` as per [MSC2290](https://github.com/matrix-org/matrix-doc/pull/2290). ([\#6043](https://github.com/matrix-org/synapse/issues/6043)) - - Add an unstable feature flag for separate add/bind 3pid APIs. ([\#6044](https://github.com/matrix-org/synapse/issues/6044)) - - Remove `bind` parameter from Client Server POST `/account` endpoint as per [MSC2290](https://github.com/matrix-org/matrix-doc/pull/2290/). ([\#6067](https://github.com/matrix-org/synapse/issues/6067)) - - Add `POST /add_threepid/msisdn/submit_token` endpoint for proxying submitToken on an `account_threepid_handler`. ([\#6078](https://github.com/matrix-org/synapse/issues/6078)) - - Add `submit_url` response parameter to `*/msisdn/requestToken` endpoints. ([\#6079](https://github.com/matrix-org/synapse/issues/6079)) - - Add `m.require_identity_server` flag to /version's unstable_features. ([\#5972](https://github.com/matrix-org/synapse/issues/5972)) -- Enhancements to OpenTracing support: - - Make OpenTracing work in worker mode. ([\#5771](https://github.com/matrix-org/synapse/issues/5771)) - - Pass OpenTracing contexts between servers when transmitting EDUs. ([\#5852](https://github.com/matrix-org/synapse/issues/5852)) - - OpenTracing for device list updates. ([\#5853](https://github.com/matrix-org/synapse/issues/5853)) - - Add a tag recording a request's authenticated entity and corresponding servlet in OpenTracing. ([\#5856](https://github.com/matrix-org/synapse/issues/5856)) - - Add minimum OpenTracing for client servlets. ([\#5983](https://github.com/matrix-org/synapse/issues/5983)) - - Check at setup that OpenTracing is installed if it's enabled in the config. ([\#5985](https://github.com/matrix-org/synapse/issues/5985)) - - Trace replication send times. ([\#5986](https://github.com/matrix-org/synapse/issues/5986)) - - Include missing OpenTracing contexts in outbout replication requests. ([\#5982](https://github.com/matrix-org/synapse/issues/5982)) - - Fix sending of EDUs when OpenTracing is enabled with an empty whitelist. ([\#5984](https://github.com/matrix-org/synapse/issues/5984)) - - Fix invalid references to None while OpenTracing if the log context slips. ([\#5988](https://github.com/matrix-org/synapse/issues/5988), [\#5991](https://github.com/matrix-org/synapse/issues/5991)) - - OpenTracing for room and e2e keys. ([\#5855](https://github.com/matrix-org/synapse/issues/5855)) - - Add OpenTracing span over HTTP push processing. ([\#6003](https://github.com/matrix-org/synapse/issues/6003)) -- Add an admin API to purge old rooms from the database. ([\#5845](https://github.com/matrix-org/synapse/issues/5845)) -- Retry well-known lookups if we have recently seen a valid well-known record for the server. ([\#5850](https://github.com/matrix-org/synapse/issues/5850)) -- Add support for filtered room-directory search requests over federation ([MSC2197](https://github.com/matrix-org/matrix-doc/pull/2197), in order to allow upcoming room directory query performance improvements. ([\#5859](https://github.com/matrix-org/synapse/issues/5859)) -- Correctly retry all hosts returned from SRV when we fail to connect. ([\#5864](https://github.com/matrix-org/synapse/issues/5864)) -- Add admin API endpoint for setting whether or not a user is a server administrator. ([\#5878](https://github.com/matrix-org/synapse/issues/5878)) -- Enable cleaning up extremities with dummy events by default to prevent undue build up of forward extremities. ([\#5884](https://github.com/matrix-org/synapse/issues/5884)) -- Add config option to sign remote key query responses with a separate key. ([\#5895](https://github.com/matrix-org/synapse/issues/5895)) -- Add support for config templating. ([\#5900](https://github.com/matrix-org/synapse/issues/5900)) -- Users with the type of "support" or "bot" are no longer required to consent. ([\#5902](https://github.com/matrix-org/synapse/issues/5902)) -- Let synctl accept a directory of config files. ([\#5904](https://github.com/matrix-org/synapse/issues/5904)) -- Increase max display name size to 256. ([\#5906](https://github.com/matrix-org/synapse/issues/5906)) -- Add admin API endpoint for getting whether or not a user is a server administrator. ([\#5914](https://github.com/matrix-org/synapse/issues/5914)) -- Redact events in the database that have been redacted for a week. ([\#5934](https://github.com/matrix-org/synapse/issues/5934)) -- New prometheus metrics: - - `synapse_federation_known_servers`: represents the total number of servers your server knows about (i.e. is in rooms with), including itself. Enable by setting `metrics_flags.known_servers` to True in the configuration.([\#5981](https://github.com/matrix-org/synapse/issues/5981)) - - `synapse_build_info`: exposes the Python version, OS version, and Synapse version of the running server. ([\#6005](https://github.com/matrix-org/synapse/issues/6005)) -- Give appropriate exit codes when synctl fails. ([\#5992](https://github.com/matrix-org/synapse/issues/5992)) -- Apply the federation blacklist to requests to identity servers. ([\#6000](https://github.com/matrix-org/synapse/issues/6000)) -- Add `report_stats_endpoint` option to configure where stats are reported to, if enabled. Contributed by @Sorunome. ([\#6012](https://github.com/matrix-org/synapse/issues/6012)) -- Add config option to increase ratelimits for room admins redacting messages. ([\#6015](https://github.com/matrix-org/synapse/issues/6015)) -- Stop sending federation transactions to servers which have been down for a long time. ([\#6026](https://github.com/matrix-org/synapse/issues/6026)) -- Make the process for mapping SAML2 users to matrix IDs more flexible. ([\#6037](https://github.com/matrix-org/synapse/issues/6037)) -- Return a clearer error message when a timeout occurs when attempting to contact an identity server. ([\#6073](https://github.com/matrix-org/synapse/issues/6073)) -- Prevent password reset's submit_token endpoint from accepting trailing slashes. ([\#6074](https://github.com/matrix-org/synapse/issues/6074)) -- Return 403 on `/register/available` if registration has been disabled. ([\#6082](https://github.com/matrix-org/synapse/issues/6082)) -- Explicitly log when a homeserver does not have the `trusted_key_servers` config field configured. ([\#6090](https://github.com/matrix-org/synapse/issues/6090)) -- Add support for pruning old rows in `user_ips` table. ([\#6098](https://github.com/matrix-org/synapse/issues/6098)) - -Bugfixes --------- - -- Don't create broken room when `power_level_content_override.users` does not contain `creator_id`. ([\#5633](https://github.com/matrix-org/synapse/issues/5633)) -- Fix database index so that different backup versions can have the same sessions. ([\#5857](https://github.com/matrix-org/synapse/issues/5857)) -- Fix Synapse looking for config options `password_reset_failure_template` and `password_reset_success_template`, when they are actually `password_reset_template_failure_html`, `password_reset_template_success_html`. ([\#5863](https://github.com/matrix-org/synapse/issues/5863)) -- Fix stack overflow when recovering an appservice which had an outage. ([\#5885](https://github.com/matrix-org/synapse/issues/5885)) -- Fix error message which referred to `public_base_url` instead of `public_baseurl`. Thanks to @aaronraimist for the fix! ([\#5909](https://github.com/matrix-org/synapse/issues/5909)) -- Fix 404 for thumbnail download when `dynamic_thumbnails` is `false` and the thumbnail was dynamically generated. Fix reported by rkfg. ([\#5915](https://github.com/matrix-org/synapse/issues/5915)) -- Fix a cache-invalidation bug for worker-based deployments. ([\#5920](https://github.com/matrix-org/synapse/issues/5920)) -- Fix admin API for listing media in a room not being available with an external media repo. ([\#5966](https://github.com/matrix-org/synapse/issues/5966)) -- Fix list media admin API always returning an error. ([\#5967](https://github.com/matrix-org/synapse/issues/5967)) -- Fix room and user stats tracking. ([\#5971](https://github.com/matrix-org/synapse/issues/5971), [\#5998](https://github.com/matrix-org/synapse/issues/5998), [\#6029](https://github.com/matrix-org/synapse/issues/6029)) -- Return a `M_MISSING_PARAM` if `sid` is not provided to `/account/3pid`. ([\#5995](https://github.com/matrix-org/synapse/issues/5995)) -- `federation_certificate_verification_whitelist` now will not cause `TypeErrors` to be raised (a regression in 1.3). Additionally, it now supports internationalised domain names in their non-canonical representation. ([\#5996](https://github.com/matrix-org/synapse/issues/5996)) -- Only count real users when checking for auto-creation of auto-join room. ([\#6004](https://github.com/matrix-org/synapse/issues/6004)) -- Ensure support users can be registered even if MAU limit is reached. ([\#6020](https://github.com/matrix-org/synapse/issues/6020)) -- Fix bug where login error was shown incorrectly on SSO fallback login. ([\#6024](https://github.com/matrix-org/synapse/issues/6024)) -- Fix bug in calculating the federation retry backoff period. ([\#6025](https://github.com/matrix-org/synapse/issues/6025)) -- Prevent exceptions being logged when extremity-cleanup events fail due to lack of user consent to the terms of service. ([\#6053](https://github.com/matrix-org/synapse/issues/6053)) -- Remove POST method from password-reset `submit_token` endpoint until we implement `submit_url` functionality. ([\#6056](https://github.com/matrix-org/synapse/issues/6056)) -- Fix logcontext spam on non-Linux platforms. ([\#6059](https://github.com/matrix-org/synapse/issues/6059)) -- Ensure query parameters in email validation links are URL-encoded. ([\#6063](https://github.com/matrix-org/synapse/issues/6063)) -- Fix a bug which caused SAML attribute maps to be overridden by defaults. ([\#6069](https://github.com/matrix-org/synapse/issues/6069)) -- Fix the logged number of updated items for the `users_set_deactivated_flag` background update. ([\#6092](https://github.com/matrix-org/synapse/issues/6092)) -- Add `sid` to `next_link` for email validation. ([\#6097](https://github.com/matrix-org/synapse/issues/6097)) -- Threepid validity checks on msisdns should not be dependent on `threepid_behaviour_email`. ([\#6104](https://github.com/matrix-org/synapse/issues/6104)) -- Ensure that servers which are not configured to support email address verification do not offer it in the registration flows. ([\#6107](https://github.com/matrix-org/synapse/issues/6107)) - - -Updates to the Docker image ---------------------------- - -- Avoid changing `UID/GID` if they are already correct. ([\#5970](https://github.com/matrix-org/synapse/issues/5970)) -- Provide `SYNAPSE_WORKER` envvar to specify python module. ([\#6058](https://github.com/matrix-org/synapse/issues/6058)) - - -Improved Documentation ----------------------- - -- Convert documentation to markdown (from rst) ([\#5849](https://github.com/matrix-org/synapse/issues/5849)) -- Update `INSTALL.md` to say that Python 2 is no longer supported. ([\#5953](https://github.com/matrix-org/synapse/issues/5953)) -- Add developer documentation for using SAML2. ([\#6032](https://github.com/matrix-org/synapse/issues/6032)) -- Add some notes on rolling back to v1.3.1. ([\#6049](https://github.com/matrix-org/synapse/issues/6049)) -- Update the upgrade notes. ([\#6050](https://github.com/matrix-org/synapse/issues/6050)) - - -Deprecations and Removals -------------------------- - -- Remove shared-secret registration from `/_matrix/client/r0/register` endpoint. Contributed by Awesome Technologies Innovationslabor GmbH. ([\#5877](https://github.com/matrix-org/synapse/issues/5877)) -- Deprecate the `trusted_third_party_id_servers` option. ([\#5875](https://github.com/matrix-org/synapse/issues/5875)) - - -Internal Changes ----------------- - -- Lay the groundwork for structured logging output. ([\#5680](https://github.com/matrix-org/synapse/issues/5680)) -- Retry well-known lookup before the cache expires, giving a grace period where the remote well-known can be down but we still use the old result. ([\#5844](https://github.com/matrix-org/synapse/issues/5844)) -- Remove log line for debugging issue #5407. ([\#5860](https://github.com/matrix-org/synapse/issues/5860)) -- Refactor the Appservice scheduler code. ([\#5886](https://github.com/matrix-org/synapse/issues/5886)) -- Compatibility with v2 Identity Service APIs other than /lookup. ([\#5892](https://github.com/matrix-org/synapse/issues/5892), [\#6013](https://github.com/matrix-org/synapse/issues/6013)) -- Stop populating some unused tables. ([\#5893](https://github.com/matrix-org/synapse/issues/5893), [\#6047](https://github.com/matrix-org/synapse/issues/6047)) -- Add missing index on `users_in_public_rooms` to improve the performance of directory queries. ([\#5894](https://github.com/matrix-org/synapse/issues/5894)) -- Improve the logging when we have an error when fetching signing keys. ([\#5896](https://github.com/matrix-org/synapse/issues/5896)) -- Add support for database engine-specific schema deltas, based on file extension. ([\#5911](https://github.com/matrix-org/synapse/issues/5911)) -- Update Buildkite pipeline to use plugins instead of buildkite-agent commands. ([\#5922](https://github.com/matrix-org/synapse/issues/5922)) -- Add link in sample config to the logging config schema. ([\#5926](https://github.com/matrix-org/synapse/issues/5926)) -- Remove unnecessary parentheses in return statements. ([\#5931](https://github.com/matrix-org/synapse/issues/5931)) -- Remove unused `jenkins/prepare_sytest.sh` file. ([\#5938](https://github.com/matrix-org/synapse/issues/5938)) -- Move Buildkite pipeline config to the pipelines repo. ([\#5943](https://github.com/matrix-org/synapse/issues/5943)) -- Remove unnecessary return statements in the codebase which were the result of a regex run. ([\#5962](https://github.com/matrix-org/synapse/issues/5962)) -- Remove left-over methods from v1 registration API. ([\#5963](https://github.com/matrix-org/synapse/issues/5963)) -- Cleanup event auth type initialisation. ([\#5975](https://github.com/matrix-org/synapse/issues/5975)) -- Clean up dependency checking at setup. ([\#5989](https://github.com/matrix-org/synapse/issues/5989)) -- Update OpenTracing docs to use the unified `trace` method. ([\#5776](https://github.com/matrix-org/synapse/issues/5776)) -- Small refactor of function arguments and docstrings in` RoomMemberHandler`. ([\#6009](https://github.com/matrix-org/synapse/issues/6009)) -- Remove unused `origin` argument on `FederationHandler.add_display_name_to_third_party_invite`. ([\#6010](https://github.com/matrix-org/synapse/issues/6010)) -- Add a `failure_ts` column to the `destinations` database table. ([\#6016](https://github.com/matrix-org/synapse/issues/6016), [\#6072](https://github.com/matrix-org/synapse/issues/6072)) -- Clean up some code in the retry logic. ([\#6017](https://github.com/matrix-org/synapse/issues/6017)) -- Fix the structured logging tests stomping on the global log configuration for subsequent tests. ([\#6023](https://github.com/matrix-org/synapse/issues/6023)) -- Clean up the sample config for SAML authentication. ([\#6064](https://github.com/matrix-org/synapse/issues/6064)) -- Change mailer logging to reflect Synapse doesn't just do chat notifications by email now. ([\#6075](https://github.com/matrix-org/synapse/issues/6075)) -- Move last-seen info into devices table. ([\#6089](https://github.com/matrix-org/synapse/issues/6089)) -- Remove unused parameter to `get_user_id_by_threepid`. ([\#6099](https://github.com/matrix-org/synapse/issues/6099)) -- Refactor the user-interactive auth handling. ([\#6105](https://github.com/matrix-org/synapse/issues/6105)) -- Refactor code for calculating registration flows. ([\#6106](https://github.com/matrix-org/synapse/issues/6106)) - - -Synapse 1.3.1 (2019-08-17) -========================== - -Features --------- - -- Drop hard dependency on `sdnotify` python package. ([\#5871](https://github.com/matrix-org/synapse/issues/5871)) - - -Bugfixes --------- - -- Fix startup issue (hang on ACME provisioning) due to ordering of Twisted reactor startup. Thanks to @chrismoos for supplying the fix. ([\#5867](https://github.com/matrix-org/synapse/issues/5867)) - - -Synapse 1.3.0 (2019-08-15) -========================== - -Bugfixes --------- - -- Fix 500 Internal Server Error on `publicRooms` when the public room list was - cached. ([\#5851](https://github.com/matrix-org/synapse/issues/5851)) - - -Synapse 1.3.0rc1 (2019-08-13) -========================== - -Features --------- - -- Use `M_USER_DEACTIVATED` instead of `M_UNKNOWN` for errcode when a deactivated user attempts to login. ([\#5686](https://github.com/matrix-org/synapse/issues/5686)) -- Add sd_notify hooks to ease systemd integration and allows usage of Type=Notify. ([\#5732](https://github.com/matrix-org/synapse/issues/5732)) -- Synapse will no longer serve any media repo admin endpoints when `enable_media_repo` is set to False in the configuration. If a media repo worker is used, the admin APIs relating to the media repo will be served from it instead. ([\#5754](https://github.com/matrix-org/synapse/issues/5754), [\#5848](https://github.com/matrix-org/synapse/issues/5848)) -- Synapse can now be configured to not join remote rooms of a given "complexity" (currently, state events) over federation. This option can be used to prevent adverse performance on resource-constrained homeservers. ([\#5783](https://github.com/matrix-org/synapse/issues/5783)) -- Allow defining HTML templates to serve the user on account renewal attempt when using the account validity feature. ([\#5807](https://github.com/matrix-org/synapse/issues/5807)) - - -Bugfixes --------- - -- Fix UISIs during homeserver outage. ([\#5693](https://github.com/matrix-org/synapse/issues/5693), [\#5789](https://github.com/matrix-org/synapse/issues/5789)) -- Fix stack overflow in server key lookup code. ([\#5724](https://github.com/matrix-org/synapse/issues/5724)) -- start.sh no longer uses deprecated cli option. ([\#5725](https://github.com/matrix-org/synapse/issues/5725)) -- Log when we receive an event receipt from an unexpected origin. ([\#5743](https://github.com/matrix-org/synapse/issues/5743)) -- Fix debian packaging scripts to correctly build sid packages. ([\#5775](https://github.com/matrix-org/synapse/issues/5775)) -- Correctly handle redactions of redactions. ([\#5788](https://github.com/matrix-org/synapse/issues/5788)) -- Return 404 instead of 403 when accessing /rooms/{roomId}/event/{eventId} for an event without the appropriate permissions. ([\#5798](https://github.com/matrix-org/synapse/issues/5798)) -- Fix check that tombstone is a state event in push rules. ([\#5804](https://github.com/matrix-org/synapse/issues/5804)) -- Fix error when trying to login as a deactivated user when using a worker to handle login. ([\#5806](https://github.com/matrix-org/synapse/issues/5806)) -- Fix bug where user `/sync` stream could get wedged in rare circumstances. ([\#5825](https://github.com/matrix-org/synapse/issues/5825)) -- The purge_remote_media.sh script was fixed. ([\#5839](https://github.com/matrix-org/synapse/issues/5839)) - - -Deprecations and Removals -------------------------- - -- Synapse now no longer accepts the `-v`/`--verbose`, `-f`/`--log-file`, or `--log-config` command line flags, and removes the deprecated `verbose` and `log_file` configuration file options. Users of these options should migrate their options into the dedicated log configuration. ([\#5678](https://github.com/matrix-org/synapse/issues/5678), [\#5729](https://github.com/matrix-org/synapse/issues/5729)) -- Remove non-functional 'expire_access_token' setting. ([\#5782](https://github.com/matrix-org/synapse/issues/5782)) - - -Internal Changes ----------------- - -- Make Jaeger fully configurable. ([\#5694](https://github.com/matrix-org/synapse/issues/5694)) -- Add precautionary measures to prevent future abuse of `window.opener` in default welcome page. ([\#5695](https://github.com/matrix-org/synapse/issues/5695)) -- Reduce database IO usage by optimising queries for current membership. ([\#5706](https://github.com/matrix-org/synapse/issues/5706), [\#5738](https://github.com/matrix-org/synapse/issues/5738), [\#5746](https://github.com/matrix-org/synapse/issues/5746), [\#5752](https://github.com/matrix-org/synapse/issues/5752), [\#5770](https://github.com/matrix-org/synapse/issues/5770), [\#5774](https://github.com/matrix-org/synapse/issues/5774), [\#5792](https://github.com/matrix-org/synapse/issues/5792), [\#5793](https://github.com/matrix-org/synapse/issues/5793)) -- Improve caching when fetching `get_filtered_current_state_ids`. ([\#5713](https://github.com/matrix-org/synapse/issues/5713)) -- Don't accept opentracing data from clients. ([\#5715](https://github.com/matrix-org/synapse/issues/5715)) -- Speed up PostgreSQL unit tests in CI. ([\#5717](https://github.com/matrix-org/synapse/issues/5717)) -- Update the coding style document. ([\#5719](https://github.com/matrix-org/synapse/issues/5719)) -- Improve database query performance when recording retry intervals for remote hosts. ([\#5720](https://github.com/matrix-org/synapse/issues/5720)) -- Add a set of opentracing utils. ([\#5722](https://github.com/matrix-org/synapse/issues/5722)) -- Cache result of get_version_string to reduce overhead of `/version` federation requests. ([\#5730](https://github.com/matrix-org/synapse/issues/5730)) -- Return 'user_type' in admin API user endpoints results. ([\#5731](https://github.com/matrix-org/synapse/issues/5731)) -- Don't package the sytest test blacklist file. ([\#5733](https://github.com/matrix-org/synapse/issues/5733)) -- Replace uses of returnValue with plain return, as returnValue is not needed on Python 3. ([\#5736](https://github.com/matrix-org/synapse/issues/5736)) -- Blacklist some flakey tests in worker mode. ([\#5740](https://github.com/matrix-org/synapse/issues/5740)) -- Fix some error cases in the caching layer. ([\#5749](https://github.com/matrix-org/synapse/issues/5749)) -- Add a prometheus metric for pending cache lookups. ([\#5750](https://github.com/matrix-org/synapse/issues/5750)) -- Stop trying to fetch events with event_id=None. ([\#5753](https://github.com/matrix-org/synapse/issues/5753)) -- Convert RedactionTestCase to modern test style. ([\#5768](https://github.com/matrix-org/synapse/issues/5768)) -- Allow looping calls to be given arguments. ([\#5780](https://github.com/matrix-org/synapse/issues/5780)) -- Set the logs emitted when checking typing and presence timeouts to DEBUG level, not INFO. ([\#5785](https://github.com/matrix-org/synapse/issues/5785)) -- Remove DelayedCall debugging from the test suite, as it is no longer required in the vast majority of Synapse's tests. ([\#5787](https://github.com/matrix-org/synapse/issues/5787)) -- Remove some spurious exceptions from the logs where we failed to talk to a remote server. ([\#5790](https://github.com/matrix-org/synapse/issues/5790)) -- Improve performance when making `.well-known` requests by sharing the SSL options between requests. ([\#5794](https://github.com/matrix-org/synapse/issues/5794)) -- Disable codecov GitHub comments on PRs. ([\#5796](https://github.com/matrix-org/synapse/issues/5796)) -- Don't allow clients to send tombstone events that reference the room it's sent in. ([\#5801](https://github.com/matrix-org/synapse/issues/5801)) -- Deny redactions of events sent in a different room. ([\#5802](https://github.com/matrix-org/synapse/issues/5802)) -- Deny sending well known state types as non-state events. ([\#5805](https://github.com/matrix-org/synapse/issues/5805)) -- Handle incorrectly encoded query params correctly by returning a 400. ([\#5808](https://github.com/matrix-org/synapse/issues/5808)) -- Handle pusher being deleted during processing rather than logging an exception. ([\#5809](https://github.com/matrix-org/synapse/issues/5809)) -- Return 502 not 500 when failing to reach any remote server. ([\#5810](https://github.com/matrix-org/synapse/issues/5810)) -- Reduce global pauses in the events stream caused by expensive state resolution during persistence. ([\#5826](https://github.com/matrix-org/synapse/issues/5826)) -- Add a lower bound to well-known lookup cache time to avoid repeated lookups. ([\#5836](https://github.com/matrix-org/synapse/issues/5836)) -- Whitelist history visbility sytests in worker mode tests. ([\#5843](https://github.com/matrix-org/synapse/issues/5843)) - - -Synapse 1.2.1 (2019-07-26) -========================== - -Security update ---------------- - -This release includes *four* security fixes: - -- Prevent an attack where a federated server could send redactions for arbitrary events in v1 and v2 rooms. ([\#5767](https://github.com/matrix-org/synapse/issues/5767)) -- Prevent a denial-of-service attack where cycles of redaction events would make Synapse spin infinitely. Thanks to `@lrizika:matrix.org` for identifying and responsibly disclosing this issue. ([0f2ecb961](https://github.com/matrix-org/synapse/commit/0f2ecb961)) -- Prevent an attack where users could be joined or parted from public rooms without their consent. Thanks to @dylangerdaly for identifying and responsibly disclosing this issue. ([\#5744](https://github.com/matrix-org/synapse/issues/5744)) -- Fix a vulnerability where a federated server could spoof read-receipts from - users on other servers. Thanks to @dylangerdaly for identifying this issue too. ([\#5743](https://github.com/matrix-org/synapse/issues/5743)) - -Additionally, the following fix was in Synapse **1.2.0**, but was not correctly -identified during the original release: - -- It was possible for a room moderator to send a redaction for an `m.room.create` event, which would downgrade the room to version 1. Thanks to `/dev/ponies` for identifying and responsibly disclosing this issue! ([\#5701](https://github.com/matrix-org/synapse/issues/5701)) - -Synapse 1.2.0 (2019-07-25) -========================== - -No significant changes. - - -Synapse 1.2.0rc2 (2019-07-24) -============================= - -Bugfixes --------- - -- Fix a regression introduced in v1.2.0rc1 which led to incorrect labels on some prometheus metrics. ([\#5734](https://github.com/matrix-org/synapse/issues/5734)) - - -Synapse 1.2.0rc1 (2019-07-22) -============================= - -Security fixes --------------- - -This update included a security fix which was initially incorrectly flagged as -a regular bug fix. - -- It was possible for a room moderator to send a redaction for an `m.room.create` event, which would downgrade the room to version 1. Thanks to `/dev/ponies` for identifying and responsibly disclosing this issue! ([\#5701](https://github.com/matrix-org/synapse/issues/5701)) - -Features --------- - -- Add support for opentracing. ([\#5544](https://github.com/matrix-org/synapse/issues/5544), [\#5712](https://github.com/matrix-org/synapse/issues/5712)) -- Add ability to pull all locally stored events out of synapse that a particular user can see. ([\#5589](https://github.com/matrix-org/synapse/issues/5589)) -- Add a basic admin command app to allow server operators to run Synapse admin commands separately from the main production instance. ([\#5597](https://github.com/matrix-org/synapse/issues/5597)) -- Add `sender` and `origin_server_ts` fields to `m.replace`. ([\#5613](https://github.com/matrix-org/synapse/issues/5613)) -- Add default push rule to ignore reactions. ([\#5623](https://github.com/matrix-org/synapse/issues/5623)) -- Include the original event when asking for its relations. ([\#5626](https://github.com/matrix-org/synapse/issues/5626)) -- Implement `session_lifetime` configuration option, after which access tokens will expire. ([\#5660](https://github.com/matrix-org/synapse/issues/5660)) -- Return "This account has been deactivated" when a deactivated user tries to login. ([\#5674](https://github.com/matrix-org/synapse/issues/5674)) -- Enable aggregations support by default ([\#5714](https://github.com/matrix-org/synapse/issues/5714)) - - -Bugfixes --------- - -- Fix 'utime went backwards' errors on daemonization. ([\#5609](https://github.com/matrix-org/synapse/issues/5609)) -- Various minor fixes to the federation request rate limiter. ([\#5621](https://github.com/matrix-org/synapse/issues/5621)) -- Forbid viewing relations on an event once it has been redacted. ([\#5629](https://github.com/matrix-org/synapse/issues/5629)) -- Fix requests to the `/store_invite` endpoint of identity servers being sent in the wrong format. ([\#5638](https://github.com/matrix-org/synapse/issues/5638)) -- Fix newly-registered users not being able to lookup their own profile without joining a room. ([\#5644](https://github.com/matrix-org/synapse/issues/5644)) -- Fix bug in #5626 that prevented the original_event field from actually having the contents of the original event in a call to `/relations`. ([\#5654](https://github.com/matrix-org/synapse/issues/5654)) -- Fix 3PID bind requests being sent to identity servers as `application/x-form-www-urlencoded` data, which is deprecated. ([\#5658](https://github.com/matrix-org/synapse/issues/5658)) -- Fix some problems with authenticating redactions in recent room versions. ([\#5699](https://github.com/matrix-org/synapse/issues/5699), [\#5700](https://github.com/matrix-org/synapse/issues/5700), [\#5707](https://github.com/matrix-org/synapse/issues/5707)) - - -Updates to the Docker image ---------------------------- - -- Base Docker image on a newer Alpine Linux version (3.8 -> 3.10). ([\#5619](https://github.com/matrix-org/synapse/issues/5619)) -- Add missing space in default logging file format generated by the Docker image. ([\#5620](https://github.com/matrix-org/synapse/issues/5620)) - - -Improved Documentation ----------------------- - -- Add information about nginx normalisation to reverse_proxy.rst. Contributed by @skalarproduktraum - thanks! ([\#5397](https://github.com/matrix-org/synapse/issues/5397)) -- --no-pep517 should be --no-use-pep517 in the documentation to setup the development environment. ([\#5651](https://github.com/matrix-org/synapse/issues/5651)) -- Improvements to Postgres setup instructions. Contributed by @Lrizika - thanks! ([\#5661](https://github.com/matrix-org/synapse/issues/5661)) -- Minor tweaks to postgres documentation. ([\#5675](https://github.com/matrix-org/synapse/issues/5675)) - - -Deprecations and Removals -------------------------- - -- Remove support for the `invite_3pid_guest` configuration setting. ([\#5625](https://github.com/matrix-org/synapse/issues/5625)) - - -Internal Changes ----------------- - -- Move logging code out of `synapse.util` and into `synapse.logging`. ([\#5606](https://github.com/matrix-org/synapse/issues/5606), [\#5617](https://github.com/matrix-org/synapse/issues/5617)) -- Add a blacklist file to the repo to blacklist certain sytests from failing CI. ([\#5611](https://github.com/matrix-org/synapse/issues/5611)) -- Make runtime errors surrounding password reset emails much clearer. ([\#5616](https://github.com/matrix-org/synapse/issues/5616)) -- Remove dead code for persiting outgoing federation transactions. ([\#5622](https://github.com/matrix-org/synapse/issues/5622)) -- Add `lint.sh` to the scripts-dev folder which will run all linting steps required by CI. ([\#5627](https://github.com/matrix-org/synapse/issues/5627)) -- Move RegistrationHandler.get_or_create_user to test code. ([\#5628](https://github.com/matrix-org/synapse/issues/5628)) -- Add some more common python virtual-environment paths to the black exclusion list. ([\#5630](https://github.com/matrix-org/synapse/issues/5630)) -- Some counter metrics exposed over Prometheus have been renamed, with the old names preserved for backwards compatibility and deprecated. See `docs/metrics-howto.rst` for details. ([\#5636](https://github.com/matrix-org/synapse/issues/5636)) -- Unblacklist some user_directory sytests. ([\#5637](https://github.com/matrix-org/synapse/issues/5637)) -- Factor out some redundant code in the login implementation. ([\#5639](https://github.com/matrix-org/synapse/issues/5639)) -- Update ModuleApi to avoid register(generate_token=True). ([\#5640](https://github.com/matrix-org/synapse/issues/5640)) -- Remove access-token support from `RegistrationHandler.register`, and rename it. ([\#5641](https://github.com/matrix-org/synapse/issues/5641)) -- Remove access-token support from `RegistrationStore.register`, and rename it. ([\#5642](https://github.com/matrix-org/synapse/issues/5642)) -- Improve logging for auto-join when a new user is created. ([\#5643](https://github.com/matrix-org/synapse/issues/5643)) -- Remove unused and unnecessary check for FederationDeniedError in _exception_to_failure. ([\#5645](https://github.com/matrix-org/synapse/issues/5645)) -- Fix a small typo in a code comment. ([\#5655](https://github.com/matrix-org/synapse/issues/5655)) -- Clean up exception handling around client access tokens. ([\#5656](https://github.com/matrix-org/synapse/issues/5656)) -- Add a mechanism for per-test homeserver configuration in the unit tests. ([\#5657](https://github.com/matrix-org/synapse/issues/5657)) -- Inline issue_access_token. ([\#5659](https://github.com/matrix-org/synapse/issues/5659)) -- Update the sytest BuildKite configuration to checkout Synapse in `/src`. ([\#5664](https://github.com/matrix-org/synapse/issues/5664)) -- Add a `docker` type to the towncrier configuration. ([\#5673](https://github.com/matrix-org/synapse/issues/5673)) -- Convert `synapse.federation.transport.server` to `async`. Might improve some stack traces. ([\#5689](https://github.com/matrix-org/synapse/issues/5689)) -- Documentation for opentracing. ([\#5703](https://github.com/matrix-org/synapse/issues/5703)) - - -Synapse 1.1.0 (2019-07-04) -========================== - -As of v1.1.0, Synapse no longer supports Python 2, nor Postgres version 9.4. -See the [upgrade notes](docs/upgrade.md#upgrading-to-v110) for more details. - -This release also deprecates the use of environment variables to configure the -docker image. See the [docker README](https://github.com/matrix-org/synapse/blob/release-v1.1.0/docker/README.md#legacy-dynamic-configuration-file-support) -for more details. - -No changes since 1.1.0rc2. - - -Synapse 1.1.0rc2 (2019-07-03) -============================= - -Bugfixes --------- - -- Fix regression in 1.1rc1 where OPTIONS requests to the media repo would fail. ([\#5593](https://github.com/matrix-org/synapse/issues/5593)) -- Removed the `SYNAPSE_SMTP_*` docker container environment variables. Using these environment variables prevented the docker container from starting in Synapse v1.0, even though they didn't actually allow any functionality anyway. ([\#5596](https://github.com/matrix-org/synapse/issues/5596)) -- Fix a number of "Starting txn from sentinel context" warnings. ([\#5605](https://github.com/matrix-org/synapse/issues/5605)) - - -Internal Changes ----------------- - -- Update github templates. ([\#5552](https://github.com/matrix-org/synapse/issues/5552)) - - -Synapse 1.1.0rc1 (2019-07-02) -============================= - -As of v1.1.0, Synapse no longer supports Python 2, nor Postgres version 9.4. -See the [upgrade notes](docs/upgrade.md#upgrading-to-v110) for more details. - -Features --------- - -- Added possibilty to disable local password authentication. Contributed by Daniel Hoffend. ([\#5092](https://github.com/matrix-org/synapse/issues/5092)) -- Add monthly active users to phonehome stats. ([\#5252](https://github.com/matrix-org/synapse/issues/5252)) -- Allow expired user to trigger renewal email sending manually. ([\#5363](https://github.com/matrix-org/synapse/issues/5363)) -- Statistics on forward extremities per room are now exposed via Prometheus. ([\#5384](https://github.com/matrix-org/synapse/issues/5384), [\#5458](https://github.com/matrix-org/synapse/issues/5458), [\#5461](https://github.com/matrix-org/synapse/issues/5461)) -- Add --no-daemonize option to run synapse in the foreground, per issue #4130. Contributed by Soham Gumaste. ([\#5412](https://github.com/matrix-org/synapse/issues/5412), [\#5587](https://github.com/matrix-org/synapse/issues/5587)) -- Fully support SAML2 authentication. Contributed by [Alexander Trost](https://github.com/galexrt) - thank you! ([\#5422](https://github.com/matrix-org/synapse/issues/5422)) -- Allow server admins to define implementations of extra rules for allowing or denying incoming events. ([\#5440](https://github.com/matrix-org/synapse/issues/5440), [\#5474](https://github.com/matrix-org/synapse/issues/5474), [\#5477](https://github.com/matrix-org/synapse/issues/5477)) -- Add support for handling pagination APIs on client reader worker. ([\#5505](https://github.com/matrix-org/synapse/issues/5505), [\#5513](https://github.com/matrix-org/synapse/issues/5513), [\#5531](https://github.com/matrix-org/synapse/issues/5531)) -- Improve help and cmdline option names for --generate-config options. ([\#5512](https://github.com/matrix-org/synapse/issues/5512)) -- Allow configuration of the path used for ACME account keys. ([\#5516](https://github.com/matrix-org/synapse/issues/5516), [\#5521](https://github.com/matrix-org/synapse/issues/5521), [\#5522](https://github.com/matrix-org/synapse/issues/5522)) -- Add --data-dir and --open-private-ports options. ([\#5524](https://github.com/matrix-org/synapse/issues/5524)) -- Split public rooms directory auth config in two settings, in order to manage client auth independently from the federation part of it. Obsoletes the "restrict_public_rooms_to_local_users" configuration setting. If "restrict_public_rooms_to_local_users" is set in the config, Synapse will act as if both new options are enabled, i.e. require authentication through the client API and deny federation requests. ([\#5534](https://github.com/matrix-org/synapse/issues/5534)) -- The minimum TLS version used for outgoing federation requests can now be set with `federation_client_minimum_tls_version`. ([\#5550](https://github.com/matrix-org/synapse/issues/5550)) -- Optimise devices changed query to not pull unnecessary rows from the database, reducing database load. ([\#5559](https://github.com/matrix-org/synapse/issues/5559)) -- Add new metrics for number of forward extremities being persisted and number of state groups involved in resolution. ([\#5476](https://github.com/matrix-org/synapse/issues/5476)) - -Bugfixes --------- - -- Fix bug processing incoming events over federation if call to `/get_missing_events` fails. ([\#5042](https://github.com/matrix-org/synapse/issues/5042)) -- Prevent more than one room upgrade happening simultaneously on the same room. ([\#5051](https://github.com/matrix-org/synapse/issues/5051)) -- Fix a bug where running synapse_port_db would cause the account validity feature to fail because it didn't set the type of the email_sent column to boolean. ([\#5325](https://github.com/matrix-org/synapse/issues/5325)) -- Warn about disabling email-based password resets when a reset occurs, and remove warning when someone attempts a phone-based reset. ([\#5387](https://github.com/matrix-org/synapse/issues/5387)) -- Fix email notifications for unnamed rooms with multiple people. ([\#5388](https://github.com/matrix-org/synapse/issues/5388)) -- Fix exceptions in federation reader worker caused by attempting to renew attestations, which should only happen on master worker. ([\#5389](https://github.com/matrix-org/synapse/issues/5389)) -- Fix handling of failures fetching remote content to not log failures as exceptions. ([\#5390](https://github.com/matrix-org/synapse/issues/5390)) -- Fix a bug where deactivated users could receive renewal emails if the account validity feature is on. ([\#5394](https://github.com/matrix-org/synapse/issues/5394)) -- Fix missing invite state after exchanging 3PID invites over federaton. ([\#5464](https://github.com/matrix-org/synapse/issues/5464)) -- Fix intermittent exceptions on Apple hardware. Also fix bug that caused database activity times to be under-reported in log lines. ([\#5498](https://github.com/matrix-org/synapse/issues/5498)) -- Fix logging error when a tampered event is detected. ([\#5500](https://github.com/matrix-org/synapse/issues/5500)) -- Fix bug where clients could tight loop calling `/sync` for a period. ([\#5507](https://github.com/matrix-org/synapse/issues/5507)) -- Fix bug with `jinja2` preventing Synapse from starting. Users who had this problem should now simply need to run `pip install matrix-synapse`. ([\#5514](https://github.com/matrix-org/synapse/issues/5514)) -- Fix a regression where homeservers on private IP addresses were incorrectly blacklisted. ([\#5523](https://github.com/matrix-org/synapse/issues/5523)) -- Fixed m.login.jwt using unregistred user_id and added pyjwt>=1.6.4 as jwt conditional dependencies. Contributed by Pau Rodriguez-Estivill. ([\#5555](https://github.com/matrix-org/synapse/issues/5555), [\#5586](https://github.com/matrix-org/synapse/issues/5586)) -- Fix a bug that would cause invited users to receive several emails for a single 3PID invite in case the inviter is rate limited. ([\#5576](https://github.com/matrix-org/synapse/issues/5576)) - - -Updates to the Docker image ---------------------------- -- Add ability to change Docker containers [timezone](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) with the `TZ` variable. ([\#5383](https://github.com/matrix-org/synapse/issues/5383)) -- Update docker image to use Python 3.7. ([\#5546](https://github.com/matrix-org/synapse/issues/5546)) -- Deprecate the use of environment variables for configuration, and make the use of a static configuration the default. ([\#5561](https://github.com/matrix-org/synapse/issues/5561), [\#5562](https://github.com/matrix-org/synapse/issues/5562), [\#5566](https://github.com/matrix-org/synapse/issues/5566), [\#5567](https://github.com/matrix-org/synapse/issues/5567)) -- Increase default log level for docker image to INFO. It can still be changed by editing the generated log.config file. ([\#5547](https://github.com/matrix-org/synapse/issues/5547)) -- Send synapse logs to the docker logging system, by default. ([\#5565](https://github.com/matrix-org/synapse/issues/5565)) -- Open the non-TLS port by default. ([\#5568](https://github.com/matrix-org/synapse/issues/5568)) -- Fix failure to start under docker with SAML support enabled. ([\#5490](https://github.com/matrix-org/synapse/issues/5490)) -- Use a sensible location for data files when generating a config file. ([\#5563](https://github.com/matrix-org/synapse/issues/5563)) - - -Deprecations and Removals -------------------------- - -- Python 2.7 is no longer a supported platform. Synapse now requires Python 3.5+ to run. ([\#5425](https://github.com/matrix-org/synapse/issues/5425)) -- PostgreSQL 9.4 is no longer supported. Synapse requires Postgres 9.5+ or above for Postgres support. ([\#5448](https://github.com/matrix-org/synapse/issues/5448)) -- Remove support for cpu_affinity setting. ([\#5525](https://github.com/matrix-org/synapse/issues/5525)) - - -Improved Documentation ----------------------- -- Improve README section on performance troubleshooting. ([\#4276](https://github.com/matrix-org/synapse/issues/4276)) -- Add information about how to install and run `black` on the codebase to code_style.rst. ([\#5537](https://github.com/matrix-org/synapse/issues/5537)) -- Improve install docs on choosing server_name. ([\#5558](https://github.com/matrix-org/synapse/issues/5558)) - - -Internal Changes ----------------- - -- Add logging to 3pid invite signature verification. ([\#5015](https://github.com/matrix-org/synapse/issues/5015)) -- Update example haproxy config to a more compatible setup. ([\#5313](https://github.com/matrix-org/synapse/issues/5313)) -- Track deactivated accounts in the database. ([\#5378](https://github.com/matrix-org/synapse/issues/5378), [\#5465](https://github.com/matrix-org/synapse/issues/5465), [\#5493](https://github.com/matrix-org/synapse/issues/5493)) -- Clean up code for sending federation EDUs. ([\#5381](https://github.com/matrix-org/synapse/issues/5381)) -- Add a sponsor button to the repo. ([\#5382](https://github.com/matrix-org/synapse/issues/5382), [\#5386](https://github.com/matrix-org/synapse/issues/5386)) -- Don't log non-200 responses from federation queries as exceptions. ([\#5383](https://github.com/matrix-org/synapse/issues/5383)) -- Update Python syntax in contrib/ to Python 3. ([\#5446](https://github.com/matrix-org/synapse/issues/5446)) -- Update federation_client dev script to support `.well-known` and work with python3. ([\#5447](https://github.com/matrix-org/synapse/issues/5447)) -- SyTest has been moved to Buildkite. ([\#5459](https://github.com/matrix-org/synapse/issues/5459)) -- Demo script now uses python3. ([\#5460](https://github.com/matrix-org/synapse/issues/5460)) -- Synapse can now handle RestServlets that return coroutines. ([\#5475](https://github.com/matrix-org/synapse/issues/5475), [\#5585](https://github.com/matrix-org/synapse/issues/5585)) -- The demo servers talk to each other again. ([\#5478](https://github.com/matrix-org/synapse/issues/5478)) -- Add an EXPERIMENTAL config option to try and periodically clean up extremities by sending dummy events. ([\#5480](https://github.com/matrix-org/synapse/issues/5480)) -- Synapse's codebase is now formatted by `black`. ([\#5482](https://github.com/matrix-org/synapse/issues/5482)) -- Some cleanups and sanity-checking in the CPU and database metrics. ([\#5499](https://github.com/matrix-org/synapse/issues/5499)) -- Improve email notification logging. ([\#5502](https://github.com/matrix-org/synapse/issues/5502)) -- Fix "Unexpected entry in 'full_schemas'" log warning. ([\#5509](https://github.com/matrix-org/synapse/issues/5509)) -- Improve logging when generating config files. ([\#5510](https://github.com/matrix-org/synapse/issues/5510)) -- Refactor and clean up Config parser for maintainability. ([\#5511](https://github.com/matrix-org/synapse/issues/5511)) -- Make the config clearer in that email.template_dir is relative to the Synapse's root directory, not the `synapse/` folder within it. ([\#5543](https://github.com/matrix-org/synapse/issues/5543)) -- Update v1.0.0 release changelog to include more information about changes to password resets. ([\#5545](https://github.com/matrix-org/synapse/issues/5545)) -- Remove non-functioning check_event_hash.py dev script. ([\#5548](https://github.com/matrix-org/synapse/issues/5548)) -- Synapse will now only allow TLS v1.2 connections when serving federation, if it terminates TLS. As Synapse's allowed ciphers were only able to be used in TLSv1.2 before, this does not change behaviour. ([\#5550](https://github.com/matrix-org/synapse/issues/5550)) -- Logging when running GC collection on generation 0 is now at the DEBUG level, not INFO. ([\#5557](https://github.com/matrix-org/synapse/issues/5557)) -- Reduce the amount of stuff we send in the docker context. ([\#5564](https://github.com/matrix-org/synapse/issues/5564)) -- Point the reverse links in the Purge History contrib scripts at the intended location. ([\#5570](https://github.com/matrix-org/synapse/issues/5570)) - - -Synapse 1.0.0 (2019-06-11) -========================== - -Bugfixes --------- - -- Fix bug where attempting to send transactions with large number of EDUs can fail. ([\#5418](https://github.com/matrix-org/synapse/issues/5418)) - - -Improved Documentation ----------------------- - -- Expand the federation guide to include relevant content from the MSC1711 FAQ ([\#5419](https://github.com/matrix-org/synapse/issues/5419)) - - -Internal Changes ----------------- - -- Move password reset links to /_matrix/client/unstable namespace. ([\#5424](https://github.com/matrix-org/synapse/issues/5424)) - - -Synapse 1.0.0rc3 (2019-06-10) -============================= - -Security: Fix authentication bug introduced in 1.0.0rc1. Please upgrade to rc3 immediately - - -Synapse 1.0.0rc2 (2019-06-10) -============================= - -Bugfixes --------- - -- Remove redundant warning about key server response validation. ([\#5392](https://github.com/matrix-org/synapse/issues/5392)) -- Fix bug where old keys stored in the database with a null valid until timestamp caused all verification requests for that key to fail. ([\#5415](https://github.com/matrix-org/synapse/issues/5415)) -- Fix excessive memory using with default `federation_verify_certificates: true` configuration. ([\#5417](https://github.com/matrix-org/synapse/issues/5417)) - - -Synapse 1.0.0rc1 (2019-06-07) -============================= - -Features --------- - -- Synapse now more efficiently collates room statistics. ([\#4338](https://github.com/matrix-org/synapse/issues/4338), [\#5260](https://github.com/matrix-org/synapse/issues/5260), [\#5324](https://github.com/matrix-org/synapse/issues/5324)) -- Add experimental support for relations (aka reactions and edits). ([\#5220](https://github.com/matrix-org/synapse/issues/5220)) -- Ability to configure default room version. ([\#5223](https://github.com/matrix-org/synapse/issues/5223), [\#5249](https://github.com/matrix-org/synapse/issues/5249)) -- Allow configuring a range for the account validity startup job. ([\#5276](https://github.com/matrix-org/synapse/issues/5276)) -- CAS login will now hit the r0 API, not the deprecated v1 one. ([\#5286](https://github.com/matrix-org/synapse/issues/5286)) -- Validate federation server TLS certificates by default (implements [MSC1711](https://github.com/matrix-org/matrix-doc/blob/master/proposals/1711-x509-for-federation.md)). ([\#5359](https://github.com/matrix-org/synapse/issues/5359)) -- Update /_matrix/client/versions to reference support for r0.5.0. ([\#5360](https://github.com/matrix-org/synapse/issues/5360)) -- Add a script to generate new signing-key files. ([\#5361](https://github.com/matrix-org/synapse/issues/5361)) -- Update upgrade and installation guides ahead of 1.0. ([\#5371](https://github.com/matrix-org/synapse/issues/5371)) -- Replace the `perspectives` configuration section with `trusted_key_servers`, and make validating the signatures on responses optional (since TLS will do this job for us). ([\#5374](https://github.com/matrix-org/synapse/issues/5374)) -- Add ability to perform password reset via email without trusting the identity server. **As a result of this PR, password resets will now be disabled on the default configuration.** - - Password reset emails are now sent from the homeserver by default, instead of the identity server. To enable this functionality, ensure `email` and `public_baseurl` config options are filled out. - - If you would like to re-enable password resets being sent from the identity server (warning: this is dangerous! See [#5345](https://github.com/matrix-org/synapse/pull/5345)), set `email.trust_identity_server_for_password_resets` to true. ([\#5377](https://github.com/matrix-org/synapse/issues/5377)) -- Set default room version to v4. ([\#5379](https://github.com/matrix-org/synapse/issues/5379)) - - -Bugfixes --------- - -- Fixes client-server API not sending "m.heroes" to lazy-load /sync requests when a rooms name or its canonical alias are empty. Thanks to @dnaf for this work! ([\#5089](https://github.com/matrix-org/synapse/issues/5089)) -- Prevent federation device list updates breaking when processing multiple updates at once. ([\#5156](https://github.com/matrix-org/synapse/issues/5156)) -- Fix worker registration bug caused by ClientReaderSlavedStore being unable to see get_profileinfo. ([\#5200](https://github.com/matrix-org/synapse/issues/5200)) -- Fix race when backfilling in rooms with worker mode. ([\#5221](https://github.com/matrix-org/synapse/issues/5221)) -- Fix appservice timestamp massaging. ([\#5233](https://github.com/matrix-org/synapse/issues/5233)) -- Ensure that server_keys fetched via a notary server are correctly signed. ([\#5251](https://github.com/matrix-org/synapse/issues/5251)) -- Show the correct error when logging out and access token is missing. ([\#5256](https://github.com/matrix-org/synapse/issues/5256)) -- Fix error code when there is an invalid parameter on /_matrix/client/r0/publicRooms ([\#5257](https://github.com/matrix-org/synapse/issues/5257)) -- Fix error when downloading thumbnail with missing width/height parameter. ([\#5258](https://github.com/matrix-org/synapse/issues/5258)) -- Fix schema update for account validity. ([\#5268](https://github.com/matrix-org/synapse/issues/5268)) -- Fix bug where we leaked extremities when we soft failed events, leading to performance degradation. ([\#5274](https://github.com/matrix-org/synapse/issues/5274), [\#5278](https://github.com/matrix-org/synapse/issues/5278), [\#5291](https://github.com/matrix-org/synapse/issues/5291)) -- Fix "db txn 'update_presence' from sentinel context" log messages. ([\#5275](https://github.com/matrix-org/synapse/issues/5275)) -- Fix dropped logcontexts during high outbound traffic. ([\#5277](https://github.com/matrix-org/synapse/issues/5277)) -- Fix a bug where it is not possible to get events in the federation format with the request `GET /_matrix/client/r0/rooms/{roomId}/messages`. ([\#5293](https://github.com/matrix-org/synapse/issues/5293)) -- Fix performance problems with the rooms stats background update. ([\#5294](https://github.com/matrix-org/synapse/issues/5294)) -- Fix noisy 'no key for server' logs. ([\#5300](https://github.com/matrix-org/synapse/issues/5300)) -- Fix bug where a notary server would sometimes forget old keys. ([\#5307](https://github.com/matrix-org/synapse/issues/5307)) -- Prevent users from setting huge displaynames and avatar URLs. ([\#5309](https://github.com/matrix-org/synapse/issues/5309)) -- Fix handling of failures when processing incoming events where calling `/event_auth` on remote server fails. ([\#5317](https://github.com/matrix-org/synapse/issues/5317)) -- Ensure that we have an up-to-date copy of the signing key when validating incoming federation requests. ([\#5321](https://github.com/matrix-org/synapse/issues/5321)) -- Fix various problems which made the signing-key notary server time out for some requests. ([\#5333](https://github.com/matrix-org/synapse/issues/5333)) -- Fix bug which would make certain operations (such as room joins) block for 20 minutes while attemoting to fetch verification keys. ([\#5334](https://github.com/matrix-org/synapse/issues/5334)) -- Fix a bug where we could rapidly mark a server as unreachable even though it was only down for a few minutes. ([\#5335](https://github.com/matrix-org/synapse/issues/5335), [\#5340](https://github.com/matrix-org/synapse/issues/5340)) -- Fix a bug where account validity renewal emails could only be sent when email notifs were enabled. ([\#5341](https://github.com/matrix-org/synapse/issues/5341)) -- Fix failure when fetching batches of events during backfill, etc. ([\#5342](https://github.com/matrix-org/synapse/issues/5342)) -- Add a new room version where the timestamps on events are checked against the validity periods on signing keys. ([\#5348](https://github.com/matrix-org/synapse/issues/5348), [\#5354](https://github.com/matrix-org/synapse/issues/5354)) -- Fix room stats and presence background updates to correctly handle missing events. ([\#5352](https://github.com/matrix-org/synapse/issues/5352)) -- Include left members in room summaries' heroes. ([\#5355](https://github.com/matrix-org/synapse/issues/5355)) -- Fix `federation_custom_ca_list` configuration option. ([\#5362](https://github.com/matrix-org/synapse/issues/5362)) -- Fix missing logcontext warnings on shutdown. ([\#5369](https://github.com/matrix-org/synapse/issues/5369)) - - -Improved Documentation ----------------------- - -- Fix docs on resetting the user directory. ([\#5282](https://github.com/matrix-org/synapse/issues/5282)) -- Fix notes about ACME in the MSC1711 faq. ([\#5357](https://github.com/matrix-org/synapse/issues/5357)) - - -Internal Changes ----------------- - -- Synapse will now serve the experimental "room complexity" API endpoint. ([\#5216](https://github.com/matrix-org/synapse/issues/5216)) -- The base classes for the v1 and v2_alpha REST APIs have been unified. ([\#5226](https://github.com/matrix-org/synapse/issues/5226), [\#5328](https://github.com/matrix-org/synapse/issues/5328)) -- Simplifications and comments in do_auth. ([\#5227](https://github.com/matrix-org/synapse/issues/5227)) -- Remove urllib3 pin as requests 2.22.0 has been released supporting urllib3 1.25.2. ([\#5230](https://github.com/matrix-org/synapse/issues/5230)) -- Preparatory work for key-validity features. ([\#5232](https://github.com/matrix-org/synapse/issues/5232), [\#5234](https://github.com/matrix-org/synapse/issues/5234), [\#5235](https://github.com/matrix-org/synapse/issues/5235), [\#5236](https://github.com/matrix-org/synapse/issues/5236), [\#5237](https://github.com/matrix-org/synapse/issues/5237), [\#5244](https://github.com/matrix-org/synapse/issues/5244), [\#5250](https://github.com/matrix-org/synapse/issues/5250), [\#5296](https://github.com/matrix-org/synapse/issues/5296), [\#5299](https://github.com/matrix-org/synapse/issues/5299), [\#5343](https://github.com/matrix-org/synapse/issues/5343), [\#5347](https://github.com/matrix-org/synapse/issues/5347), [\#5356](https://github.com/matrix-org/synapse/issues/5356)) -- Specify the type of reCAPTCHA key to use. ([\#5283](https://github.com/matrix-org/synapse/issues/5283)) -- Improve sample config for monthly active user blocking. ([\#5284](https://github.com/matrix-org/synapse/issues/5284)) -- Remove spurious debug from MatrixFederationHttpClient.get_json. ([\#5287](https://github.com/matrix-org/synapse/issues/5287)) -- Improve logging for logcontext leaks. ([\#5288](https://github.com/matrix-org/synapse/issues/5288)) -- Clarify that the admin change password API logs the user out. ([\#5303](https://github.com/matrix-org/synapse/issues/5303)) -- New installs will now use the v54 full schema, rather than the full schema v14 and applying incremental updates to v54. ([\#5320](https://github.com/matrix-org/synapse/issues/5320)) -- Improve docstrings on MatrixFederationClient. ([\#5332](https://github.com/matrix-org/synapse/issues/5332)) -- Clean up FederationClient.get_events for clarity. ([\#5344](https://github.com/matrix-org/synapse/issues/5344)) -- Various improvements to debug logging. ([\#5353](https://github.com/matrix-org/synapse/issues/5353)) -- Don't run CI build checks until sample config check has passed. ([\#5370](https://github.com/matrix-org/synapse/issues/5370)) -- Automatically retry buildkite builds (max twice) when an agent is lost. ([\#5380](https://github.com/matrix-org/synapse/issues/5380)) - -**Changelogs for versions older than 1.0.0 can be found [here](CHANGES-pre-1.0.md).** +**Changelogs for older versions can be found [here](docs/changelogs/).** diff --git a/docs/changelogs/CHANGES-2019.md b/docs/changelogs/CHANGES-2019.md new file mode 100644 index 0000000000..a356cc49a3 --- /dev/null +++ b/docs/changelogs/CHANGES-2019.md @@ -0,0 +1,1039 @@ + +Synapse 1.7.3 (2019-12-31) +========================== + +This release fixes a long-standing bug in the state resolution algorithm. + +Bugfixes +-------- + +- Fix exceptions caused by state resolution choking on malformed events. ([\#6608](https://github.com/matrix-org/synapse/issues/6608)) + + +Synapse 1.7.2 (2019-12-20) +========================== + +This release fixes some regressions introduced in Synapse 1.7.0 and 1.7.1. + +Bugfixes +-------- + +- Fix a regression introduced in Synapse 1.7.1 which caused errors when attempting to backfill rooms over federation. ([\#6576](https://github.com/matrix-org/synapse/issues/6576)) +- Fix a bug introduced in Synapse 1.7.0 which caused an error on startup when upgrading from versions before 1.3.0. ([\#6578](https://github.com/matrix-org/synapse/issues/6578)) + + +Synapse 1.7.1 (2019-12-18) +========================== + +This release includes several security fixes as well as a fix to a bug exposed by the security fixes. Administrators are encouraged to upgrade as soon as possible. + +Security updates +---------------- + +- Fix a bug which could cause room events to be incorrectly authorized using events from a different room. ([\#6501](https://github.com/matrix-org/synapse/issues/6501), [\#6503](https://github.com/matrix-org/synapse/issues/6503), [\#6521](https://github.com/matrix-org/synapse/issues/6521), [\#6524](https://github.com/matrix-org/synapse/issues/6524), [\#6530](https://github.com/matrix-org/synapse/issues/6530), [\#6531](https://github.com/matrix-org/synapse/issues/6531)) +- Fix a bug causing responses to the `/context` client endpoint to not use the pruned version of the event. ([\#6553](https://github.com/matrix-org/synapse/issues/6553)) +- Fix a cause of state resets in room versions 2 onwards. ([\#6556](https://github.com/matrix-org/synapse/issues/6556), [\#6560](https://github.com/matrix-org/synapse/issues/6560)) + +Bugfixes +-------- + +- Fix a bug which could cause the federation server to incorrectly return errors when handling certain obscure event graphs. ([\#6526](https://github.com/matrix-org/synapse/issues/6526), [\#6527](https://github.com/matrix-org/synapse/issues/6527)) + +Synapse 1.7.0 (2019-12-13) +========================== + +This release changes the default settings so that only local authenticated users can query the server's room directory. See the [upgrade notes](docs/upgrade.md#upgrading-to-v170) for details. + +Support for SQLite versions before 3.11 is now deprecated. A future release will refuse to start if used with an SQLite version before 3.11. + +Administrators are reminded that SQLite should not be used for production instances. Instructions for migrating to Postgres are available [here](docs/postgres.md). A future release of synapse will, by default, disable federation for servers using SQLite. + +No significant changes since 1.7.0rc2. + + +Synapse 1.7.0rc2 (2019-12-11) +============================= + +Bugfixes +-------- + +- Fix incorrect error message for invalid requests when setting user's avatar URL. ([\#6497](https://github.com/matrix-org/synapse/issues/6497)) +- Fix support for SQLite 3.7. ([\#6499](https://github.com/matrix-org/synapse/issues/6499)) +- Fix regression where sending email push would not work when using a pusher worker. ([\#6507](https://github.com/matrix-org/synapse/issues/6507), [\#6509](https://github.com/matrix-org/synapse/issues/6509)) + + +Synapse 1.7.0rc1 (2019-12-09) +============================= + +Features +-------- + +- Implement per-room message retention policies. ([\#5815](https://github.com/matrix-org/synapse/issues/5815), [\#6436](https://github.com/matrix-org/synapse/issues/6436)) +- Add etag and count fields to key backup endpoints to help clients guess if there are new keys. ([\#5858](https://github.com/matrix-org/synapse/issues/5858)) +- Add `/admin/v2/users` endpoint with pagination. Contributed by Awesome Technologies Innovationslabor GmbH. ([\#5925](https://github.com/matrix-org/synapse/issues/5925)) +- Require User-Interactive Authentication for `/account/3pid/add`, meaning the user's password will be required to add a third-party ID to their account. ([\#6119](https://github.com/matrix-org/synapse/issues/6119)) +- Implement the `/_matrix/federation/unstable/net.atleastfornow/state/` API as drafted in MSC2314. ([\#6176](https://github.com/matrix-org/synapse/issues/6176)) +- Configure privacy-preserving settings by default for the room directory. ([\#6355](https://github.com/matrix-org/synapse/issues/6355)) +- Add ephemeral messages support by partially implementing [MSC2228](https://github.com/matrix-org/matrix-doc/pull/2228). ([\#6409](https://github.com/matrix-org/synapse/issues/6409)) +- Add support for [MSC 2367](https://github.com/matrix-org/matrix-doc/pull/2367), which allows specifying a reason on all membership events. ([\#6434](https://github.com/matrix-org/synapse/issues/6434)) + + +Bugfixes +-------- + +- Transfer non-standard power levels on room upgrade. ([\#6237](https://github.com/matrix-org/synapse/issues/6237)) +- Fix error from the Pillow library when uploading RGBA images. ([\#6241](https://github.com/matrix-org/synapse/issues/6241)) +- Correctly apply the event filter to the `state`, `events_before` and `events_after` fields in the response to `/context` requests. ([\#6329](https://github.com/matrix-org/synapse/issues/6329)) +- Fix caching devices for remote users when using workers, so that we don't attempt to refetch (and potentially fail) each time a user requests devices. ([\#6332](https://github.com/matrix-org/synapse/issues/6332)) +- Prevent account data syncs getting lost across TCP replication. ([\#6333](https://github.com/matrix-org/synapse/issues/6333)) +- Fix bug: TypeError in `register_user()` while using LDAP auth module. ([\#6406](https://github.com/matrix-org/synapse/issues/6406)) +- Fix an intermittent exception when handling read-receipts. ([\#6408](https://github.com/matrix-org/synapse/issues/6408)) +- Fix broken guest registration when there are existing blocks of numeric user IDs. ([\#6420](https://github.com/matrix-org/synapse/issues/6420)) +- Fix startup error when http proxy is defined. ([\#6421](https://github.com/matrix-org/synapse/issues/6421)) +- Fix error when using synapse_port_db on a vanilla synapse db. ([\#6449](https://github.com/matrix-org/synapse/issues/6449)) +- Fix uploading multiple cross signing signatures for the same user. ([\#6451](https://github.com/matrix-org/synapse/issues/6451)) +- Fix bug which lead to exceptions being thrown in a loop when a cross-signed device is deleted. ([\#6462](https://github.com/matrix-org/synapse/issues/6462)) +- Fix `synapse_port_db` not exiting with a 0 code if something went wrong during the port process. ([\#6470](https://github.com/matrix-org/synapse/issues/6470)) +- Improve sanity-checking when receiving events over federation. ([\#6472](https://github.com/matrix-org/synapse/issues/6472)) +- Fix inaccurate per-block Prometheus metrics. ([\#6491](https://github.com/matrix-org/synapse/issues/6491)) +- Fix small performance regression for sending invites. ([\#6493](https://github.com/matrix-org/synapse/issues/6493)) +- Back out cross-signing code added in Synapse 1.5.0, which caused a performance regression. ([\#6494](https://github.com/matrix-org/synapse/issues/6494)) + + +Improved Documentation +---------------------- + +- Update documentation and variables in user contributed systemd reference file. ([\#6369](https://github.com/matrix-org/synapse/issues/6369), [\#6490](https://github.com/matrix-org/synapse/issues/6490)) +- Fix link in the user directory documentation. ([\#6388](https://github.com/matrix-org/synapse/issues/6388)) +- Add build instructions to the docker readme. ([\#6390](https://github.com/matrix-org/synapse/issues/6390)) +- Switch Ubuntu package install recommendation to use python3 packages in INSTALL.md. ([\#6443](https://github.com/matrix-org/synapse/issues/6443)) +- Write some docs for the quarantine_media api. ([\#6458](https://github.com/matrix-org/synapse/issues/6458)) +- Convert CONTRIBUTING.rst to markdown (among other small fixes). ([\#6461](https://github.com/matrix-org/synapse/issues/6461)) + + +Deprecations and Removals +------------------------- + +- Remove admin/v1/users_paginate endpoint. Contributed by Awesome Technologies Innovationslabor GmbH. ([\#5925](https://github.com/matrix-org/synapse/issues/5925)) +- Remove fallback for federation with old servers which lack the /federation/v1/state_ids API. ([\#6488](https://github.com/matrix-org/synapse/issues/6488)) + + +Internal Changes +---------------- + +- Add benchmarks for structured logging and improve output performance. ([\#6266](https://github.com/matrix-org/synapse/issues/6266)) +- Improve the performance of outputting structured logging. ([\#6322](https://github.com/matrix-org/synapse/issues/6322)) +- Refactor some code in the event authentication path for clarity. ([\#6343](https://github.com/matrix-org/synapse/issues/6343), [\#6468](https://github.com/matrix-org/synapse/issues/6468), [\#6480](https://github.com/matrix-org/synapse/issues/6480)) +- Clean up some unnecessary quotation marks around the codebase. ([\#6362](https://github.com/matrix-org/synapse/issues/6362)) +- Complain on startup instead of 500'ing during runtime when `public_baseurl` isn't set when necessary. ([\#6379](https://github.com/matrix-org/synapse/issues/6379)) +- Add a test scenario to make sure room history purges don't break `/messages` in the future. ([\#6392](https://github.com/matrix-org/synapse/issues/6392)) +- Clarifications for the email configuration settings. ([\#6423](https://github.com/matrix-org/synapse/issues/6423)) +- Add more tests to the blacklist when running in worker mode. ([\#6429](https://github.com/matrix-org/synapse/issues/6429)) +- Refactor data store layer to support multiple databases in the future. ([\#6454](https://github.com/matrix-org/synapse/issues/6454), [\#6464](https://github.com/matrix-org/synapse/issues/6464), [\#6469](https://github.com/matrix-org/synapse/issues/6469), [\#6487](https://github.com/matrix-org/synapse/issues/6487)) +- Port synapse.rest.client.v1 to async/await. ([\#6482](https://github.com/matrix-org/synapse/issues/6482)) +- Port synapse.rest.client.v2_alpha to async/await. ([\#6483](https://github.com/matrix-org/synapse/issues/6483)) +- Port SyncHandler to async/await. ([\#6484](https://github.com/matrix-org/synapse/issues/6484)) + +Synapse 1.6.1 (2019-11-28) +========================== + +Security updates +---------------- + +This release includes a security fix ([\#6426](https://github.com/matrix-org/synapse/issues/6426), below). Administrators are encouraged to upgrade as soon as possible. + +Bugfixes +-------- + +- Clean up local threepids from user on account deactivation. ([\#6426](https://github.com/matrix-org/synapse/issues/6426)) +- Fix startup error when http proxy is defined. ([\#6421](https://github.com/matrix-org/synapse/issues/6421)) + + +Synapse 1.6.0 (2019-11-26) +========================== + +Bugfixes +-------- + +- Fix phone home stats reporting. ([\#6418](https://github.com/matrix-org/synapse/issues/6418)) + + +Synapse 1.6.0rc2 (2019-11-25) +============================= + +Bugfixes +-------- + +- Fix a bug which could cause the background database update hander for event labels to get stuck in a loop raising exceptions. ([\#6407](https://github.com/matrix-org/synapse/issues/6407)) + + +Synapse 1.6.0rc1 (2019-11-20) +============================= + +Features +-------- + +- Add federation support for cross-signing. ([\#5727](https://github.com/matrix-org/synapse/issues/5727)) +- Increase default room version from 4 to 5, thereby enforcing server key validity period checks. ([\#6220](https://github.com/matrix-org/synapse/issues/6220)) +- Add support for outbound http proxying via http_proxy/HTTPS_PROXY env vars. ([\#6238](https://github.com/matrix-org/synapse/issues/6238)) +- Implement label-based filtering on `/sync` and `/messages` ([MSC2326](https://github.com/matrix-org/matrix-doc/pull/2326)). ([\#6301](https://github.com/matrix-org/synapse/issues/6301), [\#6310](https://github.com/matrix-org/synapse/issues/6310), [\#6340](https://github.com/matrix-org/synapse/issues/6340)) + + +Bugfixes +-------- + +- Fix LruCache callback deduplication for Python 3.8. Contributed by @V02460. ([\#6213](https://github.com/matrix-org/synapse/issues/6213)) +- Remove a room from a server's public rooms list on room upgrade. ([\#6232](https://github.com/matrix-org/synapse/issues/6232), [\#6235](https://github.com/matrix-org/synapse/issues/6235)) +- Delete keys from key backup when deleting backup versions. ([\#6253](https://github.com/matrix-org/synapse/issues/6253)) +- Make notification of cross-signing signatures work with workers. ([\#6254](https://github.com/matrix-org/synapse/issues/6254)) +- Fix exception when remote servers attempt to join a room that they're not allowed to join. ([\#6278](https://github.com/matrix-org/synapse/issues/6278)) +- Prevent errors from appearing on Synapse startup if `git` is not installed. ([\#6284](https://github.com/matrix-org/synapse/issues/6284)) +- Appservice requests will no longer contain a double slash prefix when the appservice url provided ends in a slash. ([\#6306](https://github.com/matrix-org/synapse/issues/6306)) +- Fix `/purge_room` admin API. ([\#6307](https://github.com/matrix-org/synapse/issues/6307)) +- Fix the `hidden` field in the `devices` table for SQLite versions prior to 3.23.0. ([\#6313](https://github.com/matrix-org/synapse/issues/6313)) +- Fix bug which casued rejected events to be persisted with the wrong room state. ([\#6320](https://github.com/matrix-org/synapse/issues/6320)) +- Fix bug where `rc_login` ratelimiting would prematurely kick in. ([\#6335](https://github.com/matrix-org/synapse/issues/6335)) +- Prevent the server taking a long time to start up when guest registration is enabled. ([\#6338](https://github.com/matrix-org/synapse/issues/6338)) +- Fix bug where upgrading a guest account to a full user would fail when account validity is enabled. ([\#6359](https://github.com/matrix-org/synapse/issues/6359)) +- Fix `to_device` stream ID getting reset every time Synapse restarts, which had the potential to cause unable to decrypt errors. ([\#6363](https://github.com/matrix-org/synapse/issues/6363)) +- Fix permission denied error when trying to generate a config file with the docker image. ([\#6389](https://github.com/matrix-org/synapse/issues/6389)) + + +Improved Documentation +---------------------- + +- Contributor documentation now mentions script to run linters. ([\#6164](https://github.com/matrix-org/synapse/issues/6164)) +- Modify CAPTCHA_SETUP.md to update the terms `private key` and `public key` to `secret key` and `site key` respectively. Contributed by Yash Jipkate. ([\#6257](https://github.com/matrix-org/synapse/issues/6257)) +- Update `INSTALL.md` Email section to talk about `account_threepid_delegates`. ([\#6272](https://github.com/matrix-org/synapse/issues/6272)) +- Fix a small typo in `account_threepid_delegates` configuration option. ([\#6273](https://github.com/matrix-org/synapse/issues/6273)) + + +Internal Changes +---------------- + +- Add a CI job to test the `synapse_port_db` script. ([\#6140](https://github.com/matrix-org/synapse/issues/6140), [\#6276](https://github.com/matrix-org/synapse/issues/6276)) +- Convert EventContext to an attrs. ([\#6218](https://github.com/matrix-org/synapse/issues/6218)) +- Move `persist_events` out from main data store. ([\#6240](https://github.com/matrix-org/synapse/issues/6240), [\#6300](https://github.com/matrix-org/synapse/issues/6300)) +- Reduce verbosity of user/room stats. ([\#6250](https://github.com/matrix-org/synapse/issues/6250)) +- Reduce impact of debug logging. ([\#6251](https://github.com/matrix-org/synapse/issues/6251)) +- Expose some homeserver functionality to spam checkers. ([\#6259](https://github.com/matrix-org/synapse/issues/6259)) +- Change cache descriptors to always return deferreds. ([\#6263](https://github.com/matrix-org/synapse/issues/6263), [\#6291](https://github.com/matrix-org/synapse/issues/6291)) +- Fix incorrect comment regarding the functionality of an `if` statement. ([\#6269](https://github.com/matrix-org/synapse/issues/6269)) +- Update CI to run `isort` over the `scripts` and `scripts-dev` directories. ([\#6270](https://github.com/matrix-org/synapse/issues/6270)) +- Replace every instance of `logger.warn` method with `logger.warning` as the former is deprecated. ([\#6271](https://github.com/matrix-org/synapse/issues/6271), [\#6314](https://github.com/matrix-org/synapse/issues/6314)) +- Port replication http server endpoints to async/await. ([\#6274](https://github.com/matrix-org/synapse/issues/6274)) +- Port room rest handlers to async/await. ([\#6275](https://github.com/matrix-org/synapse/issues/6275)) +- Remove redundant CLI parameters on CI's `flake8` step. ([\#6277](https://github.com/matrix-org/synapse/issues/6277)) +- Port `federation_server.py` to async/await. ([\#6279](https://github.com/matrix-org/synapse/issues/6279)) +- Port receipt and read markers to async/wait. ([\#6280](https://github.com/matrix-org/synapse/issues/6280)) +- Split out state storage into separate data store. ([\#6294](https://github.com/matrix-org/synapse/issues/6294), [\#6295](https://github.com/matrix-org/synapse/issues/6295)) +- Refactor EventContext for clarity. ([\#6298](https://github.com/matrix-org/synapse/issues/6298)) +- Update the version of black used to 19.10b0. ([\#6304](https://github.com/matrix-org/synapse/issues/6304)) +- Add some documentation about worker replication. ([\#6305](https://github.com/matrix-org/synapse/issues/6305)) +- Move admin endpoints into separate files. Contributed by Awesome Technologies Innovationslabor GmbH. ([\#6308](https://github.com/matrix-org/synapse/issues/6308)) +- Document the use of `lint.sh` for code style enforcement & extend it to run on specified paths only. ([\#6312](https://github.com/matrix-org/synapse/issues/6312)) +- Add optional python dependencies and dependant binary libraries to snapcraft packaging. ([\#6317](https://github.com/matrix-org/synapse/issues/6317)) +- Remove the dependency on psutil and replace functionality with the stdlib `resource` module. ([\#6318](https://github.com/matrix-org/synapse/issues/6318), [\#6336](https://github.com/matrix-org/synapse/issues/6336)) +- Improve documentation for EventContext fields. ([\#6319](https://github.com/matrix-org/synapse/issues/6319)) +- Add some checks that we aren't using state from rejected events. ([\#6330](https://github.com/matrix-org/synapse/issues/6330)) +- Add continuous integration for python 3.8. ([\#6341](https://github.com/matrix-org/synapse/issues/6341)) +- Correct spacing/case of various instances of the word "homeserver". ([\#6357](https://github.com/matrix-org/synapse/issues/6357)) +- Temporarily blacklist the failing unit test PurgeRoomTestCase.test_purge_room. ([\#6361](https://github.com/matrix-org/synapse/issues/6361)) + + +Synapse 1.5.1 (2019-11-06) +========================== + +Features +-------- + +- Limit the length of data returned by url previews, to prevent DoS attacks. ([\#6331](https://github.com/matrix-org/synapse/issues/6331), [\#6334](https://github.com/matrix-org/synapse/issues/6334)) + + +Synapse 1.5.0 (2019-10-29) +========================== + +Security updates +---------------- + +This release includes a security fix ([\#6262](https://github.com/matrix-org/synapse/issues/6262), below). Administrators are encouraged to upgrade as soon as possible. + +Bugfixes +-------- + +- Fix bug where room directory search was case sensitive. ([\#6268](https://github.com/matrix-org/synapse/issues/6268)) + + +Synapse 1.5.0rc2 (2019-10-28) +============================= + +Bugfixes +-------- + +- Update list of boolean columns in `synapse_port_db`. ([\#6247](https://github.com/matrix-org/synapse/issues/6247)) +- Fix /keys/query API on workers. ([\#6256](https://github.com/matrix-org/synapse/issues/6256)) +- Improve signature checking on some federation APIs. ([\#6262](https://github.com/matrix-org/synapse/issues/6262)) + + +Internal Changes +---------------- + +- Move schema delta files to the correct data store. ([\#6248](https://github.com/matrix-org/synapse/issues/6248)) +- Small performance improvement by removing repeated config lookups in room stats calculation. ([\#6255](https://github.com/matrix-org/synapse/issues/6255)) + + +Synapse 1.5.0rc1 (2019-10-24) +========================== + +Features +-------- + +- Improve quality of thumbnails for 1-bit/8-bit color palette images. ([\#2142](https://github.com/matrix-org/synapse/issues/2142)) +- Add ability to upload cross-signing signatures. ([\#5726](https://github.com/matrix-org/synapse/issues/5726)) +- Allow uploading of cross-signing keys. ([\#5769](https://github.com/matrix-org/synapse/issues/5769)) +- CAS login now provides a default display name for users if a `displayname_attribute` is set in the configuration file. ([\#6114](https://github.com/matrix-org/synapse/issues/6114)) +- Reject all pending invites for a user during deactivation. ([\#6125](https://github.com/matrix-org/synapse/issues/6125)) +- Add config option to suppress client side resource limit alerting. ([\#6173](https://github.com/matrix-org/synapse/issues/6173)) + + +Bugfixes +-------- + +- Return an HTTP 404 instead of 400 when requesting a filter by ID that is unknown to the server. Thanks to @krombel for contributing this! ([\#2380](https://github.com/matrix-org/synapse/issues/2380)) +- Fix a bug where users could be invited twice to the same group. ([\#3436](https://github.com/matrix-org/synapse/issues/3436)) +- Fix `/createRoom` failing with badly-formatted MXIDs in the invitee list. Thanks to @wener291! ([\#4088](https://github.com/matrix-org/synapse/issues/4088)) +- Make the `synapse_port_db` script create the right indexes on a new PostgreSQL database. ([\#6102](https://github.com/matrix-org/synapse/issues/6102), [\#6178](https://github.com/matrix-org/synapse/issues/6178), [\#6243](https://github.com/matrix-org/synapse/issues/6243)) +- Fix bug when uploading a large file: Synapse responds with `M_UNKNOWN` while it should be `M_TOO_LARGE` according to spec. Contributed by Anshul Angaria. ([\#6109](https://github.com/matrix-org/synapse/issues/6109)) +- Fix user push rules being deleted from a room when it is upgraded. ([\#6144](https://github.com/matrix-org/synapse/issues/6144)) +- Don't 500 when trying to exchange a revoked 3PID invite. ([\#6147](https://github.com/matrix-org/synapse/issues/6147)) +- Fix transferring notifications and tags when joining an upgraded room that is new to your server. ([\#6155](https://github.com/matrix-org/synapse/issues/6155)) +- Fix bug where guest account registration can wedge after restart. ([\#6161](https://github.com/matrix-org/synapse/issues/6161)) +- Fix monthly active user reaping when reserved users are specified. ([\#6168](https://github.com/matrix-org/synapse/issues/6168)) +- Fix `/federation/v1/state` endpoint not supporting newer room versions. ([\#6170](https://github.com/matrix-org/synapse/issues/6170)) +- Fix bug where we were updating censored events as bytes rather than text, occaisonally causing invalid JSON being inserted breaking APIs that attempted to fetch such events. ([\#6186](https://github.com/matrix-org/synapse/issues/6186)) +- Fix occasional missed updates in the room and user directories. ([\#6187](https://github.com/matrix-org/synapse/issues/6187)) +- Fix tracing of non-JSON APIs, `/media`, `/key` etc. ([\#6195](https://github.com/matrix-org/synapse/issues/6195)) +- Fix bug where presence would not get timed out correctly if a synchrotron worker is used and restarted. ([\#6212](https://github.com/matrix-org/synapse/issues/6212)) +- synapse_port_db: Add 2 additional BOOLEAN_COLUMNS to be able to convert from database schema v56. ([\#6216](https://github.com/matrix-org/synapse/issues/6216)) +- Fix a bug where the Synapse demo script blacklisted `::1` (ipv6 localhost) from receiving federation traffic. ([\#6229](https://github.com/matrix-org/synapse/issues/6229)) + + +Updates to the Docker image +--------------------------- + +- Fix logging getting lost for the docker image. ([\#6197](https://github.com/matrix-org/synapse/issues/6197)) + + +Internal Changes +---------------- + +- Update `user_filters` table to have a unique index, and non-null columns. Thanks to @pik for contributing this. ([\#1172](https://github.com/matrix-org/synapse/issues/1172), [\#6175](https://github.com/matrix-org/synapse/issues/6175), [\#6184](https://github.com/matrix-org/synapse/issues/6184)) +- Allow devices to be marked as hidden, for use by features such as cross-signing. + This adds a new field with a default value to the devices field in the database, + and so the database upgrade may take a long time depending on how many devices + are in the database. ([\#5759](https://github.com/matrix-org/synapse/issues/5759)) +- Move lookup-related functions from RoomMemberHandler to IdentityHandler. ([\#5978](https://github.com/matrix-org/synapse/issues/5978)) +- Improve performance of the public room list directory. ([\#6019](https://github.com/matrix-org/synapse/issues/6019), [\#6152](https://github.com/matrix-org/synapse/issues/6152), [\#6153](https://github.com/matrix-org/synapse/issues/6153), [\#6154](https://github.com/matrix-org/synapse/issues/6154)) +- Edit header dicts docstrings in `SimpleHttpClient` to note that `str` or `bytes` can be passed as header keys. ([\#6077](https://github.com/matrix-org/synapse/issues/6077)) +- Add snapcraft packaging information. Contributed by @devec0. ([\#6084](https://github.com/matrix-org/synapse/issues/6084), [\#6191](https://github.com/matrix-org/synapse/issues/6191)) +- Kill off half-implemented password-reset via sms. ([\#6101](https://github.com/matrix-org/synapse/issues/6101)) +- Remove `get_user_by_req` opentracing span and add some tags. ([\#6108](https://github.com/matrix-org/synapse/issues/6108)) +- Drop some unused database tables. ([\#6115](https://github.com/matrix-org/synapse/issues/6115)) +- Add env var to turn on tracking of log context changes. ([\#6127](https://github.com/matrix-org/synapse/issues/6127)) +- Refactor configuration loading to allow better typechecking. ([\#6137](https://github.com/matrix-org/synapse/issues/6137)) +- Log responder when responding to media request. ([\#6139](https://github.com/matrix-org/synapse/issues/6139)) +- Improve performance of `find_next_generated_user_id` DB query. ([\#6148](https://github.com/matrix-org/synapse/issues/6148)) +- Expand type-checking on modules imported by `synapse.config`. ([\#6150](https://github.com/matrix-org/synapse/issues/6150)) +- Use Postgres ANY for selecting many values. ([\#6156](https://github.com/matrix-org/synapse/issues/6156)) +- Add more caching to `_get_joined_users_from_context` DB query. ([\#6159](https://github.com/matrix-org/synapse/issues/6159)) +- Add some metrics on the federation sender. ([\#6160](https://github.com/matrix-org/synapse/issues/6160)) +- Add some logging to the rooms stats updates, to try to track down a flaky test. ([\#6167](https://github.com/matrix-org/synapse/issues/6167)) +- Remove unused `timeout` parameter from `_get_public_room_list`. ([\#6179](https://github.com/matrix-org/synapse/issues/6179)) +- Reject (accidental) attempts to insert bytes into postgres tables. ([\#6186](https://github.com/matrix-org/synapse/issues/6186)) +- Make `version` optional in body of `PUT /room_keys/version/{version}`, since it's redundant. ([\#6189](https://github.com/matrix-org/synapse/issues/6189)) +- Make storage layer responsible for adding device names to key, rather than the handler. ([\#6193](https://github.com/matrix-org/synapse/issues/6193)) +- Port `synapse.rest.admin` module to use async/await. ([\#6196](https://github.com/matrix-org/synapse/issues/6196)) +- Enforce that all boolean configuration values are lowercase in CI. ([\#6203](https://github.com/matrix-org/synapse/issues/6203)) +- Remove some unused event-auth code. ([\#6214](https://github.com/matrix-org/synapse/issues/6214)) +- Remove `Auth.check` method. ([\#6217](https://github.com/matrix-org/synapse/issues/6217)) +- Remove `format_tap.py` script in favour of a perl reimplementation in Sytest's repo. ([\#6219](https://github.com/matrix-org/synapse/issues/6219)) +- Refactor storage layer in preparation to support having multiple databases. ([\#6231](https://github.com/matrix-org/synapse/issues/6231)) +- Remove some extra quotation marks across the codebase. ([\#6236](https://github.com/matrix-org/synapse/issues/6236)) + + +Synapse 1.4.1 (2019-10-18) +========================== + +No changes since 1.4.1rc1. + + +Synapse 1.4.1rc1 (2019-10-17) +============================= + +Bugfixes +-------- + +- Fix bug where redacted events were sometimes incorrectly censored in the database, breaking APIs that attempted to fetch such events. ([\#6185](https://github.com/matrix-org/synapse/issues/6185), [5b0e9948](https://github.com/matrix-org/synapse/commit/5b0e9948eaae801643e594b5abc8ee4b10bd194e)) + +Synapse 1.4.0 (2019-10-03) +========================== + +Bugfixes +-------- + +- Redact `client_secret` in server logs. ([\#6158](https://github.com/matrix-org/synapse/issues/6158)) + + +Synapse 1.4.0rc2 (2019-10-02) +============================= + +Bugfixes +-------- + +- Fix bug in background update that adds last seen information to the `devices` table, and improve its performance on Postgres. ([\#6135](https://github.com/matrix-org/synapse/issues/6135)) +- Fix bad performance of censoring redactions background task. ([\#6141](https://github.com/matrix-org/synapse/issues/6141)) +- Fix fetching censored redactions from DB, which caused APIs like initial sync to fail if it tried to include the censored redaction. ([\#6145](https://github.com/matrix-org/synapse/issues/6145)) +- Fix exceptions when storing large retry intervals for down remote servers. ([\#6146](https://github.com/matrix-org/synapse/issues/6146)) + + +Internal Changes +---------------- + +- Fix up sample config entry for `redaction_retention_period` option. ([\#6117](https://github.com/matrix-org/synapse/issues/6117)) + + +Synapse 1.4.0rc1 (2019-09-26) +============================= + +Note that this release includes significant changes around 3pid +verification. Administrators are reminded to review the [upgrade notes](docs/upgrade.md#upgrading-to-v140). + +Features +-------- + +- Changes to 3pid verification: + - Add the ability to send registration emails from the homeserver rather than delegating to an identity server. ([\#5835](https://github.com/matrix-org/synapse/issues/5835), [\#5940](https://github.com/matrix-org/synapse/issues/5940), [\#5993](https://github.com/matrix-org/synapse/issues/5993), [\#5994](https://github.com/matrix-org/synapse/issues/5994), [\#5868](https://github.com/matrix-org/synapse/issues/5868)) + - Replace `trust_identity_server_for_password_resets` config option with `account_threepid_delegates`, and make the `id_server` parameteter optional on `*/requestToken` endpoints, as per [MSC2263](https://github.com/matrix-org/matrix-doc/pull/2263). ([\#5876](https://github.com/matrix-org/synapse/issues/5876), [\#5969](https://github.com/matrix-org/synapse/issues/5969), [\#6028](https://github.com/matrix-org/synapse/issues/6028)) + - Switch to using the v2 Identity Service `/lookup` API where available, with fallback to v1. (Implements [MSC2134](https://github.com/matrix-org/matrix-doc/pull/2134) plus `id_access_token authentication` for v2 Identity Service APIs from [MSC2140](https://github.com/matrix-org/matrix-doc/pull/2140)). ([\#5897](https://github.com/matrix-org/synapse/issues/5897)) + - Remove `bind_email` and `bind_msisdn` parameters from `/register` ala [MSC2140](https://github.com/matrix-org/matrix-doc/pull/2140). ([\#5964](https://github.com/matrix-org/synapse/issues/5964)) + - Add `m.id_access_token` to `unstable_features` in `/versions` as per [MSC2264](https://github.com/matrix-org/matrix-doc/pull/2264). ([\#5974](https://github.com/matrix-org/synapse/issues/5974)) + - Use the v2 Identity Service API for 3PID invites. ([\#5979](https://github.com/matrix-org/synapse/issues/5979)) + - Add `POST /_matrix/client/unstable/account/3pid/unbind` endpoint from [MSC2140](https://github.com/matrix-org/matrix-doc/pull/2140) for unbinding a 3PID from an identity server without removing it from the homeserver user account. ([\#5980](https://github.com/matrix-org/synapse/issues/5980), [\#6062](https://github.com/matrix-org/synapse/issues/6062)) + - Use `account_threepid_delegate.email` and `account_threepid_delegate.msisdn` for validating threepid sessions. ([\#6011](https://github.com/matrix-org/synapse/issues/6011)) + - Allow homeserver to handle or delegate email validation when adding an email to a user's account. ([\#6042](https://github.com/matrix-org/synapse/issues/6042)) + - Implement new Client Server API endpoints `/account/3pid/add` and `/account/3pid/bind` as per [MSC2290](https://github.com/matrix-org/matrix-doc/pull/2290). ([\#6043](https://github.com/matrix-org/synapse/issues/6043)) + - Add an unstable feature flag for separate add/bind 3pid APIs. ([\#6044](https://github.com/matrix-org/synapse/issues/6044)) + - Remove `bind` parameter from Client Server POST `/account` endpoint as per [MSC2290](https://github.com/matrix-org/matrix-doc/pull/2290/). ([\#6067](https://github.com/matrix-org/synapse/issues/6067)) + - Add `POST /add_threepid/msisdn/submit_token` endpoint for proxying submitToken on an `account_threepid_handler`. ([\#6078](https://github.com/matrix-org/synapse/issues/6078)) + - Add `submit_url` response parameter to `*/msisdn/requestToken` endpoints. ([\#6079](https://github.com/matrix-org/synapse/issues/6079)) + - Add `m.require_identity_server` flag to /version's unstable_features. ([\#5972](https://github.com/matrix-org/synapse/issues/5972)) +- Enhancements to OpenTracing support: + - Make OpenTracing work in worker mode. ([\#5771](https://github.com/matrix-org/synapse/issues/5771)) + - Pass OpenTracing contexts between servers when transmitting EDUs. ([\#5852](https://github.com/matrix-org/synapse/issues/5852)) + - OpenTracing for device list updates. ([\#5853](https://github.com/matrix-org/synapse/issues/5853)) + - Add a tag recording a request's authenticated entity and corresponding servlet in OpenTracing. ([\#5856](https://github.com/matrix-org/synapse/issues/5856)) + - Add minimum OpenTracing for client servlets. ([\#5983](https://github.com/matrix-org/synapse/issues/5983)) + - Check at setup that OpenTracing is installed if it's enabled in the config. ([\#5985](https://github.com/matrix-org/synapse/issues/5985)) + - Trace replication send times. ([\#5986](https://github.com/matrix-org/synapse/issues/5986)) + - Include missing OpenTracing contexts in outbout replication requests. ([\#5982](https://github.com/matrix-org/synapse/issues/5982)) + - Fix sending of EDUs when OpenTracing is enabled with an empty whitelist. ([\#5984](https://github.com/matrix-org/synapse/issues/5984)) + - Fix invalid references to None while OpenTracing if the log context slips. ([\#5988](https://github.com/matrix-org/synapse/issues/5988), [\#5991](https://github.com/matrix-org/synapse/issues/5991)) + - OpenTracing for room and e2e keys. ([\#5855](https://github.com/matrix-org/synapse/issues/5855)) + - Add OpenTracing span over HTTP push processing. ([\#6003](https://github.com/matrix-org/synapse/issues/6003)) +- Add an admin API to purge old rooms from the database. ([\#5845](https://github.com/matrix-org/synapse/issues/5845)) +- Retry well-known lookups if we have recently seen a valid well-known record for the server. ([\#5850](https://github.com/matrix-org/synapse/issues/5850)) +- Add support for filtered room-directory search requests over federation ([MSC2197](https://github.com/matrix-org/matrix-doc/pull/2197), in order to allow upcoming room directory query performance improvements. ([\#5859](https://github.com/matrix-org/synapse/issues/5859)) +- Correctly retry all hosts returned from SRV when we fail to connect. ([\#5864](https://github.com/matrix-org/synapse/issues/5864)) +- Add admin API endpoint for setting whether or not a user is a server administrator. ([\#5878](https://github.com/matrix-org/synapse/issues/5878)) +- Enable cleaning up extremities with dummy events by default to prevent undue build up of forward extremities. ([\#5884](https://github.com/matrix-org/synapse/issues/5884)) +- Add config option to sign remote key query responses with a separate key. ([\#5895](https://github.com/matrix-org/synapse/issues/5895)) +- Add support for config templating. ([\#5900](https://github.com/matrix-org/synapse/issues/5900)) +- Users with the type of "support" or "bot" are no longer required to consent. ([\#5902](https://github.com/matrix-org/synapse/issues/5902)) +- Let synctl accept a directory of config files. ([\#5904](https://github.com/matrix-org/synapse/issues/5904)) +- Increase max display name size to 256. ([\#5906](https://github.com/matrix-org/synapse/issues/5906)) +- Add admin API endpoint for getting whether or not a user is a server administrator. ([\#5914](https://github.com/matrix-org/synapse/issues/5914)) +- Redact events in the database that have been redacted for a week. ([\#5934](https://github.com/matrix-org/synapse/issues/5934)) +- New prometheus metrics: + - `synapse_federation_known_servers`: represents the total number of servers your server knows about (i.e. is in rooms with), including itself. Enable by setting `metrics_flags.known_servers` to True in the configuration.([\#5981](https://github.com/matrix-org/synapse/issues/5981)) + - `synapse_build_info`: exposes the Python version, OS version, and Synapse version of the running server. ([\#6005](https://github.com/matrix-org/synapse/issues/6005)) +- Give appropriate exit codes when synctl fails. ([\#5992](https://github.com/matrix-org/synapse/issues/5992)) +- Apply the federation blacklist to requests to identity servers. ([\#6000](https://github.com/matrix-org/synapse/issues/6000)) +- Add `report_stats_endpoint` option to configure where stats are reported to, if enabled. Contributed by @Sorunome. ([\#6012](https://github.com/matrix-org/synapse/issues/6012)) +- Add config option to increase ratelimits for room admins redacting messages. ([\#6015](https://github.com/matrix-org/synapse/issues/6015)) +- Stop sending federation transactions to servers which have been down for a long time. ([\#6026](https://github.com/matrix-org/synapse/issues/6026)) +- Make the process for mapping SAML2 users to matrix IDs more flexible. ([\#6037](https://github.com/matrix-org/synapse/issues/6037)) +- Return a clearer error message when a timeout occurs when attempting to contact an identity server. ([\#6073](https://github.com/matrix-org/synapse/issues/6073)) +- Prevent password reset's submit_token endpoint from accepting trailing slashes. ([\#6074](https://github.com/matrix-org/synapse/issues/6074)) +- Return 403 on `/register/available` if registration has been disabled. ([\#6082](https://github.com/matrix-org/synapse/issues/6082)) +- Explicitly log when a homeserver does not have the `trusted_key_servers` config field configured. ([\#6090](https://github.com/matrix-org/synapse/issues/6090)) +- Add support for pruning old rows in `user_ips` table. ([\#6098](https://github.com/matrix-org/synapse/issues/6098)) + +Bugfixes +-------- + +- Don't create broken room when `power_level_content_override.users` does not contain `creator_id`. ([\#5633](https://github.com/matrix-org/synapse/issues/5633)) +- Fix database index so that different backup versions can have the same sessions. ([\#5857](https://github.com/matrix-org/synapse/issues/5857)) +- Fix Synapse looking for config options `password_reset_failure_template` and `password_reset_success_template`, when they are actually `password_reset_template_failure_html`, `password_reset_template_success_html`. ([\#5863](https://github.com/matrix-org/synapse/issues/5863)) +- Fix stack overflow when recovering an appservice which had an outage. ([\#5885](https://github.com/matrix-org/synapse/issues/5885)) +- Fix error message which referred to `public_base_url` instead of `public_baseurl`. Thanks to @aaronraimist for the fix! ([\#5909](https://github.com/matrix-org/synapse/issues/5909)) +- Fix 404 for thumbnail download when `dynamic_thumbnails` is `false` and the thumbnail was dynamically generated. Fix reported by rkfg. ([\#5915](https://github.com/matrix-org/synapse/issues/5915)) +- Fix a cache-invalidation bug for worker-based deployments. ([\#5920](https://github.com/matrix-org/synapse/issues/5920)) +- Fix admin API for listing media in a room not being available with an external media repo. ([\#5966](https://github.com/matrix-org/synapse/issues/5966)) +- Fix list media admin API always returning an error. ([\#5967](https://github.com/matrix-org/synapse/issues/5967)) +- Fix room and user stats tracking. ([\#5971](https://github.com/matrix-org/synapse/issues/5971), [\#5998](https://github.com/matrix-org/synapse/issues/5998), [\#6029](https://github.com/matrix-org/synapse/issues/6029)) +- Return a `M_MISSING_PARAM` if `sid` is not provided to `/account/3pid`. ([\#5995](https://github.com/matrix-org/synapse/issues/5995)) +- `federation_certificate_verification_whitelist` now will not cause `TypeErrors` to be raised (a regression in 1.3). Additionally, it now supports internationalised domain names in their non-canonical representation. ([\#5996](https://github.com/matrix-org/synapse/issues/5996)) +- Only count real users when checking for auto-creation of auto-join room. ([\#6004](https://github.com/matrix-org/synapse/issues/6004)) +- Ensure support users can be registered even if MAU limit is reached. ([\#6020](https://github.com/matrix-org/synapse/issues/6020)) +- Fix bug where login error was shown incorrectly on SSO fallback login. ([\#6024](https://github.com/matrix-org/synapse/issues/6024)) +- Fix bug in calculating the federation retry backoff period. ([\#6025](https://github.com/matrix-org/synapse/issues/6025)) +- Prevent exceptions being logged when extremity-cleanup events fail due to lack of user consent to the terms of service. ([\#6053](https://github.com/matrix-org/synapse/issues/6053)) +- Remove POST method from password-reset `submit_token` endpoint until we implement `submit_url` functionality. ([\#6056](https://github.com/matrix-org/synapse/issues/6056)) +- Fix logcontext spam on non-Linux platforms. ([\#6059](https://github.com/matrix-org/synapse/issues/6059)) +- Ensure query parameters in email validation links are URL-encoded. ([\#6063](https://github.com/matrix-org/synapse/issues/6063)) +- Fix a bug which caused SAML attribute maps to be overridden by defaults. ([\#6069](https://github.com/matrix-org/synapse/issues/6069)) +- Fix the logged number of updated items for the `users_set_deactivated_flag` background update. ([\#6092](https://github.com/matrix-org/synapse/issues/6092)) +- Add `sid` to `next_link` for email validation. ([\#6097](https://github.com/matrix-org/synapse/issues/6097)) +- Threepid validity checks on msisdns should not be dependent on `threepid_behaviour_email`. ([\#6104](https://github.com/matrix-org/synapse/issues/6104)) +- Ensure that servers which are not configured to support email address verification do not offer it in the registration flows. ([\#6107](https://github.com/matrix-org/synapse/issues/6107)) + + +Updates to the Docker image +--------------------------- + +- Avoid changing `UID/GID` if they are already correct. ([\#5970](https://github.com/matrix-org/synapse/issues/5970)) +- Provide `SYNAPSE_WORKER` envvar to specify python module. ([\#6058](https://github.com/matrix-org/synapse/issues/6058)) + + +Improved Documentation +---------------------- + +- Convert documentation to markdown (from rst) ([\#5849](https://github.com/matrix-org/synapse/issues/5849)) +- Update `INSTALL.md` to say that Python 2 is no longer supported. ([\#5953](https://github.com/matrix-org/synapse/issues/5953)) +- Add developer documentation for using SAML2. ([\#6032](https://github.com/matrix-org/synapse/issues/6032)) +- Add some notes on rolling back to v1.3.1. ([\#6049](https://github.com/matrix-org/synapse/issues/6049)) +- Update the upgrade notes. ([\#6050](https://github.com/matrix-org/synapse/issues/6050)) + + +Deprecations and Removals +------------------------- + +- Remove shared-secret registration from `/_matrix/client/r0/register` endpoint. Contributed by Awesome Technologies Innovationslabor GmbH. ([\#5877](https://github.com/matrix-org/synapse/issues/5877)) +- Deprecate the `trusted_third_party_id_servers` option. ([\#5875](https://github.com/matrix-org/synapse/issues/5875)) + + +Internal Changes +---------------- + +- Lay the groundwork for structured logging output. ([\#5680](https://github.com/matrix-org/synapse/issues/5680)) +- Retry well-known lookup before the cache expires, giving a grace period where the remote well-known can be down but we still use the old result. ([\#5844](https://github.com/matrix-org/synapse/issues/5844)) +- Remove log line for debugging issue #5407. ([\#5860](https://github.com/matrix-org/synapse/issues/5860)) +- Refactor the Appservice scheduler code. ([\#5886](https://github.com/matrix-org/synapse/issues/5886)) +- Compatibility with v2 Identity Service APIs other than /lookup. ([\#5892](https://github.com/matrix-org/synapse/issues/5892), [\#6013](https://github.com/matrix-org/synapse/issues/6013)) +- Stop populating some unused tables. ([\#5893](https://github.com/matrix-org/synapse/issues/5893), [\#6047](https://github.com/matrix-org/synapse/issues/6047)) +- Add missing index on `users_in_public_rooms` to improve the performance of directory queries. ([\#5894](https://github.com/matrix-org/synapse/issues/5894)) +- Improve the logging when we have an error when fetching signing keys. ([\#5896](https://github.com/matrix-org/synapse/issues/5896)) +- Add support for database engine-specific schema deltas, based on file extension. ([\#5911](https://github.com/matrix-org/synapse/issues/5911)) +- Update Buildkite pipeline to use plugins instead of buildkite-agent commands. ([\#5922](https://github.com/matrix-org/synapse/issues/5922)) +- Add link in sample config to the logging config schema. ([\#5926](https://github.com/matrix-org/synapse/issues/5926)) +- Remove unnecessary parentheses in return statements. ([\#5931](https://github.com/matrix-org/synapse/issues/5931)) +- Remove unused `jenkins/prepare_sytest.sh` file. ([\#5938](https://github.com/matrix-org/synapse/issues/5938)) +- Move Buildkite pipeline config to the pipelines repo. ([\#5943](https://github.com/matrix-org/synapse/issues/5943)) +- Remove unnecessary return statements in the codebase which were the result of a regex run. ([\#5962](https://github.com/matrix-org/synapse/issues/5962)) +- Remove left-over methods from v1 registration API. ([\#5963](https://github.com/matrix-org/synapse/issues/5963)) +- Cleanup event auth type initialisation. ([\#5975](https://github.com/matrix-org/synapse/issues/5975)) +- Clean up dependency checking at setup. ([\#5989](https://github.com/matrix-org/synapse/issues/5989)) +- Update OpenTracing docs to use the unified `trace` method. ([\#5776](https://github.com/matrix-org/synapse/issues/5776)) +- Small refactor of function arguments and docstrings in` RoomMemberHandler`. ([\#6009](https://github.com/matrix-org/synapse/issues/6009)) +- Remove unused `origin` argument on `FederationHandler.add_display_name_to_third_party_invite`. ([\#6010](https://github.com/matrix-org/synapse/issues/6010)) +- Add a `failure_ts` column to the `destinations` database table. ([\#6016](https://github.com/matrix-org/synapse/issues/6016), [\#6072](https://github.com/matrix-org/synapse/issues/6072)) +- Clean up some code in the retry logic. ([\#6017](https://github.com/matrix-org/synapse/issues/6017)) +- Fix the structured logging tests stomping on the global log configuration for subsequent tests. ([\#6023](https://github.com/matrix-org/synapse/issues/6023)) +- Clean up the sample config for SAML authentication. ([\#6064](https://github.com/matrix-org/synapse/issues/6064)) +- Change mailer logging to reflect Synapse doesn't just do chat notifications by email now. ([\#6075](https://github.com/matrix-org/synapse/issues/6075)) +- Move last-seen info into devices table. ([\#6089](https://github.com/matrix-org/synapse/issues/6089)) +- Remove unused parameter to `get_user_id_by_threepid`. ([\#6099](https://github.com/matrix-org/synapse/issues/6099)) +- Refactor the user-interactive auth handling. ([\#6105](https://github.com/matrix-org/synapse/issues/6105)) +- Refactor code for calculating registration flows. ([\#6106](https://github.com/matrix-org/synapse/issues/6106)) + + +Synapse 1.3.1 (2019-08-17) +========================== + +Features +-------- + +- Drop hard dependency on `sdnotify` python package. ([\#5871](https://github.com/matrix-org/synapse/issues/5871)) + + +Bugfixes +-------- + +- Fix startup issue (hang on ACME provisioning) due to ordering of Twisted reactor startup. Thanks to @chrismoos for supplying the fix. ([\#5867](https://github.com/matrix-org/synapse/issues/5867)) + + +Synapse 1.3.0 (2019-08-15) +========================== + +Bugfixes +-------- + +- Fix 500 Internal Server Error on `publicRooms` when the public room list was + cached. ([\#5851](https://github.com/matrix-org/synapse/issues/5851)) + + +Synapse 1.3.0rc1 (2019-08-13) +========================== + +Features +-------- + +- Use `M_USER_DEACTIVATED` instead of `M_UNKNOWN` for errcode when a deactivated user attempts to login. ([\#5686](https://github.com/matrix-org/synapse/issues/5686)) +- Add sd_notify hooks to ease systemd integration and allows usage of Type=Notify. ([\#5732](https://github.com/matrix-org/synapse/issues/5732)) +- Synapse will no longer serve any media repo admin endpoints when `enable_media_repo` is set to False in the configuration. If a media repo worker is used, the admin APIs relating to the media repo will be served from it instead. ([\#5754](https://github.com/matrix-org/synapse/issues/5754), [\#5848](https://github.com/matrix-org/synapse/issues/5848)) +- Synapse can now be configured to not join remote rooms of a given "complexity" (currently, state events) over federation. This option can be used to prevent adverse performance on resource-constrained homeservers. ([\#5783](https://github.com/matrix-org/synapse/issues/5783)) +- Allow defining HTML templates to serve the user on account renewal attempt when using the account validity feature. ([\#5807](https://github.com/matrix-org/synapse/issues/5807)) + + +Bugfixes +-------- + +- Fix UISIs during homeserver outage. ([\#5693](https://github.com/matrix-org/synapse/issues/5693), [\#5789](https://github.com/matrix-org/synapse/issues/5789)) +- Fix stack overflow in server key lookup code. ([\#5724](https://github.com/matrix-org/synapse/issues/5724)) +- start.sh no longer uses deprecated cli option. ([\#5725](https://github.com/matrix-org/synapse/issues/5725)) +- Log when we receive an event receipt from an unexpected origin. ([\#5743](https://github.com/matrix-org/synapse/issues/5743)) +- Fix debian packaging scripts to correctly build sid packages. ([\#5775](https://github.com/matrix-org/synapse/issues/5775)) +- Correctly handle redactions of redactions. ([\#5788](https://github.com/matrix-org/synapse/issues/5788)) +- Return 404 instead of 403 when accessing /rooms/{roomId}/event/{eventId} for an event without the appropriate permissions. ([\#5798](https://github.com/matrix-org/synapse/issues/5798)) +- Fix check that tombstone is a state event in push rules. ([\#5804](https://github.com/matrix-org/synapse/issues/5804)) +- Fix error when trying to login as a deactivated user when using a worker to handle login. ([\#5806](https://github.com/matrix-org/synapse/issues/5806)) +- Fix bug where user `/sync` stream could get wedged in rare circumstances. ([\#5825](https://github.com/matrix-org/synapse/issues/5825)) +- The purge_remote_media.sh script was fixed. ([\#5839](https://github.com/matrix-org/synapse/issues/5839)) + + +Deprecations and Removals +------------------------- + +- Synapse now no longer accepts the `-v`/`--verbose`, `-f`/`--log-file`, or `--log-config` command line flags, and removes the deprecated `verbose` and `log_file` configuration file options. Users of these options should migrate their options into the dedicated log configuration. ([\#5678](https://github.com/matrix-org/synapse/issues/5678), [\#5729](https://github.com/matrix-org/synapse/issues/5729)) +- Remove non-functional 'expire_access_token' setting. ([\#5782](https://github.com/matrix-org/synapse/issues/5782)) + + +Internal Changes +---------------- + +- Make Jaeger fully configurable. ([\#5694](https://github.com/matrix-org/synapse/issues/5694)) +- Add precautionary measures to prevent future abuse of `window.opener` in default welcome page. ([\#5695](https://github.com/matrix-org/synapse/issues/5695)) +- Reduce database IO usage by optimising queries for current membership. ([\#5706](https://github.com/matrix-org/synapse/issues/5706), [\#5738](https://github.com/matrix-org/synapse/issues/5738), [\#5746](https://github.com/matrix-org/synapse/issues/5746), [\#5752](https://github.com/matrix-org/synapse/issues/5752), [\#5770](https://github.com/matrix-org/synapse/issues/5770), [\#5774](https://github.com/matrix-org/synapse/issues/5774), [\#5792](https://github.com/matrix-org/synapse/issues/5792), [\#5793](https://github.com/matrix-org/synapse/issues/5793)) +- Improve caching when fetching `get_filtered_current_state_ids`. ([\#5713](https://github.com/matrix-org/synapse/issues/5713)) +- Don't accept opentracing data from clients. ([\#5715](https://github.com/matrix-org/synapse/issues/5715)) +- Speed up PostgreSQL unit tests in CI. ([\#5717](https://github.com/matrix-org/synapse/issues/5717)) +- Update the coding style document. ([\#5719](https://github.com/matrix-org/synapse/issues/5719)) +- Improve database query performance when recording retry intervals for remote hosts. ([\#5720](https://github.com/matrix-org/synapse/issues/5720)) +- Add a set of opentracing utils. ([\#5722](https://github.com/matrix-org/synapse/issues/5722)) +- Cache result of get_version_string to reduce overhead of `/version` federation requests. ([\#5730](https://github.com/matrix-org/synapse/issues/5730)) +- Return 'user_type' in admin API user endpoints results. ([\#5731](https://github.com/matrix-org/synapse/issues/5731)) +- Don't package the sytest test blacklist file. ([\#5733](https://github.com/matrix-org/synapse/issues/5733)) +- Replace uses of returnValue with plain return, as returnValue is not needed on Python 3. ([\#5736](https://github.com/matrix-org/synapse/issues/5736)) +- Blacklist some flakey tests in worker mode. ([\#5740](https://github.com/matrix-org/synapse/issues/5740)) +- Fix some error cases in the caching layer. ([\#5749](https://github.com/matrix-org/synapse/issues/5749)) +- Add a prometheus metric for pending cache lookups. ([\#5750](https://github.com/matrix-org/synapse/issues/5750)) +- Stop trying to fetch events with event_id=None. ([\#5753](https://github.com/matrix-org/synapse/issues/5753)) +- Convert RedactionTestCase to modern test style. ([\#5768](https://github.com/matrix-org/synapse/issues/5768)) +- Allow looping calls to be given arguments. ([\#5780](https://github.com/matrix-org/synapse/issues/5780)) +- Set the logs emitted when checking typing and presence timeouts to DEBUG level, not INFO. ([\#5785](https://github.com/matrix-org/synapse/issues/5785)) +- Remove DelayedCall debugging from the test suite, as it is no longer required in the vast majority of Synapse's tests. ([\#5787](https://github.com/matrix-org/synapse/issues/5787)) +- Remove some spurious exceptions from the logs where we failed to talk to a remote server. ([\#5790](https://github.com/matrix-org/synapse/issues/5790)) +- Improve performance when making `.well-known` requests by sharing the SSL options between requests. ([\#5794](https://github.com/matrix-org/synapse/issues/5794)) +- Disable codecov GitHub comments on PRs. ([\#5796](https://github.com/matrix-org/synapse/issues/5796)) +- Don't allow clients to send tombstone events that reference the room it's sent in. ([\#5801](https://github.com/matrix-org/synapse/issues/5801)) +- Deny redactions of events sent in a different room. ([\#5802](https://github.com/matrix-org/synapse/issues/5802)) +- Deny sending well known state types as non-state events. ([\#5805](https://github.com/matrix-org/synapse/issues/5805)) +- Handle incorrectly encoded query params correctly by returning a 400. ([\#5808](https://github.com/matrix-org/synapse/issues/5808)) +- Handle pusher being deleted during processing rather than logging an exception. ([\#5809](https://github.com/matrix-org/synapse/issues/5809)) +- Return 502 not 500 when failing to reach any remote server. ([\#5810](https://github.com/matrix-org/synapse/issues/5810)) +- Reduce global pauses in the events stream caused by expensive state resolution during persistence. ([\#5826](https://github.com/matrix-org/synapse/issues/5826)) +- Add a lower bound to well-known lookup cache time to avoid repeated lookups. ([\#5836](https://github.com/matrix-org/synapse/issues/5836)) +- Whitelist history visbility sytests in worker mode tests. ([\#5843](https://github.com/matrix-org/synapse/issues/5843)) + + +Synapse 1.2.1 (2019-07-26) +========================== + +Security update +--------------- + +This release includes *four* security fixes: + +- Prevent an attack where a federated server could send redactions for arbitrary events in v1 and v2 rooms. ([\#5767](https://github.com/matrix-org/synapse/issues/5767)) +- Prevent a denial-of-service attack where cycles of redaction events would make Synapse spin infinitely. Thanks to `@lrizika:matrix.org` for identifying and responsibly disclosing this issue. ([0f2ecb961](https://github.com/matrix-org/synapse/commit/0f2ecb961)) +- Prevent an attack where users could be joined or parted from public rooms without their consent. Thanks to @dylangerdaly for identifying and responsibly disclosing this issue. ([\#5744](https://github.com/matrix-org/synapse/issues/5744)) +- Fix a vulnerability where a federated server could spoof read-receipts from + users on other servers. Thanks to @dylangerdaly for identifying this issue too. ([\#5743](https://github.com/matrix-org/synapse/issues/5743)) + +Additionally, the following fix was in Synapse **1.2.0**, but was not correctly +identified during the original release: + +- It was possible for a room moderator to send a redaction for an `m.room.create` event, which would downgrade the room to version 1. Thanks to `/dev/ponies` for identifying and responsibly disclosing this issue! ([\#5701](https://github.com/matrix-org/synapse/issues/5701)) + +Synapse 1.2.0 (2019-07-25) +========================== + +No significant changes. + + +Synapse 1.2.0rc2 (2019-07-24) +============================= + +Bugfixes +-------- + +- Fix a regression introduced in v1.2.0rc1 which led to incorrect labels on some prometheus metrics. ([\#5734](https://github.com/matrix-org/synapse/issues/5734)) + + +Synapse 1.2.0rc1 (2019-07-22) +============================= + +Security fixes +-------------- + +This update included a security fix which was initially incorrectly flagged as +a regular bug fix. + +- It was possible for a room moderator to send a redaction for an `m.room.create` event, which would downgrade the room to version 1. Thanks to `/dev/ponies` for identifying and responsibly disclosing this issue! ([\#5701](https://github.com/matrix-org/synapse/issues/5701)) + +Features +-------- + +- Add support for opentracing. ([\#5544](https://github.com/matrix-org/synapse/issues/5544), [\#5712](https://github.com/matrix-org/synapse/issues/5712)) +- Add ability to pull all locally stored events out of synapse that a particular user can see. ([\#5589](https://github.com/matrix-org/synapse/issues/5589)) +- Add a basic admin command app to allow server operators to run Synapse admin commands separately from the main production instance. ([\#5597](https://github.com/matrix-org/synapse/issues/5597)) +- Add `sender` and `origin_server_ts` fields to `m.replace`. ([\#5613](https://github.com/matrix-org/synapse/issues/5613)) +- Add default push rule to ignore reactions. ([\#5623](https://github.com/matrix-org/synapse/issues/5623)) +- Include the original event when asking for its relations. ([\#5626](https://github.com/matrix-org/synapse/issues/5626)) +- Implement `session_lifetime` configuration option, after which access tokens will expire. ([\#5660](https://github.com/matrix-org/synapse/issues/5660)) +- Return "This account has been deactivated" when a deactivated user tries to login. ([\#5674](https://github.com/matrix-org/synapse/issues/5674)) +- Enable aggregations support by default ([\#5714](https://github.com/matrix-org/synapse/issues/5714)) + + +Bugfixes +-------- + +- Fix 'utime went backwards' errors on daemonization. ([\#5609](https://github.com/matrix-org/synapse/issues/5609)) +- Various minor fixes to the federation request rate limiter. ([\#5621](https://github.com/matrix-org/synapse/issues/5621)) +- Forbid viewing relations on an event once it has been redacted. ([\#5629](https://github.com/matrix-org/synapse/issues/5629)) +- Fix requests to the `/store_invite` endpoint of identity servers being sent in the wrong format. ([\#5638](https://github.com/matrix-org/synapse/issues/5638)) +- Fix newly-registered users not being able to lookup their own profile without joining a room. ([\#5644](https://github.com/matrix-org/synapse/issues/5644)) +- Fix bug in #5626 that prevented the original_event field from actually having the contents of the original event in a call to `/relations`. ([\#5654](https://github.com/matrix-org/synapse/issues/5654)) +- Fix 3PID bind requests being sent to identity servers as `application/x-form-www-urlencoded` data, which is deprecated. ([\#5658](https://github.com/matrix-org/synapse/issues/5658)) +- Fix some problems with authenticating redactions in recent room versions. ([\#5699](https://github.com/matrix-org/synapse/issues/5699), [\#5700](https://github.com/matrix-org/synapse/issues/5700), [\#5707](https://github.com/matrix-org/synapse/issues/5707)) + + +Updates to the Docker image +--------------------------- + +- Base Docker image on a newer Alpine Linux version (3.8 -> 3.10). ([\#5619](https://github.com/matrix-org/synapse/issues/5619)) +- Add missing space in default logging file format generated by the Docker image. ([\#5620](https://github.com/matrix-org/synapse/issues/5620)) + + +Improved Documentation +---------------------- + +- Add information about nginx normalisation to reverse_proxy.rst. Contributed by @skalarproduktraum - thanks! ([\#5397](https://github.com/matrix-org/synapse/issues/5397)) +- --no-pep517 should be --no-use-pep517 in the documentation to setup the development environment. ([\#5651](https://github.com/matrix-org/synapse/issues/5651)) +- Improvements to Postgres setup instructions. Contributed by @Lrizika - thanks! ([\#5661](https://github.com/matrix-org/synapse/issues/5661)) +- Minor tweaks to postgres documentation. ([\#5675](https://github.com/matrix-org/synapse/issues/5675)) + + +Deprecations and Removals +------------------------- + +- Remove support for the `invite_3pid_guest` configuration setting. ([\#5625](https://github.com/matrix-org/synapse/issues/5625)) + + +Internal Changes +---------------- + +- Move logging code out of `synapse.util` and into `synapse.logging`. ([\#5606](https://github.com/matrix-org/synapse/issues/5606), [\#5617](https://github.com/matrix-org/synapse/issues/5617)) +- Add a blacklist file to the repo to blacklist certain sytests from failing CI. ([\#5611](https://github.com/matrix-org/synapse/issues/5611)) +- Make runtime errors surrounding password reset emails much clearer. ([\#5616](https://github.com/matrix-org/synapse/issues/5616)) +- Remove dead code for persiting outgoing federation transactions. ([\#5622](https://github.com/matrix-org/synapse/issues/5622)) +- Add `lint.sh` to the scripts-dev folder which will run all linting steps required by CI. ([\#5627](https://github.com/matrix-org/synapse/issues/5627)) +- Move RegistrationHandler.get_or_create_user to test code. ([\#5628](https://github.com/matrix-org/synapse/issues/5628)) +- Add some more common python virtual-environment paths to the black exclusion list. ([\#5630](https://github.com/matrix-org/synapse/issues/5630)) +- Some counter metrics exposed over Prometheus have been renamed, with the old names preserved for backwards compatibility and deprecated. See `docs/metrics-howto.rst` for details. ([\#5636](https://github.com/matrix-org/synapse/issues/5636)) +- Unblacklist some user_directory sytests. ([\#5637](https://github.com/matrix-org/synapse/issues/5637)) +- Factor out some redundant code in the login implementation. ([\#5639](https://github.com/matrix-org/synapse/issues/5639)) +- Update ModuleApi to avoid register(generate_token=True). ([\#5640](https://github.com/matrix-org/synapse/issues/5640)) +- Remove access-token support from `RegistrationHandler.register`, and rename it. ([\#5641](https://github.com/matrix-org/synapse/issues/5641)) +- Remove access-token support from `RegistrationStore.register`, and rename it. ([\#5642](https://github.com/matrix-org/synapse/issues/5642)) +- Improve logging for auto-join when a new user is created. ([\#5643](https://github.com/matrix-org/synapse/issues/5643)) +- Remove unused and unnecessary check for FederationDeniedError in _exception_to_failure. ([\#5645](https://github.com/matrix-org/synapse/issues/5645)) +- Fix a small typo in a code comment. ([\#5655](https://github.com/matrix-org/synapse/issues/5655)) +- Clean up exception handling around client access tokens. ([\#5656](https://github.com/matrix-org/synapse/issues/5656)) +- Add a mechanism for per-test homeserver configuration in the unit tests. ([\#5657](https://github.com/matrix-org/synapse/issues/5657)) +- Inline issue_access_token. ([\#5659](https://github.com/matrix-org/synapse/issues/5659)) +- Update the sytest BuildKite configuration to checkout Synapse in `/src`. ([\#5664](https://github.com/matrix-org/synapse/issues/5664)) +- Add a `docker` type to the towncrier configuration. ([\#5673](https://github.com/matrix-org/synapse/issues/5673)) +- Convert `synapse.federation.transport.server` to `async`. Might improve some stack traces. ([\#5689](https://github.com/matrix-org/synapse/issues/5689)) +- Documentation for opentracing. ([\#5703](https://github.com/matrix-org/synapse/issues/5703)) + + +Synapse 1.1.0 (2019-07-04) +========================== + +As of v1.1.0, Synapse no longer supports Python 2, nor Postgres version 9.4. +See the [upgrade notes](docs/upgrade.md#upgrading-to-v110) for more details. + +This release also deprecates the use of environment variables to configure the +docker image. See the [docker README](https://github.com/matrix-org/synapse/blob/release-v1.1.0/docker/README.md#legacy-dynamic-configuration-file-support) +for more details. + +No changes since 1.1.0rc2. + + +Synapse 1.1.0rc2 (2019-07-03) +============================= + +Bugfixes +-------- + +- Fix regression in 1.1rc1 where OPTIONS requests to the media repo would fail. ([\#5593](https://github.com/matrix-org/synapse/issues/5593)) +- Removed the `SYNAPSE_SMTP_*` docker container environment variables. Using these environment variables prevented the docker container from starting in Synapse v1.0, even though they didn't actually allow any functionality anyway. ([\#5596](https://github.com/matrix-org/synapse/issues/5596)) +- Fix a number of "Starting txn from sentinel context" warnings. ([\#5605](https://github.com/matrix-org/synapse/issues/5605)) + + +Internal Changes +---------------- + +- Update github templates. ([\#5552](https://github.com/matrix-org/synapse/issues/5552)) + + +Synapse 1.1.0rc1 (2019-07-02) +============================= + +As of v1.1.0, Synapse no longer supports Python 2, nor Postgres version 9.4. +See the [upgrade notes](docs/upgrade.md#upgrading-to-v110) for more details. + +Features +-------- + +- Added possibilty to disable local password authentication. Contributed by Daniel Hoffend. ([\#5092](https://github.com/matrix-org/synapse/issues/5092)) +- Add monthly active users to phonehome stats. ([\#5252](https://github.com/matrix-org/synapse/issues/5252)) +- Allow expired user to trigger renewal email sending manually. ([\#5363](https://github.com/matrix-org/synapse/issues/5363)) +- Statistics on forward extremities per room are now exposed via Prometheus. ([\#5384](https://github.com/matrix-org/synapse/issues/5384), [\#5458](https://github.com/matrix-org/synapse/issues/5458), [\#5461](https://github.com/matrix-org/synapse/issues/5461)) +- Add --no-daemonize option to run synapse in the foreground, per issue #4130. Contributed by Soham Gumaste. ([\#5412](https://github.com/matrix-org/synapse/issues/5412), [\#5587](https://github.com/matrix-org/synapse/issues/5587)) +- Fully support SAML2 authentication. Contributed by [Alexander Trost](https://github.com/galexrt) - thank you! ([\#5422](https://github.com/matrix-org/synapse/issues/5422)) +- Allow server admins to define implementations of extra rules for allowing or denying incoming events. ([\#5440](https://github.com/matrix-org/synapse/issues/5440), [\#5474](https://github.com/matrix-org/synapse/issues/5474), [\#5477](https://github.com/matrix-org/synapse/issues/5477)) +- Add support for handling pagination APIs on client reader worker. ([\#5505](https://github.com/matrix-org/synapse/issues/5505), [\#5513](https://github.com/matrix-org/synapse/issues/5513), [\#5531](https://github.com/matrix-org/synapse/issues/5531)) +- Improve help and cmdline option names for --generate-config options. ([\#5512](https://github.com/matrix-org/synapse/issues/5512)) +- Allow configuration of the path used for ACME account keys. ([\#5516](https://github.com/matrix-org/synapse/issues/5516), [\#5521](https://github.com/matrix-org/synapse/issues/5521), [\#5522](https://github.com/matrix-org/synapse/issues/5522)) +- Add --data-dir and --open-private-ports options. ([\#5524](https://github.com/matrix-org/synapse/issues/5524)) +- Split public rooms directory auth config in two settings, in order to manage client auth independently from the federation part of it. Obsoletes the "restrict_public_rooms_to_local_users" configuration setting. If "restrict_public_rooms_to_local_users" is set in the config, Synapse will act as if both new options are enabled, i.e. require authentication through the client API and deny federation requests. ([\#5534](https://github.com/matrix-org/synapse/issues/5534)) +- The minimum TLS version used for outgoing federation requests can now be set with `federation_client_minimum_tls_version`. ([\#5550](https://github.com/matrix-org/synapse/issues/5550)) +- Optimise devices changed query to not pull unnecessary rows from the database, reducing database load. ([\#5559](https://github.com/matrix-org/synapse/issues/5559)) +- Add new metrics for number of forward extremities being persisted and number of state groups involved in resolution. ([\#5476](https://github.com/matrix-org/synapse/issues/5476)) + +Bugfixes +-------- + +- Fix bug processing incoming events over federation if call to `/get_missing_events` fails. ([\#5042](https://github.com/matrix-org/synapse/issues/5042)) +- Prevent more than one room upgrade happening simultaneously on the same room. ([\#5051](https://github.com/matrix-org/synapse/issues/5051)) +- Fix a bug where running synapse_port_db would cause the account validity feature to fail because it didn't set the type of the email_sent column to boolean. ([\#5325](https://github.com/matrix-org/synapse/issues/5325)) +- Warn about disabling email-based password resets when a reset occurs, and remove warning when someone attempts a phone-based reset. ([\#5387](https://github.com/matrix-org/synapse/issues/5387)) +- Fix email notifications for unnamed rooms with multiple people. ([\#5388](https://github.com/matrix-org/synapse/issues/5388)) +- Fix exceptions in federation reader worker caused by attempting to renew attestations, which should only happen on master worker. ([\#5389](https://github.com/matrix-org/synapse/issues/5389)) +- Fix handling of failures fetching remote content to not log failures as exceptions. ([\#5390](https://github.com/matrix-org/synapse/issues/5390)) +- Fix a bug where deactivated users could receive renewal emails if the account validity feature is on. ([\#5394](https://github.com/matrix-org/synapse/issues/5394)) +- Fix missing invite state after exchanging 3PID invites over federaton. ([\#5464](https://github.com/matrix-org/synapse/issues/5464)) +- Fix intermittent exceptions on Apple hardware. Also fix bug that caused database activity times to be under-reported in log lines. ([\#5498](https://github.com/matrix-org/synapse/issues/5498)) +- Fix logging error when a tampered event is detected. ([\#5500](https://github.com/matrix-org/synapse/issues/5500)) +- Fix bug where clients could tight loop calling `/sync` for a period. ([\#5507](https://github.com/matrix-org/synapse/issues/5507)) +- Fix bug with `jinja2` preventing Synapse from starting. Users who had this problem should now simply need to run `pip install matrix-synapse`. ([\#5514](https://github.com/matrix-org/synapse/issues/5514)) +- Fix a regression where homeservers on private IP addresses were incorrectly blacklisted. ([\#5523](https://github.com/matrix-org/synapse/issues/5523)) +- Fixed m.login.jwt using unregistred user_id and added pyjwt>=1.6.4 as jwt conditional dependencies. Contributed by Pau Rodriguez-Estivill. ([\#5555](https://github.com/matrix-org/synapse/issues/5555), [\#5586](https://github.com/matrix-org/synapse/issues/5586)) +- Fix a bug that would cause invited users to receive several emails for a single 3PID invite in case the inviter is rate limited. ([\#5576](https://github.com/matrix-org/synapse/issues/5576)) + + +Updates to the Docker image +--------------------------- +- Add ability to change Docker containers [timezone](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) with the `TZ` variable. ([\#5383](https://github.com/matrix-org/synapse/issues/5383)) +- Update docker image to use Python 3.7. ([\#5546](https://github.com/matrix-org/synapse/issues/5546)) +- Deprecate the use of environment variables for configuration, and make the use of a static configuration the default. ([\#5561](https://github.com/matrix-org/synapse/issues/5561), [\#5562](https://github.com/matrix-org/synapse/issues/5562), [\#5566](https://github.com/matrix-org/synapse/issues/5566), [\#5567](https://github.com/matrix-org/synapse/issues/5567)) +- Increase default log level for docker image to INFO. It can still be changed by editing the generated log.config file. ([\#5547](https://github.com/matrix-org/synapse/issues/5547)) +- Send synapse logs to the docker logging system, by default. ([\#5565](https://github.com/matrix-org/synapse/issues/5565)) +- Open the non-TLS port by default. ([\#5568](https://github.com/matrix-org/synapse/issues/5568)) +- Fix failure to start under docker with SAML support enabled. ([\#5490](https://github.com/matrix-org/synapse/issues/5490)) +- Use a sensible location for data files when generating a config file. ([\#5563](https://github.com/matrix-org/synapse/issues/5563)) + + +Deprecations and Removals +------------------------- + +- Python 2.7 is no longer a supported platform. Synapse now requires Python 3.5+ to run. ([\#5425](https://github.com/matrix-org/synapse/issues/5425)) +- PostgreSQL 9.4 is no longer supported. Synapse requires Postgres 9.5+ or above for Postgres support. ([\#5448](https://github.com/matrix-org/synapse/issues/5448)) +- Remove support for cpu_affinity setting. ([\#5525](https://github.com/matrix-org/synapse/issues/5525)) + + +Improved Documentation +---------------------- +- Improve README section on performance troubleshooting. ([\#4276](https://github.com/matrix-org/synapse/issues/4276)) +- Add information about how to install and run `black` on the codebase to code_style.rst. ([\#5537](https://github.com/matrix-org/synapse/issues/5537)) +- Improve install docs on choosing server_name. ([\#5558](https://github.com/matrix-org/synapse/issues/5558)) + + +Internal Changes +---------------- + +- Add logging to 3pid invite signature verification. ([\#5015](https://github.com/matrix-org/synapse/issues/5015)) +- Update example haproxy config to a more compatible setup. ([\#5313](https://github.com/matrix-org/synapse/issues/5313)) +- Track deactivated accounts in the database. ([\#5378](https://github.com/matrix-org/synapse/issues/5378), [\#5465](https://github.com/matrix-org/synapse/issues/5465), [\#5493](https://github.com/matrix-org/synapse/issues/5493)) +- Clean up code for sending federation EDUs. ([\#5381](https://github.com/matrix-org/synapse/issues/5381)) +- Add a sponsor button to the repo. ([\#5382](https://github.com/matrix-org/synapse/issues/5382), [\#5386](https://github.com/matrix-org/synapse/issues/5386)) +- Don't log non-200 responses from federation queries as exceptions. ([\#5383](https://github.com/matrix-org/synapse/issues/5383)) +- Update Python syntax in contrib/ to Python 3. ([\#5446](https://github.com/matrix-org/synapse/issues/5446)) +- Update federation_client dev script to support `.well-known` and work with python3. ([\#5447](https://github.com/matrix-org/synapse/issues/5447)) +- SyTest has been moved to Buildkite. ([\#5459](https://github.com/matrix-org/synapse/issues/5459)) +- Demo script now uses python3. ([\#5460](https://github.com/matrix-org/synapse/issues/5460)) +- Synapse can now handle RestServlets that return coroutines. ([\#5475](https://github.com/matrix-org/synapse/issues/5475), [\#5585](https://github.com/matrix-org/synapse/issues/5585)) +- The demo servers talk to each other again. ([\#5478](https://github.com/matrix-org/synapse/issues/5478)) +- Add an EXPERIMENTAL config option to try and periodically clean up extremities by sending dummy events. ([\#5480](https://github.com/matrix-org/synapse/issues/5480)) +- Synapse's codebase is now formatted by `black`. ([\#5482](https://github.com/matrix-org/synapse/issues/5482)) +- Some cleanups and sanity-checking in the CPU and database metrics. ([\#5499](https://github.com/matrix-org/synapse/issues/5499)) +- Improve email notification logging. ([\#5502](https://github.com/matrix-org/synapse/issues/5502)) +- Fix "Unexpected entry in 'full_schemas'" log warning. ([\#5509](https://github.com/matrix-org/synapse/issues/5509)) +- Improve logging when generating config files. ([\#5510](https://github.com/matrix-org/synapse/issues/5510)) +- Refactor and clean up Config parser for maintainability. ([\#5511](https://github.com/matrix-org/synapse/issues/5511)) +- Make the config clearer in that email.template_dir is relative to the Synapse's root directory, not the `synapse/` folder within it. ([\#5543](https://github.com/matrix-org/synapse/issues/5543)) +- Update v1.0.0 release changelog to include more information about changes to password resets. ([\#5545](https://github.com/matrix-org/synapse/issues/5545)) +- Remove non-functioning check_event_hash.py dev script. ([\#5548](https://github.com/matrix-org/synapse/issues/5548)) +- Synapse will now only allow TLS v1.2 connections when serving federation, if it terminates TLS. As Synapse's allowed ciphers were only able to be used in TLSv1.2 before, this does not change behaviour. ([\#5550](https://github.com/matrix-org/synapse/issues/5550)) +- Logging when running GC collection on generation 0 is now at the DEBUG level, not INFO. ([\#5557](https://github.com/matrix-org/synapse/issues/5557)) +- Reduce the amount of stuff we send in the docker context. ([\#5564](https://github.com/matrix-org/synapse/issues/5564)) +- Point the reverse links in the Purge History contrib scripts at the intended location. ([\#5570](https://github.com/matrix-org/synapse/issues/5570)) + + +Synapse 1.0.0 (2019-06-11) +========================== + +Bugfixes +-------- + +- Fix bug where attempting to send transactions with large number of EDUs can fail. ([\#5418](https://github.com/matrix-org/synapse/issues/5418)) + + +Improved Documentation +---------------------- + +- Expand the federation guide to include relevant content from the MSC1711 FAQ ([\#5419](https://github.com/matrix-org/synapse/issues/5419)) + + +Internal Changes +---------------- + +- Move password reset links to /_matrix/client/unstable namespace. ([\#5424](https://github.com/matrix-org/synapse/issues/5424)) + + +Synapse 1.0.0rc3 (2019-06-10) +============================= + +Security: Fix authentication bug introduced in 1.0.0rc1. Please upgrade to rc3 immediately + + +Synapse 1.0.0rc2 (2019-06-10) +============================= + +Bugfixes +-------- + +- Remove redundant warning about key server response validation. ([\#5392](https://github.com/matrix-org/synapse/issues/5392)) +- Fix bug where old keys stored in the database with a null valid until timestamp caused all verification requests for that key to fail. ([\#5415](https://github.com/matrix-org/synapse/issues/5415)) +- Fix excessive memory using with default `federation_verify_certificates: true` configuration. ([\#5417](https://github.com/matrix-org/synapse/issues/5417)) + + +Synapse 1.0.0rc1 (2019-06-07) +============================= + +Features +-------- + +- Synapse now more efficiently collates room statistics. ([\#4338](https://github.com/matrix-org/synapse/issues/4338), [\#5260](https://github.com/matrix-org/synapse/issues/5260), [\#5324](https://github.com/matrix-org/synapse/issues/5324)) +- Add experimental support for relations (aka reactions and edits). ([\#5220](https://github.com/matrix-org/synapse/issues/5220)) +- Ability to configure default room version. ([\#5223](https://github.com/matrix-org/synapse/issues/5223), [\#5249](https://github.com/matrix-org/synapse/issues/5249)) +- Allow configuring a range for the account validity startup job. ([\#5276](https://github.com/matrix-org/synapse/issues/5276)) +- CAS login will now hit the r0 API, not the deprecated v1 one. ([\#5286](https://github.com/matrix-org/synapse/issues/5286)) +- Validate federation server TLS certificates by default (implements [MSC1711](https://github.com/matrix-org/matrix-doc/blob/master/proposals/1711-x509-for-federation.md)). ([\#5359](https://github.com/matrix-org/synapse/issues/5359)) +- Update /_matrix/client/versions to reference support for r0.5.0. ([\#5360](https://github.com/matrix-org/synapse/issues/5360)) +- Add a script to generate new signing-key files. ([\#5361](https://github.com/matrix-org/synapse/issues/5361)) +- Update upgrade and installation guides ahead of 1.0. ([\#5371](https://github.com/matrix-org/synapse/issues/5371)) +- Replace the `perspectives` configuration section with `trusted_key_servers`, and make validating the signatures on responses optional (since TLS will do this job for us). ([\#5374](https://github.com/matrix-org/synapse/issues/5374)) +- Add ability to perform password reset via email without trusting the identity server. **As a result of this PR, password resets will now be disabled on the default configuration.** + + Password reset emails are now sent from the homeserver by default, instead of the identity server. To enable this functionality, ensure `email` and `public_baseurl` config options are filled out. + + If you would like to re-enable password resets being sent from the identity server (warning: this is dangerous! See [#5345](https://github.com/matrix-org/synapse/pull/5345)), set `email.trust_identity_server_for_password_resets` to true. ([\#5377](https://github.com/matrix-org/synapse/issues/5377)) +- Set default room version to v4. ([\#5379](https://github.com/matrix-org/synapse/issues/5379)) + + +Bugfixes +-------- + +- Fixes client-server API not sending "m.heroes" to lazy-load /sync requests when a rooms name or its canonical alias are empty. Thanks to @dnaf for this work! ([\#5089](https://github.com/matrix-org/synapse/issues/5089)) +- Prevent federation device list updates breaking when processing multiple updates at once. ([\#5156](https://github.com/matrix-org/synapse/issues/5156)) +- Fix worker registration bug caused by ClientReaderSlavedStore being unable to see get_profileinfo. ([\#5200](https://github.com/matrix-org/synapse/issues/5200)) +- Fix race when backfilling in rooms with worker mode. ([\#5221](https://github.com/matrix-org/synapse/issues/5221)) +- Fix appservice timestamp massaging. ([\#5233](https://github.com/matrix-org/synapse/issues/5233)) +- Ensure that server_keys fetched via a notary server are correctly signed. ([\#5251](https://github.com/matrix-org/synapse/issues/5251)) +- Show the correct error when logging out and access token is missing. ([\#5256](https://github.com/matrix-org/synapse/issues/5256)) +- Fix error code when there is an invalid parameter on /_matrix/client/r0/publicRooms ([\#5257](https://github.com/matrix-org/synapse/issues/5257)) +- Fix error when downloading thumbnail with missing width/height parameter. ([\#5258](https://github.com/matrix-org/synapse/issues/5258)) +- Fix schema update for account validity. ([\#5268](https://github.com/matrix-org/synapse/issues/5268)) +- Fix bug where we leaked extremities when we soft failed events, leading to performance degradation. ([\#5274](https://github.com/matrix-org/synapse/issues/5274), [\#5278](https://github.com/matrix-org/synapse/issues/5278), [\#5291](https://github.com/matrix-org/synapse/issues/5291)) +- Fix "db txn 'update_presence' from sentinel context" log messages. ([\#5275](https://github.com/matrix-org/synapse/issues/5275)) +- Fix dropped logcontexts during high outbound traffic. ([\#5277](https://github.com/matrix-org/synapse/issues/5277)) +- Fix a bug where it is not possible to get events in the federation format with the request `GET /_matrix/client/r0/rooms/{roomId}/messages`. ([\#5293](https://github.com/matrix-org/synapse/issues/5293)) +- Fix performance problems with the rooms stats background update. ([\#5294](https://github.com/matrix-org/synapse/issues/5294)) +- Fix noisy 'no key for server' logs. ([\#5300](https://github.com/matrix-org/synapse/issues/5300)) +- Fix bug where a notary server would sometimes forget old keys. ([\#5307](https://github.com/matrix-org/synapse/issues/5307)) +- Prevent users from setting huge displaynames and avatar URLs. ([\#5309](https://github.com/matrix-org/synapse/issues/5309)) +- Fix handling of failures when processing incoming events where calling `/event_auth` on remote server fails. ([\#5317](https://github.com/matrix-org/synapse/issues/5317)) +- Ensure that we have an up-to-date copy of the signing key when validating incoming federation requests. ([\#5321](https://github.com/matrix-org/synapse/issues/5321)) +- Fix various problems which made the signing-key notary server time out for some requests. ([\#5333](https://github.com/matrix-org/synapse/issues/5333)) +- Fix bug which would make certain operations (such as room joins) block for 20 minutes while attemoting to fetch verification keys. ([\#5334](https://github.com/matrix-org/synapse/issues/5334)) +- Fix a bug where we could rapidly mark a server as unreachable even though it was only down for a few minutes. ([\#5335](https://github.com/matrix-org/synapse/issues/5335), [\#5340](https://github.com/matrix-org/synapse/issues/5340)) +- Fix a bug where account validity renewal emails could only be sent when email notifs were enabled. ([\#5341](https://github.com/matrix-org/synapse/issues/5341)) +- Fix failure when fetching batches of events during backfill, etc. ([\#5342](https://github.com/matrix-org/synapse/issues/5342)) +- Add a new room version where the timestamps on events are checked against the validity periods on signing keys. ([\#5348](https://github.com/matrix-org/synapse/issues/5348), [\#5354](https://github.com/matrix-org/synapse/issues/5354)) +- Fix room stats and presence background updates to correctly handle missing events. ([\#5352](https://github.com/matrix-org/synapse/issues/5352)) +- Include left members in room summaries' heroes. ([\#5355](https://github.com/matrix-org/synapse/issues/5355)) +- Fix `federation_custom_ca_list` configuration option. ([\#5362](https://github.com/matrix-org/synapse/issues/5362)) +- Fix missing logcontext warnings on shutdown. ([\#5369](https://github.com/matrix-org/synapse/issues/5369)) + + +Improved Documentation +---------------------- + +- Fix docs on resetting the user directory. ([\#5282](https://github.com/matrix-org/synapse/issues/5282)) +- Fix notes about ACME in the MSC1711 faq. ([\#5357](https://github.com/matrix-org/synapse/issues/5357)) + + +Internal Changes +---------------- + +- Synapse will now serve the experimental "room complexity" API endpoint. ([\#5216](https://github.com/matrix-org/synapse/issues/5216)) +- The base classes for the v1 and v2_alpha REST APIs have been unified. ([\#5226](https://github.com/matrix-org/synapse/issues/5226), [\#5328](https://github.com/matrix-org/synapse/issues/5328)) +- Simplifications and comments in do_auth. ([\#5227](https://github.com/matrix-org/synapse/issues/5227)) +- Remove urllib3 pin as requests 2.22.0 has been released supporting urllib3 1.25.2. ([\#5230](https://github.com/matrix-org/synapse/issues/5230)) +- Preparatory work for key-validity features. ([\#5232](https://github.com/matrix-org/synapse/issues/5232), [\#5234](https://github.com/matrix-org/synapse/issues/5234), [\#5235](https://github.com/matrix-org/synapse/issues/5235), [\#5236](https://github.com/matrix-org/synapse/issues/5236), [\#5237](https://github.com/matrix-org/synapse/issues/5237), [\#5244](https://github.com/matrix-org/synapse/issues/5244), [\#5250](https://github.com/matrix-org/synapse/issues/5250), [\#5296](https://github.com/matrix-org/synapse/issues/5296), [\#5299](https://github.com/matrix-org/synapse/issues/5299), [\#5343](https://github.com/matrix-org/synapse/issues/5343), [\#5347](https://github.com/matrix-org/synapse/issues/5347), [\#5356](https://github.com/matrix-org/synapse/issues/5356)) +- Specify the type of reCAPTCHA key to use. ([\#5283](https://github.com/matrix-org/synapse/issues/5283)) +- Improve sample config for monthly active user blocking. ([\#5284](https://github.com/matrix-org/synapse/issues/5284)) +- Remove spurious debug from MatrixFederationHttpClient.get_json. ([\#5287](https://github.com/matrix-org/synapse/issues/5287)) +- Improve logging for logcontext leaks. ([\#5288](https://github.com/matrix-org/synapse/issues/5288)) +- Clarify that the admin change password API logs the user out. ([\#5303](https://github.com/matrix-org/synapse/issues/5303)) +- New installs will now use the v54 full schema, rather than the full schema v14 and applying incremental updates to v54. ([\#5320](https://github.com/matrix-org/synapse/issues/5320)) +- Improve docstrings on MatrixFederationClient. ([\#5332](https://github.com/matrix-org/synapse/issues/5332)) +- Clean up FederationClient.get_events for clarity. ([\#5344](https://github.com/matrix-org/synapse/issues/5344)) +- Various improvements to debug logging. ([\#5353](https://github.com/matrix-org/synapse/issues/5353)) +- Don't run CI build checks until sample config check has passed. ([\#5370](https://github.com/matrix-org/synapse/issues/5370)) +- Automatically retry buildkite builds (max twice) when an agent is lost. ([\#5380](https://github.com/matrix-org/synapse/issues/5380)) + +**Changelogs for versions older than 1.0.0 can be found [here](CHANGES-pre-1.0.md).** diff --git a/docs/changelogs/CHANGES-2020.md b/docs/changelogs/CHANGES-2020.md new file mode 100644 index 0000000000..6b87022251 --- /dev/null +++ b/docs/changelogs/CHANGES-2020.md @@ -0,0 +1,2145 @@ +Synapse 1.24.0 (2020-12-09) +=========================== + +Due to the two security issues highlighted below, server administrators are +encouraged to update Synapse. We are not aware of these vulnerabilities being +exploited in the wild. + +Security advisory +----------------- + +The following issues are fixed in v1.23.1 and v1.24.0. + +- There is a denial of service attack + ([CVE-2020-26257](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-26257)) + against the federation APIs in which future events will not be correctly sent + to other servers over federation. This affects all servers that participate in + open federation. (Fixed in [#8776](https://github.com/matrix-org/synapse/pull/8776)). + +- Synapse may be affected by OpenSSL + [CVE-2020-1971](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-1971). + Synapse administrators should ensure that they have the latest versions of + the cryptography Python package installed. + +To upgrade Synapse along with the cryptography package: + +* Administrators using the [`matrix.org` Docker + image](https://hub.docker.com/r/matrixdotorg/synapse/) or the [Debian/Ubuntu + packages from + `matrix.org`](https://matrix-org.github.io/synapse/latest/setup/installation.html#matrixorg-packages) + should ensure that they have version 1.24.0 or 1.23.1 installed: these images include + the updated packages. +* Administrators who have [installed Synapse from + source](https://matrix-org.github.io/synapse/latest/setup/installation.html#installing-from-source) + should upgrade the cryptography package within their virtualenv by running: + ```sh + /bin/pip install 'cryptography>=3.3' + ``` +* Administrators who have installed Synapse from distribution packages should + consult the information from their distributions. + +Internal Changes +---------------- + +- Add a maximum version for pysaml2 on Python 3.5. ([\#8898](https://github.com/matrix-org/synapse/issues/8898)) + + +Synapse 1.23.1 (2020-12-09) +=========================== + +Due to the two security issues highlighted below, server administrators are +encouraged to update Synapse. We are not aware of these vulnerabilities being +exploited in the wild. + +Security advisory +----------------- + +The following issues are fixed in v1.23.1 and v1.24.0. + +- There is a denial of service attack + ([CVE-2020-26257](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-26257)) + against the federation APIs in which future events will not be correctly sent + to other servers over federation. This affects all servers that participate in + open federation. (Fixed in [#8776](https://github.com/matrix-org/synapse/pull/8776)). + +- Synapse may be affected by OpenSSL + [CVE-2020-1971](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-1971). + Synapse administrators should ensure that they have the latest versions of + the cryptography Python package installed. + +To upgrade Synapse along with the cryptography package: + +* Administrators using the [`matrix.org` Docker + image](https://hub.docker.com/r/matrixdotorg/synapse/) or the [Debian/Ubuntu + packages from + `matrix.org`](https://matrix-org.github.io/synapse/latest/setup/installation.html#matrixorg-packages) + should ensure that they have version 1.24.0 or 1.23.1 installed: these images include + the updated packages. +* Administrators who have [installed Synapse from + source](https://matrix-org.github.io/synapse/latest/setup/installation.html#installing-from-source) + should upgrade the cryptography package within their virtualenv by running: + ```sh + /bin/pip install 'cryptography>=3.3' + ``` +* Administrators who have installed Synapse from distribution packages should + consult the information from their distributions. + +Bugfixes +-------- + +- Fix a bug in some federation APIs which could lead to unexpected behaviour if different parameters were set in the URI and the request body. ([\#8776](https://github.com/matrix-org/synapse/issues/8776)) + + +Internal Changes +---------------- + +- Add a maximum version for pysaml2 on Python 3.5. ([\#8898](https://github.com/matrix-org/synapse/issues/8898)) + + +Synapse 1.24.0rc2 (2020-12-04) +============================== + +Bugfixes +-------- + +- Fix a regression in v1.24.0rc1 which failed to allow SAML mapping providers which were unable to redirect users to an additional page. ([\#8878](https://github.com/matrix-org/synapse/issues/8878)) + + +Internal Changes +---------------- + +- Add support for the `prometheus_client` newer than 0.9.0. Contributed by Jordan Bancino. ([\#8875](https://github.com/matrix-org/synapse/issues/8875)) + + +Synapse 1.24.0rc1 (2020-12-02) +============================== + +Features +-------- + +- Add admin API for logging in as a user. ([\#8617](https://github.com/matrix-org/synapse/issues/8617)) +- Allow specification of the SAML IdP if the metadata returns multiple IdPs. ([\#8630](https://github.com/matrix-org/synapse/issues/8630)) +- Add support for re-trying generation of a localpart for OpenID Connect mapping providers. ([\#8801](https://github.com/matrix-org/synapse/issues/8801), [\#8855](https://github.com/matrix-org/synapse/issues/8855)) +- Allow the `Date` header through CORS. Contributed by Nicolas Chamo. ([\#8804](https://github.com/matrix-org/synapse/issues/8804)) +- Add a config option, `push.group_by_unread_count`, which controls whether unread message counts in push notifications are defined as "the number of rooms with unread messages" or "total unread messages". ([\#8820](https://github.com/matrix-org/synapse/issues/8820)) +- Add `force_purge` option to delete-room admin api. ([\#8843](https://github.com/matrix-org/synapse/issues/8843)) + + +Bugfixes +-------- + +- Fix a bug where appservices may be sent an excessive amount of read receipts and presence. Broke in v1.22.0. ([\#8744](https://github.com/matrix-org/synapse/issues/8744)) +- Fix a bug in some federation APIs which could lead to unexpected behaviour if different parameters were set in the URI and the request body. ([\#8776](https://github.com/matrix-org/synapse/issues/8776)) +- Fix a bug where synctl could spawn duplicate copies of a worker. Contributed by Waylon Cude. ([\#8798](https://github.com/matrix-org/synapse/issues/8798)) +- Allow per-room profiles to be used for the server notice user. ([\#8799](https://github.com/matrix-org/synapse/issues/8799)) +- Fix a bug where logging could break after a call to SIGHUP. ([\#8817](https://github.com/matrix-org/synapse/issues/8817)) +- Fix `register_new_matrix_user` failing with "Bad Request" when trailing slash is included in server URL. Contributed by @angdraug. ([\#8823](https://github.com/matrix-org/synapse/issues/8823)) +- Fix a minor long-standing bug in login, where we would offer the `password` login type if a custom auth provider supported it, even if password login was disabled. ([\#8835](https://github.com/matrix-org/synapse/issues/8835)) +- Fix a long-standing bug which caused Synapse to require unspecified parameters during user-interactive authentication. ([\#8848](https://github.com/matrix-org/synapse/issues/8848)) +- Fix a bug introduced in v1.20.0 where the user-agent and IP address reported during user registration for CAS, OpenID Connect, and SAML were of the wrong form. ([\#8784](https://github.com/matrix-org/synapse/issues/8784)) + + +Improved Documentation +---------------------- + +- Clarify the usecase for a msisdn delegate. Contributed by Adrian Wannenmacher. ([\#8734](https://github.com/matrix-org/synapse/issues/8734)) +- Remove extraneous comma from JSON example in User Admin API docs. ([\#8771](https://github.com/matrix-org/synapse/issues/8771)) +- Update `turn-howto.md` with troubleshooting notes. ([\#8779](https://github.com/matrix-org/synapse/issues/8779)) +- Fix the example on how to set the `Content-Type` header in nginx for the Client Well-Known URI. ([\#8793](https://github.com/matrix-org/synapse/issues/8793)) +- Improve the documentation for the admin API to list all media in a room with respect to encrypted events. ([\#8795](https://github.com/matrix-org/synapse/issues/8795)) +- Update the formatting of the `push` section of the homeserver config file to better align with the [code style guidelines](https://github.com/matrix-org/synapse/blob/develop/docs/code_style.md#configuration-file-format). ([\#8818](https://github.com/matrix-org/synapse/issues/8818)) +- Improve documentation how to configure prometheus for workers. ([\#8822](https://github.com/matrix-org/synapse/issues/8822)) +- Update example prometheus console. ([\#8824](https://github.com/matrix-org/synapse/issues/8824)) + + +Deprecations and Removals +------------------------- + +- Remove old `/_matrix/client/*/admin` endpoints which were deprecated since Synapse 1.20.0. ([\#8785](https://github.com/matrix-org/synapse/issues/8785)) +- Disable pretty printing JSON responses for curl. Users who want pretty-printed output should use [jq](https://stedolan.github.io/jq/) in combination with curl. Contributed by @tulir. ([\#8833](https://github.com/matrix-org/synapse/issues/8833)) + + +Internal Changes +---------------- + +- Simplify the way the `HomeServer` object caches its internal attributes. ([\#8565](https://github.com/matrix-org/synapse/issues/8565), [\#8851](https://github.com/matrix-org/synapse/issues/8851)) +- Add an example and documentation for clock skew to the SAML2 sample configuration to allow for clock/time difference between the homserver and IdP. Contributed by @localguru. ([\#8731](https://github.com/matrix-org/synapse/issues/8731)) +- Generalise `RoomMemberHandler._locally_reject_invite` to apply to more flows than just invite. ([\#8751](https://github.com/matrix-org/synapse/issues/8751)) +- Generalise `RoomStore.maybe_store_room_on_invite` to handle other, non-invite membership events. ([\#8754](https://github.com/matrix-org/synapse/issues/8754)) +- Refactor test utilities for injecting HTTP requests. ([\#8757](https://github.com/matrix-org/synapse/issues/8757), [\#8758](https://github.com/matrix-org/synapse/issues/8758), [\#8759](https://github.com/matrix-org/synapse/issues/8759), [\#8760](https://github.com/matrix-org/synapse/issues/8760), [\#8761](https://github.com/matrix-org/synapse/issues/8761), [\#8777](https://github.com/matrix-org/synapse/issues/8777)) +- Consolidate logic between the OpenID Connect and SAML code. ([\#8765](https://github.com/matrix-org/synapse/issues/8765)) +- Use `TYPE_CHECKING` instead of magic `MYPY` variable. ([\#8770](https://github.com/matrix-org/synapse/issues/8770)) +- Add a commandline script to sign arbitrary json objects. ([\#8772](https://github.com/matrix-org/synapse/issues/8772)) +- Minor log line improvements for the SSO mapping code used to generate Matrix IDs from SSO IDs. ([\#8773](https://github.com/matrix-org/synapse/issues/8773)) +- Add additional error checking for OpenID Connect and SAML mapping providers. ([\#8774](https://github.com/matrix-org/synapse/issues/8774), [\#8800](https://github.com/matrix-org/synapse/issues/8800)) +- Add type hints to HTTP abstractions. ([\#8806](https://github.com/matrix-org/synapse/issues/8806), [\#8812](https://github.com/matrix-org/synapse/issues/8812)) +- Remove unnecessary function arguments and add typing to several membership replication classes. ([\#8809](https://github.com/matrix-org/synapse/issues/8809)) +- Optimise the lookup for an invite from another homeserver when trying to reject it. ([\#8815](https://github.com/matrix-org/synapse/issues/8815)) +- Add tests for `password_auth_provider`s. ([\#8819](https://github.com/matrix-org/synapse/issues/8819)) +- Drop redundant database index on `event_json`. ([\#8845](https://github.com/matrix-org/synapse/issues/8845)) +- Simplify `uk.half-shot.msc2778.login.application_service` login handler. ([\#8847](https://github.com/matrix-org/synapse/issues/8847)) +- Refactor `password_auth_provider` support code. ([\#8849](https://github.com/matrix-org/synapse/issues/8849)) +- Add missing `ordering` to background database updates. ([\#8850](https://github.com/matrix-org/synapse/issues/8850)) +- Allow for specifying a room version when creating a room in unit tests via `RestHelper.create_room_as`. ([\#8854](https://github.com/matrix-org/synapse/issues/8854)) + + +Synapse 1.23.0 (2020-11-18) +=========================== + +This release changes the way structured logging is configured. See the [upgrade notes](docs/upgrade.md#upgrading-to-v1230) for details. + +**Note**: We are aware of a trivially exploitable denial of service vulnerability in versions of Synapse prior to 1.20.0. Complete details will be disclosed on Monday, November 23rd. If you have not upgraded recently, please do so. + +Bugfixes +-------- + +- Fix a dependency versioning bug in the Dockerfile that prevented Synapse from starting. ([\#8767](https://github.com/matrix-org/synapse/issues/8767)) + + +Synapse 1.23.0rc1 (2020-11-13) +============================== + +Features +-------- + +- Add a push rule that highlights when a jitsi conference is created in a room. ([\#8286](https://github.com/matrix-org/synapse/issues/8286)) +- Add an admin api to delete a single file or files that were not used for a defined time from server. Contributed by @dklimpel. ([\#8519](https://github.com/matrix-org/synapse/issues/8519)) +- Split admin API for reported events (`GET /_synapse/admin/v1/event_reports`) into detail and list endpoints. This is a breaking change to #8217 which was introduced in Synapse v1.21.0. Those who already use this API should check their scripts. Contributed by @dklimpel. ([\#8539](https://github.com/matrix-org/synapse/issues/8539)) +- Support generating structured logs via the standard logging configuration. ([\#8607](https://github.com/matrix-org/synapse/issues/8607), [\#8685](https://github.com/matrix-org/synapse/issues/8685)) +- Add an admin API to allow server admins to list users' pushers. Contributed by @dklimpel. ([\#8610](https://github.com/matrix-org/synapse/issues/8610), [\#8689](https://github.com/matrix-org/synapse/issues/8689)) +- Add an admin API `GET /_synapse/admin/v1/users//media` to get information about uploaded media. Contributed by @dklimpel. ([\#8647](https://github.com/matrix-org/synapse/issues/8647)) +- Add an admin API for local user media statistics. Contributed by @dklimpel. ([\#8700](https://github.com/matrix-org/synapse/issues/8700)) +- Add `displayname` to Shared-Secret Registration for admins. ([\#8722](https://github.com/matrix-org/synapse/issues/8722)) + + +Bugfixes +-------- + +- Fix fetching of E2E cross signing keys over federation when only one of the master key and device signing key is cached already. ([\#8455](https://github.com/matrix-org/synapse/issues/8455)) +- Fix a bug where Synapse would blindly forward bad responses from federation to clients when retrieving profile information. ([\#8580](https://github.com/matrix-org/synapse/issues/8580)) +- Fix a bug where the account validity endpoint would silently fail if the user ID did not have an expiration time. It now returns a 400 error. ([\#8620](https://github.com/matrix-org/synapse/issues/8620)) +- Fix email notifications for invites without local state. ([\#8627](https://github.com/matrix-org/synapse/issues/8627)) +- Fix handling of invalid group IDs to return a 400 rather than log an exception and return a 500. ([\#8628](https://github.com/matrix-org/synapse/issues/8628)) +- Fix handling of User-Agent headers that are invalid UTF-8, which caused user agents of users to not get correctly recorded. ([\#8632](https://github.com/matrix-org/synapse/issues/8632)) +- Fix a bug in the `joined_rooms` admin API if the user has never joined any rooms. The bug was introduced, along with the API, in v1.21.0. ([\#8643](https://github.com/matrix-org/synapse/issues/8643)) +- Fix exception during handling multiple concurrent requests for remote media when using multiple media repositories. ([\#8682](https://github.com/matrix-org/synapse/issues/8682)) +- Fix bug that prevented Synapse from recovering after losing connection to the database. ([\#8726](https://github.com/matrix-org/synapse/issues/8726)) +- Fix bug where the `/_synapse/admin/v1/send_server_notice` API could send notices to non-notice rooms. ([\#8728](https://github.com/matrix-org/synapse/issues/8728)) +- Fix PostgreSQL port script fails when DB has no backfilled events. Broke in v1.21.0. ([\#8729](https://github.com/matrix-org/synapse/issues/8729)) +- Fix PostgreSQL port script to correctly handle foreign key constraints. Broke in v1.21.0. ([\#8730](https://github.com/matrix-org/synapse/issues/8730)) +- Fix PostgreSQL port script so that it can be run again after a failure. Broke in v1.21.0. ([\#8755](https://github.com/matrix-org/synapse/issues/8755)) + + +Improved Documentation +---------------------- + +- Instructions for Azure AD in the OpenID Connect documentation. Contributed by peterk. ([\#8582](https://github.com/matrix-org/synapse/issues/8582)) +- Improve the sample configuration for single sign-on providers. ([\#8635](https://github.com/matrix-org/synapse/issues/8635)) +- Fix the filepath of Dex's example config and the link to Dex's Getting Started guide in the OpenID Connect docs. ([\#8657](https://github.com/matrix-org/synapse/issues/8657)) +- Note support for Python 3.9. ([\#8665](https://github.com/matrix-org/synapse/issues/8665)) +- Minor updates to docs on running tests. ([\#8666](https://github.com/matrix-org/synapse/issues/8666)) +- Interlink prometheus/grafana documentation. ([\#8667](https://github.com/matrix-org/synapse/issues/8667)) +- Notes on SSO logins and media_repository worker. ([\#8701](https://github.com/matrix-org/synapse/issues/8701)) +- Document experimental support for running multiple event persisters. ([\#8706](https://github.com/matrix-org/synapse/issues/8706)) +- Add information regarding the various sources of, and expected contributions to, Synapse's documentation to `CONTRIBUTING.md`. ([\#8714](https://github.com/matrix-org/synapse/issues/8714)) +- Migrate documentation `docs/admin_api/event_reports` to markdown. ([\#8742](https://github.com/matrix-org/synapse/issues/8742)) +- Add some helpful hints to the README for new Synapse developers. Contributed by @chagai95. ([\#8746](https://github.com/matrix-org/synapse/issues/8746)) + + +Internal Changes +---------------- + +- Optimise `/createRoom` with multiple invited users. ([\#8559](https://github.com/matrix-org/synapse/issues/8559)) +- Implement and use an `@lru_cache` decorator. ([\#8595](https://github.com/matrix-org/synapse/issues/8595)) +- Don't instansiate Requester directly. ([\#8614](https://github.com/matrix-org/synapse/issues/8614)) +- Type hints for `RegistrationStore`. ([\#8615](https://github.com/matrix-org/synapse/issues/8615)) +- Change schema to support access tokens belonging to one user but granting access to another. ([\#8616](https://github.com/matrix-org/synapse/issues/8616)) +- Remove unused OPTIONS handlers. ([\#8621](https://github.com/matrix-org/synapse/issues/8621)) +- Run `mypy` as part of the lint.sh script. ([\#8633](https://github.com/matrix-org/synapse/issues/8633)) +- Correct Synapse's PyPI package name in the OpenID Connect installation instructions. ([\#8634](https://github.com/matrix-org/synapse/issues/8634)) +- Catch exceptions during initialization of `password_providers`. Contributed by Nicolai Søborg. ([\#8636](https://github.com/matrix-org/synapse/issues/8636)) +- Fix typos and spelling errors in the code. ([\#8639](https://github.com/matrix-org/synapse/issues/8639)) +- Reduce number of OpenTracing spans started. ([\#8640](https://github.com/matrix-org/synapse/issues/8640), [\#8668](https://github.com/matrix-org/synapse/issues/8668), [\#8670](https://github.com/matrix-org/synapse/issues/8670)) +- Add field `total` to device list in admin API. ([\#8644](https://github.com/matrix-org/synapse/issues/8644)) +- Add more type hints to the application services code. ([\#8655](https://github.com/matrix-org/synapse/issues/8655), [\#8693](https://github.com/matrix-org/synapse/issues/8693)) +- Tell Black to format code for Python 3.5. ([\#8664](https://github.com/matrix-org/synapse/issues/8664)) +- Don't pull event from DB when handling replication traffic. ([\#8669](https://github.com/matrix-org/synapse/issues/8669)) +- Abstract some invite-related code in preparation for landing knocking. ([\#8671](https://github.com/matrix-org/synapse/issues/8671), [\#8688](https://github.com/matrix-org/synapse/issues/8688)) +- Clarify representation of events in logfiles. ([\#8679](https://github.com/matrix-org/synapse/issues/8679)) +- Don't require `hiredis` package to be installed to run unit tests. ([\#8680](https://github.com/matrix-org/synapse/issues/8680)) +- Fix typing info on cache call signature to accept `on_invalidate`. ([\#8684](https://github.com/matrix-org/synapse/issues/8684)) +- Fail tests if they do not await coroutines. ([\#8690](https://github.com/matrix-org/synapse/issues/8690)) +- Improve start time by adding an index to `e2e_cross_signing_keys.stream_id`. ([\#8694](https://github.com/matrix-org/synapse/issues/8694)) +- Re-organize the structured logging code to separate the TCP transport handling from the JSON formatting. ([\#8697](https://github.com/matrix-org/synapse/issues/8697)) +- Use Python 3.8 in Docker images by default. ([\#8698](https://github.com/matrix-org/synapse/issues/8698)) +- Remove the "draft" status of the Room Details Admin API. ([\#8702](https://github.com/matrix-org/synapse/issues/8702)) +- Improve the error returned when a non-string displayname or avatar_url is used when updating a user's profile. ([\#8705](https://github.com/matrix-org/synapse/issues/8705)) +- Block attempts by clients to send server ACLs, or redactions of server ACLs, that would result in the local server being blocked from the room. ([\#8708](https://github.com/matrix-org/synapse/issues/8708)) +- Add metrics the allow the local sysadmin to track 3PID `/requestToken` requests. ([\#8712](https://github.com/matrix-org/synapse/issues/8712)) +- Consolidate duplicated lists of purged tables that are checked in tests. ([\#8713](https://github.com/matrix-org/synapse/issues/8713)) +- Add some `mdui:UIInfo` element examples for `saml2_config` in the homeserver config. ([\#8718](https://github.com/matrix-org/synapse/issues/8718)) +- Improve the error message returned when a remote server incorrectly sets the `Content-Type` header in response to a JSON request. ([\#8719](https://github.com/matrix-org/synapse/issues/8719)) +- Speed up repeated state resolutions on the same room by caching event ID to auth event ID lookups. ([\#8752](https://github.com/matrix-org/synapse/issues/8752)) + + +Synapse 1.22.1 (2020-10-30) +=========================== + +Bugfixes +-------- + +- Fix a bug where an appservice may not be forwarded events for a room it was recently invited to. Broke in v1.22.0. ([\#8676](https://github.com/matrix-org/synapse/issues/8676)) +- Fix `Object of type frozendict is not JSON serializable` exceptions when using third-party event rules. Broke in v1.22.0. ([\#8678](https://github.com/matrix-org/synapse/issues/8678)) + + +Synapse 1.22.0 (2020-10-27) +=========================== + +No significant changes. + + +Synapse 1.22.0rc2 (2020-10-26) +============================== + +Bugfixes +-------- + +- Fix bugs where ephemeral events were not sent to appservices. Broke in v1.22.0rc1. ([\#8648](https://github.com/matrix-org/synapse/issues/8648), [\#8656](https://github.com/matrix-org/synapse/issues/8656)) +- Fix `user_daily_visits` table to not have duplicate rows per user/device due to multiple user agents. Broke in v1.22.0rc1. ([\#8654](https://github.com/matrix-org/synapse/issues/8654)) + +Synapse 1.22.0rc1 (2020-10-22) +============================== + +Features +-------- + +- Add a configuration option for always using the "userinfo endpoint" for OpenID Connect. This fixes support for some identity providers, e.g. GitLab. Contributed by Benjamin Koch. ([\#7658](https://github.com/matrix-org/synapse/issues/7658)) +- Add ability for `ThirdPartyEventRules` modules to query and manipulate whether a room is in the public rooms directory. ([\#8292](https://github.com/matrix-org/synapse/issues/8292), [\#8467](https://github.com/matrix-org/synapse/issues/8467)) +- Add support for olm fallback keys ([MSC2732](https://github.com/matrix-org/matrix-doc/pull/2732)). ([\#8312](https://github.com/matrix-org/synapse/issues/8312), [\#8501](https://github.com/matrix-org/synapse/issues/8501)) +- Add support for running background tasks in a separate worker process. ([\#8369](https://github.com/matrix-org/synapse/issues/8369), [\#8458](https://github.com/matrix-org/synapse/issues/8458), [\#8489](https://github.com/matrix-org/synapse/issues/8489), [\#8513](https://github.com/matrix-org/synapse/issues/8513), [\#8544](https://github.com/matrix-org/synapse/issues/8544), [\#8599](https://github.com/matrix-org/synapse/issues/8599)) +- Add support for device dehydration ([MSC2697](https://github.com/matrix-org/matrix-doc/pull/2697)). ([\#8380](https://github.com/matrix-org/synapse/issues/8380)) +- Add support for [MSC2409](https://github.com/matrix-org/matrix-doc/pull/2409), which allows sending typing, read receipts, and presence events to appservices. ([\#8437](https://github.com/matrix-org/synapse/issues/8437), [\#8590](https://github.com/matrix-org/synapse/issues/8590)) +- Change default room version to "6", per [MSC2788](https://github.com/matrix-org/matrix-doc/pull/2788). ([\#8461](https://github.com/matrix-org/synapse/issues/8461)) +- Add the ability to send non-membership events into a room via the `ModuleApi`. ([\#8479](https://github.com/matrix-org/synapse/issues/8479)) +- Increase default upload size limit from 10M to 50M. Contributed by @Akkowicz. ([\#8502](https://github.com/matrix-org/synapse/issues/8502)) +- Add support for modifying event content in `ThirdPartyRules` modules. ([\#8535](https://github.com/matrix-org/synapse/issues/8535), [\#8564](https://github.com/matrix-org/synapse/issues/8564)) + + +Bugfixes +-------- + +- Fix a longstanding bug where invalid ignored users in account data could break clients. ([\#8454](https://github.com/matrix-org/synapse/issues/8454)) +- Fix a bug where backfilling a room with an event that was missing the `redacts` field would break. ([\#8457](https://github.com/matrix-org/synapse/issues/8457)) +- Don't attempt to respond to some requests if the client has already disconnected. ([\#8465](https://github.com/matrix-org/synapse/issues/8465)) +- Fix message duplication if something goes wrong after persisting the event. ([\#8476](https://github.com/matrix-org/synapse/issues/8476)) +- Fix incremental sync returning an incorrect `prev_batch` token in timeline section, which when used to paginate returned events that were included in the incremental sync. Broken since v0.16.0. ([\#8486](https://github.com/matrix-org/synapse/issues/8486)) +- Expose the `uk.half-shot.msc2778.login.application_service` to clients from the login API. This feature was added in v1.21.0, but was not exposed as a potential login flow. ([\#8504](https://github.com/matrix-org/synapse/issues/8504)) +- Fix error code for `/profile/{userId}/displayname` to be `M_BAD_JSON`. ([\#8517](https://github.com/matrix-org/synapse/issues/8517)) +- Fix a bug introduced in v1.7.0 that could cause Synapse to insert values from non-state `m.room.retention` events into the `room_retention` database table. ([\#8527](https://github.com/matrix-org/synapse/issues/8527)) +- Fix not sending events over federation when using sharded event writers. ([\#8536](https://github.com/matrix-org/synapse/issues/8536)) +- Fix a long standing bug where email notifications for encrypted messages were blank. ([\#8545](https://github.com/matrix-org/synapse/issues/8545)) +- Fix increase in the number of `There was no active span...` errors logged when using OpenTracing. ([\#8567](https://github.com/matrix-org/synapse/issues/8567)) +- Fix a bug that prevented errors encountered during execution of the `synapse_port_db` from being correctly printed. ([\#8585](https://github.com/matrix-org/synapse/issues/8585)) +- Fix appservice transactions to only include a maximum of 100 persistent and 100 ephemeral events. ([\#8606](https://github.com/matrix-org/synapse/issues/8606)) + + +Updates to the Docker image +--------------------------- + +- Added multi-arch support (arm64,arm/v7) for the docker images. Contributed by @maquis196. ([\#7921](https://github.com/matrix-org/synapse/issues/7921)) +- Add support for passing commandline args to the synapse process. Contributed by @samuel-p. ([\#8390](https://github.com/matrix-org/synapse/issues/8390)) + + +Improved Documentation +---------------------- + +- Update the directions for using the manhole with coroutines. ([\#8462](https://github.com/matrix-org/synapse/issues/8462)) +- Improve readme by adding new shield.io badges. ([\#8493](https://github.com/matrix-org/synapse/issues/8493)) +- Added note about docker in manhole.md regarding which ip address to bind to. Contributed by @Maquis196. ([\#8526](https://github.com/matrix-org/synapse/issues/8526)) +- Document the new behaviour of the `allowed_lifetime_min` and `allowed_lifetime_max` settings in the room retention configuration. ([\#8529](https://github.com/matrix-org/synapse/issues/8529)) + + +Deprecations and Removals +------------------------- + +- Drop unused `device_max_stream_id` table. ([\#8589](https://github.com/matrix-org/synapse/issues/8589)) + + +Internal Changes +---------------- + +- Check for unreachable code with mypy. ([\#8432](https://github.com/matrix-org/synapse/issues/8432)) +- Add unit test for event persister sharding. ([\#8433](https://github.com/matrix-org/synapse/issues/8433)) +- Allow events to be sent to clients sooner when using sharded event persisters. ([\#8439](https://github.com/matrix-org/synapse/issues/8439), [\#8488](https://github.com/matrix-org/synapse/issues/8488), [\#8496](https://github.com/matrix-org/synapse/issues/8496), [\#8499](https://github.com/matrix-org/synapse/issues/8499)) +- Configure `public_baseurl` when using demo scripts. ([\#8443](https://github.com/matrix-org/synapse/issues/8443)) +- Add SQL logging on queries that happen during startup. ([\#8448](https://github.com/matrix-org/synapse/issues/8448)) +- Speed up unit tests when using PostgreSQL. ([\#8450](https://github.com/matrix-org/synapse/issues/8450)) +- Remove redundant database loads of stream_ordering for events we already have. ([\#8452](https://github.com/matrix-org/synapse/issues/8452)) +- Reduce inconsistencies between codepaths for membership and non-membership events. ([\#8463](https://github.com/matrix-org/synapse/issues/8463)) +- Combine `SpamCheckerApi` with the more generic `ModuleApi`. ([\#8464](https://github.com/matrix-org/synapse/issues/8464)) +- Additional testing for `ThirdPartyEventRules`. ([\#8468](https://github.com/matrix-org/synapse/issues/8468)) +- Add `-d` option to `./scripts-dev/lint.sh` to lint files that have changed since the last git commit. ([\#8472](https://github.com/matrix-org/synapse/issues/8472)) +- Unblacklist some sytests. ([\#8474](https://github.com/matrix-org/synapse/issues/8474)) +- Include the log level in the phone home stats. ([\#8477](https://github.com/matrix-org/synapse/issues/8477)) +- Remove outdated sphinx documentation, scripts and configuration. ([\#8480](https://github.com/matrix-org/synapse/issues/8480)) +- Clarify error message when plugin config parsers raise an error. ([\#8492](https://github.com/matrix-org/synapse/issues/8492)) +- Remove the deprecated `Handlers` object. ([\#8494](https://github.com/matrix-org/synapse/issues/8494)) +- Fix a threadsafety bug in unit tests. ([\#8497](https://github.com/matrix-org/synapse/issues/8497)) +- Add user agent to user_daily_visits table. ([\#8503](https://github.com/matrix-org/synapse/issues/8503)) +- Add type hints to various parts of the code base. ([\#8407](https://github.com/matrix-org/synapse/issues/8407), [\#8505](https://github.com/matrix-org/synapse/issues/8505), [\#8507](https://github.com/matrix-org/synapse/issues/8507), [\#8547](https://github.com/matrix-org/synapse/issues/8547), [\#8562](https://github.com/matrix-org/synapse/issues/8562), [\#8609](https://github.com/matrix-org/synapse/issues/8609)) +- Remove unused code from the test framework. ([\#8514](https://github.com/matrix-org/synapse/issues/8514)) +- Apply some internal fixes to the `HomeServer` class to make its code more idiomatic and statically-verifiable. ([\#8515](https://github.com/matrix-org/synapse/issues/8515)) +- Factor out common code between `RoomMemberHandler._locally_reject_invite` and `EventCreationHandler.create_event`. ([\#8537](https://github.com/matrix-org/synapse/issues/8537)) +- Improve database performance by executing more queries without starting transactions. ([\#8542](https://github.com/matrix-org/synapse/issues/8542)) +- Rename `Cache` to `DeferredCache`, to better reflect its purpose. ([\#8548](https://github.com/matrix-org/synapse/issues/8548)) +- Move metric registration code down into `LruCache`. ([\#8561](https://github.com/matrix-org/synapse/issues/8561), [\#8591](https://github.com/matrix-org/synapse/issues/8591)) +- Replace `DeferredCache` with the lighter-weight `LruCache` where possible. ([\#8563](https://github.com/matrix-org/synapse/issues/8563)) +- Add virtualenv-generated folders to `.gitignore`. ([\#8566](https://github.com/matrix-org/synapse/issues/8566)) +- Add `get_immediate` method to `DeferredCache`. ([\#8568](https://github.com/matrix-org/synapse/issues/8568)) +- Fix mypy not properly checking across the codebase, additionally, fix a typing assertion error in `handlers/auth.py`. ([\#8569](https://github.com/matrix-org/synapse/issues/8569)) +- Fix `synmark` benchmark runner. ([\#8571](https://github.com/matrix-org/synapse/issues/8571)) +- Modify `DeferredCache.get()` to return `Deferred`s instead of `ObservableDeferred`s. ([\#8572](https://github.com/matrix-org/synapse/issues/8572)) +- Adjust a protocol-type definition to fit `sqlite3` assertions. ([\#8577](https://github.com/matrix-org/synapse/issues/8577)) +- Support macOS on the `synmark` benchmark runner. ([\#8578](https://github.com/matrix-org/synapse/issues/8578)) +- Update `mypy` static type checker to 0.790. ([\#8583](https://github.com/matrix-org/synapse/issues/8583), [\#8600](https://github.com/matrix-org/synapse/issues/8600)) +- Re-organize the structured logging code to separate the TCP transport handling from the JSON formatting. ([\#8587](https://github.com/matrix-org/synapse/issues/8587)) +- Remove extraneous unittest logging decorators from unit tests. ([\#8592](https://github.com/matrix-org/synapse/issues/8592)) +- Minor optimisations in caching code. ([\#8593](https://github.com/matrix-org/synapse/issues/8593), [\#8594](https://github.com/matrix-org/synapse/issues/8594)) + + +Synapse 1.21.2 (2020-10-15) +=========================== + +Debian packages and Docker images have been rebuilt using the latest versions of dependency libraries, including authlib 0.15.1. Please see bugfixes below. + +Security advisory +----------------- + +* HTML pages served via Synapse were vulnerable to cross-site scripting (XSS) + attacks. All server administrators are encouraged to upgrade. + ([\#8444](https://github.com/matrix-org/synapse/pull/8444)) + ([CVE-2020-26891](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-26891)) + + This fix was originally included in v1.21.0 but was missing a security advisory. + + This was reported by [Denis Kasak](https://github.com/dkasak). + +Bugfixes +-------- + +- Fix rare bug where sending an event would fail due to a racey assertion. ([\#8530](https://github.com/matrix-org/synapse/issues/8530)) +- An updated version of the authlib dependency is included in the Docker and Debian images to fix an issue using OpenID Connect. See [\#8534](https://github.com/matrix-org/synapse/issues/8534) for details. + + +Synapse 1.21.1 (2020-10-13) +=========================== + +This release fixes a regression in v1.21.0 that prevented debian packages from being built. +It is otherwise identical to v1.21.0. + +Synapse 1.21.0 (2020-10-12) +=========================== + +No significant changes since v1.21.0rc3. + +As [noted in +v1.20.0](https://github.com/matrix-org/synapse/blob/release-v1.21.0/CHANGES.md#synapse-1200-2020-09-22), +a future release will drop support for accessing Synapse's +[Admin API](https://github.com/matrix-org/synapse/tree/master/docs/admin_api) under the +`/_matrix/client/*` endpoint prefixes. At that point, the Admin API will only +be accessible under `/_synapse/admin`. + + +Synapse 1.21.0rc3 (2020-10-08) +============================== + +Bugfixes +-------- + +- Fix duplication of events on high traffic servers, caused by PostgreSQL `could not serialize access due to concurrent update` errors. ([\#8456](https://github.com/matrix-org/synapse/issues/8456)) + + +Internal Changes +---------------- + +- Add Groovy Gorilla to the list of distributions we build `.deb`s for. ([\#8475](https://github.com/matrix-org/synapse/issues/8475)) + + +Synapse 1.21.0rc2 (2020-10-02) +============================== + +Features +-------- + +- Convert additional templates from inline HTML to Jinja2 templates. ([\#8444](https://github.com/matrix-org/synapse/issues/8444)) + +Bugfixes +-------- + +- Fix a regression in v1.21.0rc1 which broke thumbnails of remote media. ([\#8438](https://github.com/matrix-org/synapse/issues/8438)) +- Do not expose the experimental `uk.half-shot.msc2778.login.application_service` flow in the login API, which caused a compatibility problem with Element iOS. ([\#8440](https://github.com/matrix-org/synapse/issues/8440)) +- Fix malformed log line in new federation "catch up" logic. ([\#8442](https://github.com/matrix-org/synapse/issues/8442)) +- Fix DB query on startup for negative streams which caused long start up times. Introduced in [\#8374](https://github.com/matrix-org/synapse/issues/8374). ([\#8447](https://github.com/matrix-org/synapse/issues/8447)) + + +Synapse 1.21.0rc1 (2020-10-01) +============================== + +Features +-------- + +- Require the user to confirm that their password should be reset after clicking the email confirmation link. ([\#8004](https://github.com/matrix-org/synapse/issues/8004)) +- Add an admin API `GET /_synapse/admin/v1/event_reports` to read entries of table `event_reports`. Contributed by @dklimpel. ([\#8217](https://github.com/matrix-org/synapse/issues/8217)) +- Consolidate the SSO error template across all configuration. ([\#8248](https://github.com/matrix-org/synapse/issues/8248), [\#8405](https://github.com/matrix-org/synapse/issues/8405)) +- Add a configuration option to specify a whitelist of domains that a user can be redirected to after validating their email or phone number. ([\#8275](https://github.com/matrix-org/synapse/issues/8275), [\#8417](https://github.com/matrix-org/synapse/issues/8417)) +- Add experimental support for sharding event persister. ([\#8294](https://github.com/matrix-org/synapse/issues/8294), [\#8387](https://github.com/matrix-org/synapse/issues/8387), [\#8396](https://github.com/matrix-org/synapse/issues/8396), [\#8419](https://github.com/matrix-org/synapse/issues/8419)) +- Add the room topic and avatar to the room details admin API. ([\#8305](https://github.com/matrix-org/synapse/issues/8305)) +- Add an admin API for querying rooms where a user is a member. Contributed by @dklimpel. ([\#8306](https://github.com/matrix-org/synapse/issues/8306)) +- Add `uk.half-shot.msc2778.login.application_service` login type to allow appservices to login. ([\#8320](https://github.com/matrix-org/synapse/issues/8320)) +- Add a configuration option that allows existing users to log in with OpenID Connect. Contributed by @BBBSnowball and @OmmyZhang. ([\#8345](https://github.com/matrix-org/synapse/issues/8345)) +- Add prometheus metrics for replication requests. ([\#8406](https://github.com/matrix-org/synapse/issues/8406)) +- Support passing additional single sign-on parameters to the client. ([\#8413](https://github.com/matrix-org/synapse/issues/8413)) +- Add experimental reporting of metrics on expensive rooms for state-resolution. ([\#8420](https://github.com/matrix-org/synapse/issues/8420)) +- Add experimental prometheus metric to track numbers of "large" rooms for state resolutiom. ([\#8425](https://github.com/matrix-org/synapse/issues/8425)) +- Add prometheus metrics to track federation delays. ([\#8430](https://github.com/matrix-org/synapse/issues/8430)) + + +Bugfixes +-------- + +- Fix a bug in the media repository where remote thumbnails with the same size but different crop methods would overwrite each other. Contributed by @deepbluev7. ([\#7124](https://github.com/matrix-org/synapse/issues/7124)) +- Fix inconsistent handling of non-existent push rules, and stop tracking the `enabled` state of removed push rules. ([\#7796](https://github.com/matrix-org/synapse/issues/7796)) +- Fix a longstanding bug when storing a media file with an empty `upload_name`. ([\#7905](https://github.com/matrix-org/synapse/issues/7905)) +- Fix messages not being sent over federation until an event is sent into the same room. ([\#8230](https://github.com/matrix-org/synapse/issues/8230), [\#8247](https://github.com/matrix-org/synapse/issues/8247), [\#8258](https://github.com/matrix-org/synapse/issues/8258), [\#8272](https://github.com/matrix-org/synapse/issues/8272), [\#8322](https://github.com/matrix-org/synapse/issues/8322)) +- Fix a longstanding bug where files that could not be thumbnailed would result in an Internal Server Error. ([\#8236](https://github.com/matrix-org/synapse/issues/8236), [\#8435](https://github.com/matrix-org/synapse/issues/8435)) +- Upgrade minimum version of `canonicaljson` to version 1.4.0, to fix an unicode encoding issue. ([\#8262](https://github.com/matrix-org/synapse/issues/8262)) +- Fix longstanding bug which could lead to incomplete database upgrades on SQLite. ([\#8265](https://github.com/matrix-org/synapse/issues/8265)) +- Fix stack overflow when stderr is redirected to the logging system, and the logging system encounters an error. ([\#8268](https://github.com/matrix-org/synapse/issues/8268)) +- Fix a bug which cause the logging system to report errors, if `DEBUG` was enabled and no `context` filter was applied. ([\#8278](https://github.com/matrix-org/synapse/issues/8278)) +- Fix edge case where push could get delayed for a user until a later event was pushed. ([\#8287](https://github.com/matrix-org/synapse/issues/8287)) +- Fix fetching malformed events from remote servers. ([\#8324](https://github.com/matrix-org/synapse/issues/8324)) +- Fix `UnboundLocalError` from occuring when appservices send a malformed register request. ([\#8329](https://github.com/matrix-org/synapse/issues/8329)) +- Don't send push notifications to expired user accounts. ([\#8353](https://github.com/matrix-org/synapse/issues/8353)) +- Fix a regression in v1.19.0 with reactivating users through the admin API. ([\#8362](https://github.com/matrix-org/synapse/issues/8362)) +- Fix a bug where during device registration the length of the device name wasn't limited. ([\#8364](https://github.com/matrix-org/synapse/issues/8364)) +- Include `guest_access` in the fields that are checked for null bytes when updating `room_stats_state`. Broke in v1.7.2. ([\#8373](https://github.com/matrix-org/synapse/issues/8373)) +- Fix theoretical race condition where events are not sent down `/sync` if the synchrotron worker is restarted without restarting other workers. ([\#8374](https://github.com/matrix-org/synapse/issues/8374)) +- Fix a bug which could cause errors in rooms with malformed membership events, on servers using sqlite. ([\#8385](https://github.com/matrix-org/synapse/issues/8385)) +- Fix "Re-starting finished log context" warning when receiving an event we already had over federation. ([\#8398](https://github.com/matrix-org/synapse/issues/8398)) +- Fix incorrect handling of timeouts on outgoing HTTP requests. ([\#8400](https://github.com/matrix-org/synapse/issues/8400)) +- Fix a regression in v1.20.0 in the `synapse_port_db` script regarding the `ui_auth_sessions_ips` table. ([\#8410](https://github.com/matrix-org/synapse/issues/8410)) +- Remove unnecessary 3PID registration check when resetting password via an email address. Bug introduced in v0.34.0rc2. ([\#8414](https://github.com/matrix-org/synapse/issues/8414)) + + +Improved Documentation +---------------------- + +- Add `/_synapse/client` to the reverse proxy documentation. ([\#8227](https://github.com/matrix-org/synapse/issues/8227)) +- Add note to the reverse proxy settings documentation about disabling Apache's mod_security2. Contributed by Julian Fietkau (@jfietkau). ([\#8375](https://github.com/matrix-org/synapse/issues/8375)) +- Improve description of `server_name` config option in `homserver.yaml`. ([\#8415](https://github.com/matrix-org/synapse/issues/8415)) + + +Deprecations and Removals +------------------------- + +- Drop support for `prometheus_client` older than 0.4.0. ([\#8426](https://github.com/matrix-org/synapse/issues/8426)) + + +Internal Changes +---------------- + +- Fix tests on distros which disable TLSv1.0. Contributed by @danc86. ([\#8208](https://github.com/matrix-org/synapse/issues/8208)) +- Simplify the distributor code to avoid unnecessary work. ([\#8216](https://github.com/matrix-org/synapse/issues/8216)) +- Remove the `populate_stats_process_rooms_2` background job and restore functionality to `populate_stats_process_rooms`. ([\#8243](https://github.com/matrix-org/synapse/issues/8243)) +- Clean up type hints for `PaginationConfig`. ([\#8250](https://github.com/matrix-org/synapse/issues/8250), [\#8282](https://github.com/matrix-org/synapse/issues/8282)) +- Track the latest event for every destination and room for catch-up after federation outage. ([\#8256](https://github.com/matrix-org/synapse/issues/8256)) +- Fix non-user visible bug in implementation of `MultiWriterIdGenerator.get_current_token_for_writer`. ([\#8257](https://github.com/matrix-org/synapse/issues/8257)) +- Switch to the JSON implementation from the standard library. ([\#8259](https://github.com/matrix-org/synapse/issues/8259)) +- Add type hints to `synapse.util.async_helpers`. ([\#8260](https://github.com/matrix-org/synapse/issues/8260)) +- Simplify tests that mock asynchronous functions. ([\#8261](https://github.com/matrix-org/synapse/issues/8261)) +- Add type hints to `StreamToken` and `RoomStreamToken` classes. ([\#8279](https://github.com/matrix-org/synapse/issues/8279)) +- Change `StreamToken.room_key` to be a `RoomStreamToken` instance. ([\#8281](https://github.com/matrix-org/synapse/issues/8281)) +- Refactor notifier code to correctly use the max event stream position. ([\#8288](https://github.com/matrix-org/synapse/issues/8288)) +- Use slotted classes where possible. ([\#8296](https://github.com/matrix-org/synapse/issues/8296)) +- Support testing the local Synapse checkout against the [Complement homeserver test suite](https://github.com/matrix-org/complement/). ([\#8317](https://github.com/matrix-org/synapse/issues/8317)) +- Update outdated usages of `metaclass` to python 3 syntax. ([\#8326](https://github.com/matrix-org/synapse/issues/8326)) +- Move lint-related dependencies to package-extra field, update CONTRIBUTING.md to utilise this. ([\#8330](https://github.com/matrix-org/synapse/issues/8330), [\#8377](https://github.com/matrix-org/synapse/issues/8377)) +- Use the `admin_patterns` helper in additional locations. ([\#8331](https://github.com/matrix-org/synapse/issues/8331)) +- Fix test logging to allow braces in log output. ([\#8335](https://github.com/matrix-org/synapse/issues/8335)) +- Remove `__future__` imports related to Python 2 compatibility. ([\#8337](https://github.com/matrix-org/synapse/issues/8337)) +- Simplify `super()` calls to Python 3 syntax. ([\#8344](https://github.com/matrix-org/synapse/issues/8344)) +- Fix bad merge from `release-v1.20.0` branch to `develop`. ([\#8354](https://github.com/matrix-org/synapse/issues/8354)) +- Factor out a `_send_dummy_event_for_room` method. ([\#8370](https://github.com/matrix-org/synapse/issues/8370)) +- Improve logging of state resolution. ([\#8371](https://github.com/matrix-org/synapse/issues/8371)) +- Add type annotations to `SimpleHttpClient`. ([\#8372](https://github.com/matrix-org/synapse/issues/8372)) +- Refactor ID generators to use `async with` syntax. ([\#8383](https://github.com/matrix-org/synapse/issues/8383)) +- Add `EventStreamPosition` type. ([\#8388](https://github.com/matrix-org/synapse/issues/8388)) +- Create a mechanism for marking tests "logcontext clean". ([\#8399](https://github.com/matrix-org/synapse/issues/8399)) +- A pair of tiny cleanups in the federation request code. ([\#8401](https://github.com/matrix-org/synapse/issues/8401)) +- Add checks on startup that PostgreSQL sequences are consistent with their associated tables. ([\#8402](https://github.com/matrix-org/synapse/issues/8402)) +- Do not include appservice users when calculating the total MAU for a server. ([\#8404](https://github.com/matrix-org/synapse/issues/8404)) +- Typing fixes for `synapse.handlers.federation`. ([\#8422](https://github.com/matrix-org/synapse/issues/8422)) +- Various refactors to simplify stream token handling. ([\#8423](https://github.com/matrix-org/synapse/issues/8423)) +- Make stream token serializing/deserializing async. ([\#8427](https://github.com/matrix-org/synapse/issues/8427)) + + +Synapse 1.20.1 (2020-09-24) +=========================== + +Bugfixes +-------- + +- Fix a bug introduced in v1.20.0 which caused the `synapse_port_db` script to fail. ([\#8386](https://github.com/matrix-org/synapse/issues/8386)) +- Fix a bug introduced in v1.20.0 which caused variables to be incorrectly escaped in Jinja2 templates. ([\#8394](https://github.com/matrix-org/synapse/issues/8394)) + + +Synapse 1.20.0 (2020-09-22) +=========================== + +No significant changes since v1.20.0rc5. + +Removal warning +--------------- + +Historically, the [Synapse Admin +API](https://github.com/matrix-org/synapse/tree/master/docs) has been +accessible under the `/_matrix/client/api/v1/admin`, +`/_matrix/client/unstable/admin`, `/_matrix/client/r0/admin` and +`/_synapse/admin` prefixes. In a future release, we will be dropping support +for accessing Synapse's Admin API using the `/_matrix/client/*` prefixes. + +From that point, the Admin API will only be accessible under `/_synapse/admin`. +This makes it easier for homeserver admins to lock down external access to the +Admin API endpoints. + +Synapse 1.20.0rc5 (2020-09-18) +============================== + +In addition to the below, Synapse 1.20.0rc5 also includes the bug fix that was included in 1.19.3. + +Features +-------- + +- Add flags to the `/versions` endpoint for whether new rooms default to using E2EE. ([\#8343](https://github.com/matrix-org/synapse/issues/8343)) + + +Bugfixes +-------- + +- Fix rate limiting of federation `/send` requests. ([\#8342](https://github.com/matrix-org/synapse/issues/8342)) +- Fix a longstanding bug where back pagination over federation could get stuck if it failed to handle a received event. ([\#8349](https://github.com/matrix-org/synapse/issues/8349)) + + +Internal Changes +---------------- + +- Blacklist [MSC2753](https://github.com/matrix-org/matrix-doc/pull/2753) SyTests until it is implemented. ([\#8285](https://github.com/matrix-org/synapse/issues/8285)) + + +Synapse 1.19.3 (2020-09-18) +=========================== + +Bugfixes +-------- + +- Partially mitigate bug where newly joined servers couldn't get past events in a room when there is a malformed event. ([\#8350](https://github.com/matrix-org/synapse/issues/8350)) + + +Synapse 1.20.0rc4 (2020-09-16) +============================== + +Synapse 1.20.0rc4 is identical to 1.20.0rc3, with the addition of the security fix that was included in 1.19.2. + + +Synapse 1.19.2 (2020-09-16) +=========================== + +Due to the issue below server admins are encouraged to upgrade as soon as possible. + +Bugfixes +-------- + +- Fix joining rooms over federation that include malformed events. ([\#8324](https://github.com/matrix-org/synapse/issues/8324)) + + +Synapse 1.20.0rc3 (2020-09-11) +============================== + +Bugfixes +-------- + +- Fix a bug introduced in v1.20.0rc1 where the wrong exception was raised when invalid JSON data is encountered. ([\#8291](https://github.com/matrix-org/synapse/issues/8291)) + + +Synapse 1.20.0rc2 (2020-09-09) +============================== + +Bugfixes +-------- + +- Fix a bug introduced in v1.20.0rc1 causing some features related to notifications to misbehave following the implementation of unread counts. ([\#8280](https://github.com/matrix-org/synapse/issues/8280)) + + +Synapse 1.20.0rc1 (2020-09-08) +============================== + +Removal warning +--------------- + +Some older clients used a [disallowed character](https://matrix.org/docs/spec/client_server/r0.6.1#post-matrix-client-r0-register-email-requesttoken) (`:`) in the `client_secret` parameter of various endpoints. The incorrect behaviour was allowed for backwards compatibility, but is now being removed from Synapse as most users have updated their client. Further context can be found at [\#6766](https://github.com/matrix-org/synapse/issues/6766). + +Features +-------- + +- Add an endpoint to query your shared rooms with another user as an implementation of [MSC2666](https://github.com/matrix-org/matrix-doc/pull/2666). ([\#7785](https://github.com/matrix-org/synapse/issues/7785)) +- Iteratively encode JSON to avoid blocking the reactor. ([\#8013](https://github.com/matrix-org/synapse/issues/8013), [\#8116](https://github.com/matrix-org/synapse/issues/8116)) +- Add support for shadow-banning users (ignoring any message send requests). ([\#8034](https://github.com/matrix-org/synapse/issues/8034), [\#8092](https://github.com/matrix-org/synapse/issues/8092), [\#8095](https://github.com/matrix-org/synapse/issues/8095), [\#8142](https://github.com/matrix-org/synapse/issues/8142), [\#8152](https://github.com/matrix-org/synapse/issues/8152), [\#8157](https://github.com/matrix-org/synapse/issues/8157), [\#8158](https://github.com/matrix-org/synapse/issues/8158), [\#8176](https://github.com/matrix-org/synapse/issues/8176)) +- Use the default template file when its equivalent is not found in a custom template directory. ([\#8037](https://github.com/matrix-org/synapse/issues/8037), [\#8107](https://github.com/matrix-org/synapse/issues/8107), [\#8252](https://github.com/matrix-org/synapse/issues/8252)) +- Add unread messages count to sync responses, as specified in [MSC2654](https://github.com/matrix-org/matrix-doc/pull/2654). ([\#8059](https://github.com/matrix-org/synapse/issues/8059), [\#8254](https://github.com/matrix-org/synapse/issues/8254), [\#8270](https://github.com/matrix-org/synapse/issues/8270), [\#8274](https://github.com/matrix-org/synapse/issues/8274)) +- Optimise `/federation/v1/user/devices/` API by only returning devices with encryption keys. ([\#8198](https://github.com/matrix-org/synapse/issues/8198)) + + +Bugfixes +-------- + +- Fix a memory leak by limiting the length of time that messages will be queued for a remote server that has been unreachable. ([\#7864](https://github.com/matrix-org/synapse/issues/7864)) +- Fix `Re-starting finished log context PUT-nnnn` warning when event persistence failed. ([\#8081](https://github.com/matrix-org/synapse/issues/8081)) +- Synapse now correctly enforces the valid characters in the `client_secret` parameter used in various endpoints. ([\#8101](https://github.com/matrix-org/synapse/issues/8101)) +- Fix a bug introduced in v1.7.2 impacting message retention policies that would allow federated homeservers to dictate a retention period that's lower than the configured minimum allowed duration in the configuration file. ([\#8104](https://github.com/matrix-org/synapse/issues/8104)) +- Fix a long-standing bug where invalid JSON would be accepted by Synapse. ([\#8106](https://github.com/matrix-org/synapse/issues/8106)) +- Fix a bug introduced in Synapse v1.12.0 which could cause `/sync` requests to fail with a 404 if you had a very old outstanding room invite. ([\#8110](https://github.com/matrix-org/synapse/issues/8110)) +- Return a proper error code when the rooms of an invalid group are requested. ([\#8129](https://github.com/matrix-org/synapse/issues/8129)) +- Fix a bug which could cause a leaked postgres connection if synapse was set to daemonize. ([\#8131](https://github.com/matrix-org/synapse/issues/8131)) +- Clarify the error code if a user tries to register with a numeric ID. This bug was introduced in v1.15.0. ([\#8135](https://github.com/matrix-org/synapse/issues/8135)) +- Fix a bug where appservices with ratelimiting disabled would still be ratelimited when joining rooms. This bug was introduced in v1.19.0. ([\#8139](https://github.com/matrix-org/synapse/issues/8139)) +- Fix logging in via OpenID Connect with a provider that uses integer user IDs. ([\#8190](https://github.com/matrix-org/synapse/issues/8190)) +- Fix a longstanding bug where user directory updates could break when unexpected profile data was included in events. ([\#8223](https://github.com/matrix-org/synapse/issues/8223)) +- Fix a longstanding bug where stats updates could break when unexpected profile data was included in events. ([\#8226](https://github.com/matrix-org/synapse/issues/8226)) +- Fix slow start times for large servers by removing a table scan of the `users` table from startup code. ([\#8271](https://github.com/matrix-org/synapse/issues/8271)) + + +Updates to the Docker image +--------------------------- + +- Fix builds of the Docker image on non-x86 platforms. ([\#8144](https://github.com/matrix-org/synapse/issues/8144)) +- Added curl for healthcheck support and readme updates for the change. Contributed by @maquis196. ([\#8147](https://github.com/matrix-org/synapse/issues/8147)) + + +Improved Documentation +---------------------- + +- Link to matrix-synapse-rest-password-provider in the password provider documentation. ([\#8111](https://github.com/matrix-org/synapse/issues/8111)) +- Updated documentation to note that Synapse does not follow `HTTP 308` redirects due to an upstream library not supporting them. Contributed by Ryan Cole. ([\#8120](https://github.com/matrix-org/synapse/issues/8120)) +- Explain better what GDPR-erased means when deactivating a user. ([\#8189](https://github.com/matrix-org/synapse/issues/8189)) + + +Internal Changes +---------------- + +- Add filter `name` to the `/users` admin API, which filters by user ID or displayname. Contributed by Awesome Technologies Innovationslabor GmbH. ([\#7377](https://github.com/matrix-org/synapse/issues/7377), [\#8163](https://github.com/matrix-org/synapse/issues/8163)) +- Reduce run times of some unit tests by advancing the reactor a fewer number of times. ([\#7757](https://github.com/matrix-org/synapse/issues/7757)) +- Don't fail `/submit_token` requests on incorrect session ID if `request_token_inhibit_3pid_errors` is turned on. ([\#7991](https://github.com/matrix-org/synapse/issues/7991)) +- Convert various parts of the codebase to async/await. ([\#8071](https://github.com/matrix-org/synapse/issues/8071), [\#8072](https://github.com/matrix-org/synapse/issues/8072), [\#8074](https://github.com/matrix-org/synapse/issues/8074), [\#8075](https://github.com/matrix-org/synapse/issues/8075), [\#8076](https://github.com/matrix-org/synapse/issues/8076), [\#8087](https://github.com/matrix-org/synapse/issues/8087), [\#8100](https://github.com/matrix-org/synapse/issues/8100), [\#8119](https://github.com/matrix-org/synapse/issues/8119), [\#8121](https://github.com/matrix-org/synapse/issues/8121), [\#8133](https://github.com/matrix-org/synapse/issues/8133), [\#8156](https://github.com/matrix-org/synapse/issues/8156), [\#8162](https://github.com/matrix-org/synapse/issues/8162), [\#8166](https://github.com/matrix-org/synapse/issues/8166), [\#8168](https://github.com/matrix-org/synapse/issues/8168), [\#8173](https://github.com/matrix-org/synapse/issues/8173), [\#8191](https://github.com/matrix-org/synapse/issues/8191), [\#8192](https://github.com/matrix-org/synapse/issues/8192), [\#8193](https://github.com/matrix-org/synapse/issues/8193), [\#8194](https://github.com/matrix-org/synapse/issues/8194), [\#8195](https://github.com/matrix-org/synapse/issues/8195), [\#8197](https://github.com/matrix-org/synapse/issues/8197), [\#8199](https://github.com/matrix-org/synapse/issues/8199), [\#8200](https://github.com/matrix-org/synapse/issues/8200), [\#8201](https://github.com/matrix-org/synapse/issues/8201), [\#8202](https://github.com/matrix-org/synapse/issues/8202), [\#8207](https://github.com/matrix-org/synapse/issues/8207), [\#8213](https://github.com/matrix-org/synapse/issues/8213), [\#8214](https://github.com/matrix-org/synapse/issues/8214)) +- Remove some unused database functions. ([\#8085](https://github.com/matrix-org/synapse/issues/8085)) +- Add type hints to various parts of the codebase. ([\#8090](https://github.com/matrix-org/synapse/issues/8090), [\#8127](https://github.com/matrix-org/synapse/issues/8127), [\#8187](https://github.com/matrix-org/synapse/issues/8187), [\#8241](https://github.com/matrix-org/synapse/issues/8241), [\#8140](https://github.com/matrix-org/synapse/issues/8140), [\#8183](https://github.com/matrix-org/synapse/issues/8183), [\#8232](https://github.com/matrix-org/synapse/issues/8232), [\#8235](https://github.com/matrix-org/synapse/issues/8235), [\#8237](https://github.com/matrix-org/synapse/issues/8237), [\#8244](https://github.com/matrix-org/synapse/issues/8244)) +- Return the previous stream token if a non-member event is a duplicate. ([\#8093](https://github.com/matrix-org/synapse/issues/8093), [\#8112](https://github.com/matrix-org/synapse/issues/8112)) +- Separate `get_current_token` into two since there are two different use cases for it. ([\#8113](https://github.com/matrix-org/synapse/issues/8113)) +- Remove `ChainedIdGenerator`. ([\#8123](https://github.com/matrix-org/synapse/issues/8123)) +- Reduce the amount of whitespace in JSON stored and sent in responses. ([\#8124](https://github.com/matrix-org/synapse/issues/8124)) +- Update the test federation client to handle streaming responses. ([\#8130](https://github.com/matrix-org/synapse/issues/8130)) +- Micro-optimisations to `get_auth_chain_ids`. ([\#8132](https://github.com/matrix-org/synapse/issues/8132)) +- Refactor `StreamIdGenerator` and `MultiWriterIdGenerator` to have the same interface. ([\#8161](https://github.com/matrix-org/synapse/issues/8161)) +- Add functions to `MultiWriterIdGen` used by events stream. ([\#8164](https://github.com/matrix-org/synapse/issues/8164), [\#8179](https://github.com/matrix-org/synapse/issues/8179)) +- Fix tests that were broken due to the merge of 1.19.1. ([\#8167](https://github.com/matrix-org/synapse/issues/8167)) +- Make `SlavedIdTracker.advance` have the same interface as `MultiWriterIDGenerator`. ([\#8171](https://github.com/matrix-org/synapse/issues/8171)) +- Remove unused `is_guest` parameter from, and add safeguard to, `MessageHandler.get_room_data`. ([\#8174](https://github.com/matrix-org/synapse/issues/8174), [\#8181](https://github.com/matrix-org/synapse/issues/8181)) +- Standardize the mypy configuration. ([\#8175](https://github.com/matrix-org/synapse/issues/8175)) +- Refactor some of `LoginRestServlet`'s helper methods, and move them to `AuthHandler` for easier reuse. ([\#8182](https://github.com/matrix-org/synapse/issues/8182)) +- Fix `wait_for_stream_position` to allow multiple waiters on same stream ID. ([\#8196](https://github.com/matrix-org/synapse/issues/8196)) +- Make `MultiWriterIDGenerator` work for streams that use negative values. ([\#8203](https://github.com/matrix-org/synapse/issues/8203)) +- Refactor queries for device keys and cross-signatures. ([\#8204](https://github.com/matrix-org/synapse/issues/8204), [\#8205](https://github.com/matrix-org/synapse/issues/8205), [\#8222](https://github.com/matrix-org/synapse/issues/8222), [\#8224](https://github.com/matrix-org/synapse/issues/8224), [\#8225](https://github.com/matrix-org/synapse/issues/8225), [\#8231](https://github.com/matrix-org/synapse/issues/8231), [\#8233](https://github.com/matrix-org/synapse/issues/8233), [\#8234](https://github.com/matrix-org/synapse/issues/8234)) +- Fix type hints for functions decorated with `@cached`. ([\#8240](https://github.com/matrix-org/synapse/issues/8240)) +- Remove obsolete `order` field from federation send queues. ([\#8245](https://github.com/matrix-org/synapse/issues/8245)) +- Stop sub-classing from object. ([\#8249](https://github.com/matrix-org/synapse/issues/8249)) +- Add more logging to debug slow startup. ([\#8264](https://github.com/matrix-org/synapse/issues/8264)) +- Do not attempt to upgrade database schema on worker processes. ([\#8266](https://github.com/matrix-org/synapse/issues/8266), [\#8276](https://github.com/matrix-org/synapse/issues/8276)) + + +Synapse 1.19.1 (2020-08-27) +=========================== + +No significant changes. + + +Synapse 1.19.1rc1 (2020-08-25) +============================== + +Bugfixes +-------- + +- Fix a bug introduced in v1.19.0 where appservices with ratelimiting disabled would still be ratelimited when joining rooms. ([\#8139](https://github.com/matrix-org/synapse/issues/8139)) +- Fix a bug introduced in v1.19.0 that would cause e.g. profile updates to fail due to incorrect application of rate limits on join requests. ([\#8153](https://github.com/matrix-org/synapse/issues/8153)) + + +Synapse 1.19.0 (2020-08-17) +=========================== + +No significant changes since 1.19.0rc1. + +Removal warning +--------------- + +As outlined in the [previous release](https://github.com/matrix-org/synapse/releases/tag/v1.18.0), +we are no longer publishing Docker images with the `-py3` tag suffix. On top of that, we have also removed the +`latest-py3` tag. Please see +[the announcement in the upgrade notes for 1.18.0](https://github.com/matrix-org/synapse/blob/develop/docs/upgrade.md#upgrading-to-v1180). + + +Synapse 1.19.0rc1 (2020-08-13) +============================== + +Features +-------- + +- Add option to allow server admins to join rooms which fail complexity checks. Contributed by @lugino-emeritus. ([\#7902](https://github.com/matrix-org/synapse/issues/7902)) +- Add an option to purge room or not with delete room admin endpoint (`POST /_synapse/admin/v1/rooms//delete`). Contributed by @dklimpel. ([\#7964](https://github.com/matrix-org/synapse/issues/7964)) +- Add rate limiting to users joining rooms. ([\#8008](https://github.com/matrix-org/synapse/issues/8008)) +- Add a `/health` endpoint to every configured HTTP listener that can be used as a health check endpoint by load balancers. ([\#8048](https://github.com/matrix-org/synapse/issues/8048)) +- Allow login to be blocked based on the values of SAML attributes. ([\#8052](https://github.com/matrix-org/synapse/issues/8052)) +- Allow guest access to the `GET /_matrix/client/r0/rooms/{room_id}/members` endpoint, according to MSC2689. Contributed by Awesome Technologies Innovationslabor GmbH. ([\#7314](https://github.com/matrix-org/synapse/issues/7314)) + + +Bugfixes +-------- + +- Fix a bug introduced in Synapse v1.7.2 which caused inaccurate membership counts in the room directory. ([\#7977](https://github.com/matrix-org/synapse/issues/7977)) +- Fix a long standing bug: 'Duplicate key value violates unique constraint "event_relations_id"' when message retention is configured. ([\#7978](https://github.com/matrix-org/synapse/issues/7978)) +- Fix "no create event in auth events" when trying to reject invitation after inviter leaves. Bug introduced in Synapse v1.10.0. ([\#7980](https://github.com/matrix-org/synapse/issues/7980)) +- Fix various comments and minor discrepencies in server notices code. ([\#7996](https://github.com/matrix-org/synapse/issues/7996)) +- Fix a long standing bug where HTTP HEAD requests resulted in a 400 error. ([\#7999](https://github.com/matrix-org/synapse/issues/7999)) +- Fix a long-standing bug which caused two copies of some log lines to be written when synctl was used along with a MemoryHandler logger. ([\#8011](https://github.com/matrix-org/synapse/issues/8011), [\#8012](https://github.com/matrix-org/synapse/issues/8012)) + + +Updates to the Docker image +--------------------------- + +- We no longer publish Docker images with the `-py3` tag suffix, as [announced in the upgrade notes](https://github.com/matrix-org/synapse/blob/develop/docs/upgrade.md#upgrading-to-v1180). ([\#8056](https://github.com/matrix-org/synapse/issues/8056)) + + +Improved Documentation +---------------------- + +- Document how to set up a client .well-known file and fix several pieces of outdated documentation. ([\#7899](https://github.com/matrix-org/synapse/issues/7899)) +- Improve workers docs. ([\#7990](https://github.com/matrix-org/synapse/issues/7990), [\#8000](https://github.com/matrix-org/synapse/issues/8000)) +- Fix typo in `docs/workers.md`. ([\#7992](https://github.com/matrix-org/synapse/issues/7992)) +- Add documentation for how to undo a room shutdown. ([\#7998](https://github.com/matrix-org/synapse/issues/7998), [\#8010](https://github.com/matrix-org/synapse/issues/8010)) + + +Internal Changes +---------------- + +- Reduce the amount of whitespace in JSON stored and sent in responses. Contributed by David Vo. ([\#7372](https://github.com/matrix-org/synapse/issues/7372)) +- Switch to the JSON implementation from the standard library and bump the minimum version of the canonicaljson library to 1.2.0. ([\#7936](https://github.com/matrix-org/synapse/issues/7936), [\#7979](https://github.com/matrix-org/synapse/issues/7979)) +- Convert various parts of the codebase to async/await. ([\#7947](https://github.com/matrix-org/synapse/issues/7947), [\#7948](https://github.com/matrix-org/synapse/issues/7948), [\#7949](https://github.com/matrix-org/synapse/issues/7949), [\#7951](https://github.com/matrix-org/synapse/issues/7951), [\#7963](https://github.com/matrix-org/synapse/issues/7963), [\#7973](https://github.com/matrix-org/synapse/issues/7973), [\#7975](https://github.com/matrix-org/synapse/issues/7975), [\#7976](https://github.com/matrix-org/synapse/issues/7976), [\#7981](https://github.com/matrix-org/synapse/issues/7981), [\#7987](https://github.com/matrix-org/synapse/issues/7987), [\#7989](https://github.com/matrix-org/synapse/issues/7989), [\#8003](https://github.com/matrix-org/synapse/issues/8003), [\#8014](https://github.com/matrix-org/synapse/issues/8014), [\#8016](https://github.com/matrix-org/synapse/issues/8016), [\#8027](https://github.com/matrix-org/synapse/issues/8027), [\#8031](https://github.com/matrix-org/synapse/issues/8031), [\#8032](https://github.com/matrix-org/synapse/issues/8032), [\#8035](https://github.com/matrix-org/synapse/issues/8035), [\#8042](https://github.com/matrix-org/synapse/issues/8042), [\#8044](https://github.com/matrix-org/synapse/issues/8044), [\#8045](https://github.com/matrix-org/synapse/issues/8045), [\#8061](https://github.com/matrix-org/synapse/issues/8061), [\#8062](https://github.com/matrix-org/synapse/issues/8062), [\#8063](https://github.com/matrix-org/synapse/issues/8063), [\#8066](https://github.com/matrix-org/synapse/issues/8066), [\#8069](https://github.com/matrix-org/synapse/issues/8069), [\#8070](https://github.com/matrix-org/synapse/issues/8070)) +- Move some database-related log lines from the default logger to the database/transaction loggers. ([\#7952](https://github.com/matrix-org/synapse/issues/7952)) +- Add a script to detect source code files using non-unix line terminators. ([\#7965](https://github.com/matrix-org/synapse/issues/7965), [\#7970](https://github.com/matrix-org/synapse/issues/7970)) +- Log the SAML session ID during creation. ([\#7971](https://github.com/matrix-org/synapse/issues/7971)) +- Implement new experimental push rules for some users. ([\#7997](https://github.com/matrix-org/synapse/issues/7997)) +- Remove redundant and unreliable signature check for v1 Identity Service lookup responses. ([\#8001](https://github.com/matrix-org/synapse/issues/8001)) +- Improve the performance of the register endpoint. ([\#8009](https://github.com/matrix-org/synapse/issues/8009)) +- Reduce less useful output in the newsfragment CI step. Add a link to the changelog section of the contributing guide on error. ([\#8024](https://github.com/matrix-org/synapse/issues/8024)) +- Rename storage layer objects to be more sensible. ([\#8033](https://github.com/matrix-org/synapse/issues/8033)) +- Change the default log config to reduce disk I/O and storage for new servers. ([\#8040](https://github.com/matrix-org/synapse/issues/8040)) +- Add an assertion on `prev_events` in `create_new_client_event`. ([\#8041](https://github.com/matrix-org/synapse/issues/8041)) +- Add a comment to `ServerContextFactory` about the use of `SSLv23_METHOD`. ([\#8043](https://github.com/matrix-org/synapse/issues/8043)) +- Log `OPTIONS` requests at `DEBUG` rather than `INFO` level to reduce amount logged at `INFO`. ([\#8049](https://github.com/matrix-org/synapse/issues/8049)) +- Reduce amount of outbound request logging at `INFO` level. ([\#8050](https://github.com/matrix-org/synapse/issues/8050)) +- It is no longer necessary to explicitly define `filters` in the logging configuration. (Continuing to do so is redundant but harmless.) ([\#8051](https://github.com/matrix-org/synapse/issues/8051)) +- Add and improve type hints. ([\#8058](https://github.com/matrix-org/synapse/issues/8058), [\#8064](https://github.com/matrix-org/synapse/issues/8064), [\#8060](https://github.com/matrix-org/synapse/issues/8060), [\#8067](https://github.com/matrix-org/synapse/issues/8067)) + + +Synapse 1.18.0 (2020-07-30) +=========================== + +Deprecation Warnings +-------------------- + +### Docker Tags with `-py3` Suffix + +From 10th August 2020, we will no longer publish Docker images with the `-py3` tag suffix. The images tagged with the `-py3` suffix have been identical to the non-suffixed tags since release 0.99.0, and the suffix is obsolete. + +On 10th August, we will remove the `latest-py3` tag. Existing per-release tags (such as `v1.18.0-py3`) will not be removed, but no new `-py3` tags will be added. + +Scripts relying on the `-py3` suffix will need to be updated. + + +### TCP-based Replication + +When setting up worker processes, we now recommend the use of a Redis server for replication. The old direct TCP connection method is deprecated and will be removed in a future release. See [docs/workers.md](https://github.com/matrix-org/synapse/blob/release-v1.18.0/docs/workers.md) for more details. + + +Improved Documentation +---------------------- + +- Update worker docs with latest enhancements. ([\#7969](https://github.com/matrix-org/synapse/issues/7969)) + + +Synapse 1.18.0rc2 (2020-07-28) +============================== + +Bugfixes +-------- + +- Fix an `AssertionError` exception introduced in v1.18.0rc1. ([\#7876](https://github.com/matrix-org/synapse/issues/7876)) +- Fix experimental support for moving typing off master when worker is restarted, which is broken in v1.18.0rc1. ([\#7967](https://github.com/matrix-org/synapse/issues/7967)) + + +Internal Changes +---------------- + +- Further optimise queueing of inbound replication commands. ([\#7876](https://github.com/matrix-org/synapse/issues/7876)) + + +Synapse 1.18.0rc1 (2020-07-27) +============================== + +Features +-------- + +- Include room states on invite events that are sent to application services. Contributed by @Sorunome. ([\#6455](https://github.com/matrix-org/synapse/issues/6455)) +- Add delete room admin endpoint (`POST /_synapse/admin/v1/rooms//delete`). Contributed by @dklimpel. ([\#7613](https://github.com/matrix-org/synapse/issues/7613), [\#7953](https://github.com/matrix-org/synapse/issues/7953)) +- Add experimental support for running multiple federation sender processes. ([\#7798](https://github.com/matrix-org/synapse/issues/7798)) +- Add the option to validate the `iss` and `aud` claims for JWT logins. ([\#7827](https://github.com/matrix-org/synapse/issues/7827)) +- Add support for handling registration requests across multiple client reader workers. ([\#7830](https://github.com/matrix-org/synapse/issues/7830)) +- Add an admin API to list the users in a room. Contributed by Awesome Technologies Innovationslabor GmbH. ([\#7842](https://github.com/matrix-org/synapse/issues/7842)) +- Allow email subjects to be customised through Synapse's configuration. ([\#7846](https://github.com/matrix-org/synapse/issues/7846)) +- Add the ability to re-activate an account from the admin API. ([\#7847](https://github.com/matrix-org/synapse/issues/7847), [\#7908](https://github.com/matrix-org/synapse/issues/7908)) +- Add experimental support for running multiple pusher workers. ([\#7855](https://github.com/matrix-org/synapse/issues/7855)) +- Add experimental support for moving typing off master. ([\#7869](https://github.com/matrix-org/synapse/issues/7869), [\#7959](https://github.com/matrix-org/synapse/issues/7959)) +- Report CPU metrics to prometheus for time spent processing replication commands. ([\#7879](https://github.com/matrix-org/synapse/issues/7879)) +- Support oEmbed for media previews. ([\#7920](https://github.com/matrix-org/synapse/issues/7920)) +- Abort federation requests where the client disconnects before the ratelimiter expires. ([\#7930](https://github.com/matrix-org/synapse/issues/7930)) +- Cache responses to `/_matrix/federation/v1/state_ids` to reduce duplicated work. ([\#7931](https://github.com/matrix-org/synapse/issues/7931)) + + +Bugfixes +-------- + +- Fix detection of out of sync remote device lists when receiving events from remote users. ([\#7815](https://github.com/matrix-org/synapse/issues/7815)) +- Fix bug where Synapse fails to process an incoming event over federation if the server is missing too much of the event's auth chain. ([\#7817](https://github.com/matrix-org/synapse/issues/7817)) +- Fix a bug causing Synapse to misinterpret the value `off` for `encryption_enabled_by_default_for_room_type` in its configuration file(s) if that value isn't surrounded by quotes. This bug was introduced in v1.16.0. ([\#7822](https://github.com/matrix-org/synapse/issues/7822)) +- Fix bug where we did not always pass in `app_name` or `server_name` to email templates, including e.g. for registration emails. ([\#7829](https://github.com/matrix-org/synapse/issues/7829)) +- Errors which occur while using the non-standard JWT login now return the proper error: `403 Forbidden` with an error code of `M_FORBIDDEN`. ([\#7844](https://github.com/matrix-org/synapse/issues/7844)) +- Fix "AttributeError: 'str' object has no attribute 'get'" error message when applying per-room message retention policies. The bug was introduced in Synapse 1.7.0. ([\#7850](https://github.com/matrix-org/synapse/issues/7850)) +- Fix a bug introduced in Synapse 1.10.0 which could cause a "no create event in auth events" error during room creation. ([\#7854](https://github.com/matrix-org/synapse/issues/7854)) +- Fix a bug which allowed empty rooms to be rejoined over federation. ([\#7859](https://github.com/matrix-org/synapse/issues/7859)) +- Fix 'Unable to find a suitable guest user ID' error when using multiple client_reader workers. ([\#7866](https://github.com/matrix-org/synapse/issues/7866)) +- Fix a long standing bug where the tracing of async functions with opentracing was broken. ([\#7872](https://github.com/matrix-org/synapse/issues/7872), [\#7961](https://github.com/matrix-org/synapse/issues/7961)) +- Fix "TypeError in `synapse.notifier`" exceptions. ([\#7880](https://github.com/matrix-org/synapse/issues/7880)) +- Fix deprecation warning due to invalid escape sequences. ([\#7895](https://github.com/matrix-org/synapse/issues/7895)) + + +Updates to the Docker image +--------------------------- + +- Base docker image on Debian Buster rather than Alpine Linux. Contributed by @maquis196. ([\#7839](https://github.com/matrix-org/synapse/issues/7839)) + + +Improved Documentation +---------------------- + +- Provide instructions on using `register_new_matrix_user` via docker. ([\#7885](https://github.com/matrix-org/synapse/issues/7885)) +- Change the sample config postgres user section to use `synapse_user` instead of `synapse` to align with the documentation. ([\#7889](https://github.com/matrix-org/synapse/issues/7889)) +- Reorder database paragraphs to promote postgres over sqlite. ([\#7933](https://github.com/matrix-org/synapse/issues/7933)) +- Update the dates of ACME v1's end of life in [`ACME.md`](https://github.com/matrix-org/synapse/blob/master/docs/ACME.md). ([\#7934](https://github.com/matrix-org/synapse/issues/7934)) + + +Deprecations and Removals +------------------------- + +- Remove unused `synapse_replication_tcp_resource_invalidate_cache` prometheus metric. ([\#7878](https://github.com/matrix-org/synapse/issues/7878)) +- Remove Ubuntu Eoan from the list of `.deb` packages that we build as it is now end-of-life. Contributed by @gary-kim. ([\#7888](https://github.com/matrix-org/synapse/issues/7888)) + + +Internal Changes +---------------- + +- Switch parts of the codebase from `simplejson` to the standard library `json`. ([\#7802](https://github.com/matrix-org/synapse/issues/7802)) +- Add type hints to the http server code and remove an unused parameter. ([\#7813](https://github.com/matrix-org/synapse/issues/7813)) +- Add type hints to synapse.api.errors module. ([\#7820](https://github.com/matrix-org/synapse/issues/7820)) +- Ensure that calls to `json.dumps` are compatible with the standard library json. ([\#7836](https://github.com/matrix-org/synapse/issues/7836)) +- Remove redundant `retry_on_integrity_error` wrapper for event persistence code. ([\#7848](https://github.com/matrix-org/synapse/issues/7848)) +- Consistently use `db_to_json` to convert from database values to JSON objects. ([\#7849](https://github.com/matrix-org/synapse/issues/7849)) +- Convert various parts of the codebase to async/await. ([\#7851](https://github.com/matrix-org/synapse/issues/7851), [\#7860](https://github.com/matrix-org/synapse/issues/7860), [\#7868](https://github.com/matrix-org/synapse/issues/7868), [\#7871](https://github.com/matrix-org/synapse/issues/7871), [\#7873](https://github.com/matrix-org/synapse/issues/7873), [\#7874](https://github.com/matrix-org/synapse/issues/7874), [\#7884](https://github.com/matrix-org/synapse/issues/7884), [\#7912](https://github.com/matrix-org/synapse/issues/7912), [\#7935](https://github.com/matrix-org/synapse/issues/7935), [\#7939](https://github.com/matrix-org/synapse/issues/7939), [\#7942](https://github.com/matrix-org/synapse/issues/7942), [\#7944](https://github.com/matrix-org/synapse/issues/7944)) +- Add support for handling registration requests across multiple client reader workers. ([\#7853](https://github.com/matrix-org/synapse/issues/7853)) +- Small performance improvement in typing processing. ([\#7856](https://github.com/matrix-org/synapse/issues/7856)) +- The default value of `filter_timeline_limit` was changed from -1 (no limit) to 100. ([\#7858](https://github.com/matrix-org/synapse/issues/7858)) +- Optimise queueing of inbound replication commands. ([\#7861](https://github.com/matrix-org/synapse/issues/7861)) +- Add some type annotations to `HomeServer` and `BaseHandler`. ([\#7870](https://github.com/matrix-org/synapse/issues/7870)) +- Clean up `PreserveLoggingContext`. ([\#7877](https://github.com/matrix-org/synapse/issues/7877)) +- Change "unknown room version" logging from 'error' to 'warning'. ([\#7881](https://github.com/matrix-org/synapse/issues/7881)) +- Stop using `device_max_stream_id` table and just use `device_inbox.stream_id`. ([\#7882](https://github.com/matrix-org/synapse/issues/7882)) +- Return an empty body for OPTIONS requests. ([\#7886](https://github.com/matrix-org/synapse/issues/7886)) +- Fix typo in generated config file. Contributed by @ThiefMaster. ([\#7890](https://github.com/matrix-org/synapse/issues/7890)) +- Import ABC from `collections.abc` for Python 3.10 compatibility. ([\#7892](https://github.com/matrix-org/synapse/issues/7892)) +- Remove unused functions `time_function`, `trace_function`, `get_previous_frames` + and `get_previous_frame` from `synapse.logging.utils` module. ([\#7897](https://github.com/matrix-org/synapse/issues/7897)) +- Lint the `contrib/` directory in CI and linting scripts, add `synctl` to the linting script for consistency with CI. ([\#7914](https://github.com/matrix-org/synapse/issues/7914)) +- Use Element CSS and logo in notification emails when app name is Element. ([\#7919](https://github.com/matrix-org/synapse/issues/7919)) +- Optimisation to /sync handling: skip serializing the response if the client has already disconnected. ([\#7927](https://github.com/matrix-org/synapse/issues/7927)) +- When a client disconnects, don't log it as 'Error processing request'. ([\#7928](https://github.com/matrix-org/synapse/issues/7928)) +- Add debugging to `/sync` response generation (disabled by default). ([\#7929](https://github.com/matrix-org/synapse/issues/7929)) +- Update comments that refer to Deferreds for async functions. ([\#7945](https://github.com/matrix-org/synapse/issues/7945)) +- Simplify error handling in federation handler. ([\#7950](https://github.com/matrix-org/synapse/issues/7950)) + + +Synapse 1.17.0 (2020-07-13) +=========================== + +Synapse 1.17.0 is identical to 1.17.0rc1, with the addition of the fix that was included in 1.16.1. + + +Synapse 1.16.1 (2020-07-10) +=========================== + +In some distributions of Synapse 1.16.0, we incorrectly included a database migration which added a new, unused table. This release removes the redundant table. + +Bugfixes +-------- + +- Drop table `local_rejections_stream` which was incorrectly added in Synapse 1.16.0. ([\#7816](https://github.com/matrix-org/synapse/issues/7816), [b1beb3ff5](https://github.com/matrix-org/synapse/commit/b1beb3ff5)) + + +Synapse 1.17.0rc1 (2020-07-09) +============================== + +Bugfixes +-------- + +- Fix inconsistent handling of upper and lower case in email addresses when used as identifiers for login, etc. Contributed by @dklimpel. ([\#7021](https://github.com/matrix-org/synapse/issues/7021)) +- Fix "Tried to close a non-active scope!" error messages when opentracing is enabled. ([\#7732](https://github.com/matrix-org/synapse/issues/7732)) +- Fix incorrect error message when database CTYPE was set incorrectly. ([\#7760](https://github.com/matrix-org/synapse/issues/7760)) +- Fix to not ignore `set_tweak` actions in Push Rules that have no `value`, as permitted by the specification. ([\#7766](https://github.com/matrix-org/synapse/issues/7766)) +- Fix synctl to handle empty config files correctly. Contributed by @kotovalexarian. ([\#7779](https://github.com/matrix-org/synapse/issues/7779)) +- Fixes a long standing bug in worker mode where worker information was saved in the devices table instead of the original IP address and user agent. ([\#7797](https://github.com/matrix-org/synapse/issues/7797)) +- Fix 'stuck invites' which happen when we are unable to reject a room invite received over federation. ([\#7804](https://github.com/matrix-org/synapse/issues/7804), [\#7809](https://github.com/matrix-org/synapse/issues/7809), [\#7810](https://github.com/matrix-org/synapse/issues/7810)) + + +Updates to the Docker image +--------------------------- + +- Include libwebp in the Docker file to properly handle webp image uploads. ([\#7791](https://github.com/matrix-org/synapse/issues/7791)) + + +Improved Documentation +---------------------- + +- Improve the documentation of the non-standard JSON web token login type. ([\#7776](https://github.com/matrix-org/synapse/issues/7776)) +- Update doc links for caddy. Contributed by Nicolai Søborg. ([\#7789](https://github.com/matrix-org/synapse/issues/7789)) + + +Internal Changes +---------------- + +- Refactor getting replication updates from database. ([\#7740](https://github.com/matrix-org/synapse/issues/7740)) +- Send push notifications with a high or low priority depending upon whether they may generate user-observable effects. ([\#7765](https://github.com/matrix-org/synapse/issues/7765)) +- Use symbolic names for replication stream names. ([\#7768](https://github.com/matrix-org/synapse/issues/7768)) +- Add early returns to `_check_for_soft_fail`. ([\#7769](https://github.com/matrix-org/synapse/issues/7769)) +- Fix up `synapse.handlers.federation` to pass mypy. ([\#7770](https://github.com/matrix-org/synapse/issues/7770)) +- Convert the appserver handler to async/await. ([\#7775](https://github.com/matrix-org/synapse/issues/7775)) +- Allow to use higher versions of prometheus_client <0.9.0 which are expected to introduce no breaking changes. Contributed by Oliver Kurz. ([\#7780](https://github.com/matrix-org/synapse/issues/7780)) +- Update linting scripts and codebase to be compatible with `isort` v5. ([\#7786](https://github.com/matrix-org/synapse/issues/7786)) +- Stop populating unused table `local_invites`. ([\#7793](https://github.com/matrix-org/synapse/issues/7793)) +- Ensure that strings (not bytes) are passed into JSON serialization. ([\#7799](https://github.com/matrix-org/synapse/issues/7799)) +- Switch from simplejson to the standard library json. ([\#7800](https://github.com/matrix-org/synapse/issues/7800)) +- Add `signing_key` property to `HomeServer` to save code duplication. ([\#7805](https://github.com/matrix-org/synapse/issues/7805)) +- Improve stacktraces from exceptions in background processes. ([\#7808](https://github.com/matrix-org/synapse/issues/7808)) +- Fix various spelling errors in comments and log lines. ([\#7811](https://github.com/matrix-org/synapse/issues/7811)) + + +Synapse 1.16.0 (2020-07-08) +=========================== + +No significant changes since 1.16.0rc2. + +Note that this release deprecates the `m.login.jwt` login method, renaming it +to `org.matrix.login.jwt`, as `m.login.jwt` is not part of the Matrix spec. +Otherwise the behaviour is identical. Synapse will accept both names for now, +but this may change in a future release. + +Synapse 1.16.0rc2 (2020-07-02) +============================== + +Synapse 1.16.0rc2 includes the security fixes released with Synapse 1.15.2. +Please see [below](#synapse-1152-2020-07-02) for more details. + +Improved Documentation +---------------------- + +- Update postgres image in example `docker-compose.yaml` to tag `12-alpine`. ([\#7696](https://github.com/matrix-org/synapse/issues/7696)) + + +Internal Changes +---------------- + +- Add some metrics for inbound and outbound federation latencies: `synapse_federation_server_pdu_process_time` and `synapse_event_processing_lag_by_event`. ([\#7771](https://github.com/matrix-org/synapse/issues/7771)) + + +Synapse 1.15.2 (2020-07-02) +=========================== + +Due to the two security issues highlighted below, server administrators are +encouraged to update Synapse. We are not aware of these vulnerabilities being +exploited in the wild. + +Security advisory +----------------- + +* A malicious homeserver could force Synapse to reset the state in a room to a + small subset of the correct state. This affects all Synapse deployments which + federate with untrusted servers. ([96e9afe6](https://github.com/matrix-org/synapse/commit/96e9afe62500310977dc3cbc99a8d16d3d2fa15c)) +* HTML pages served via Synapse were vulnerable to clickjacking attacks. This + predominantly affects homeservers with single-sign-on enabled, but all server + administrators are encouraged to upgrade. ([ea26e9a9](https://github.com/matrix-org/synapse/commit/ea26e9a98b0541fc886a1cb826a38352b7599dbe)) + + This was reported by [Quentin Gliech](https://sandhose.fr/). + + +Synapse 1.16.0rc1 (2020-07-01) +============================== + +Features +-------- + +- Add an option to enable encryption by default for new rooms. ([\#7639](https://github.com/matrix-org/synapse/issues/7639)) +- Add support for running multiple media repository workers. See [docs/workers.md](https://github.com/matrix-org/synapse/blob/release-v1.16.0/docs/workers.md) for instructions. ([\#7706](https://github.com/matrix-org/synapse/issues/7706)) +- Media can now be marked as safe from quarantined. ([\#7718](https://github.com/matrix-org/synapse/issues/7718)) +- Expand the configuration options for auto-join rooms. ([\#7763](https://github.com/matrix-org/synapse/issues/7763)) + + +Bugfixes +-------- + +- Remove `user_id` from the response to `GET /_matrix/client/r0/presence/{userId}/status` to match the specification. ([\#7606](https://github.com/matrix-org/synapse/issues/7606)) +- In worker mode, ensure that replicated data has not already been received. ([\#7648](https://github.com/matrix-org/synapse/issues/7648)) +- Fix intermittent exception during startup, introduced in Synapse 1.14.0. ([\#7663](https://github.com/matrix-org/synapse/issues/7663)) +- Include a user-agent for federation and well-known requests. ([\#7677](https://github.com/matrix-org/synapse/issues/7677)) +- Accept the proper field (`phone`) for the `m.id.phone` identifier type. The legacy field of `number` is still accepted as a fallback. Bug introduced in v0.20.0. ([\#7687](https://github.com/matrix-org/synapse/issues/7687)) +- Fix "Starting db txn 'get_completed_ui_auth_stages' from sentinel context" warning. The bug was introduced in 1.13.0. ([\#7688](https://github.com/matrix-org/synapse/issues/7688)) +- Compare the URI and method during user interactive authentication (instead of the URI twice). Bug introduced in 1.13.0. ([\#7689](https://github.com/matrix-org/synapse/issues/7689)) +- Fix a long standing bug where the response to the `GET room_keys/version` endpoint had the incorrect type for the `etag` field. ([\#7691](https://github.com/matrix-org/synapse/issues/7691)) +- Fix logged error during device resync in opentracing. Broke in v1.14.0. ([\#7698](https://github.com/matrix-org/synapse/issues/7698)) +- Do not break push rule evaluation when receiving an event with a non-string body. This is a long-standing bug. ([\#7701](https://github.com/matrix-org/synapse/issues/7701)) +- Fixs a long standing bug which resulted in an exception: "TypeError: argument of type 'ObservableDeferred' is not iterable". ([\#7708](https://github.com/matrix-org/synapse/issues/7708)) +- The `synapse_port_db` script no longer fails when the `ui_auth_sessions` table is non-empty. This bug has existed since v1.13.0. ([\#7711](https://github.com/matrix-org/synapse/issues/7711)) +- Synapse will now fetch media from the proper specified URL (using the r0 prefix instead of the unspecified v1). ([\#7714](https://github.com/matrix-org/synapse/issues/7714)) +- Fix the tables ignored by `synapse_port_db` to be in sync the current database schema. ([\#7717](https://github.com/matrix-org/synapse/issues/7717)) +- Fix missing `Content-Length` on HTTP responses from the metrics handler. ([\#7730](https://github.com/matrix-org/synapse/issues/7730)) +- Fix large state resolutions from stalling Synapse for seconds at a time. ([\#7735](https://github.com/matrix-org/synapse/issues/7735), [\#7746](https://github.com/matrix-org/synapse/issues/7746)) + + +Improved Documentation +---------------------- + +- Spelling correction in sample_config.yaml. ([\#7652](https://github.com/matrix-org/synapse/issues/7652)) +- Added instructions for how to use Keycloak via OpenID Connect to authenticate with Synapse. ([\#7659](https://github.com/matrix-org/synapse/issues/7659)) +- Corrected misspelling of PostgreSQL. ([\#7724](https://github.com/matrix-org/synapse/issues/7724)) + + +Deprecations and Removals +------------------------- + +- Deprecate `m.login.jwt` login method in favour of `org.matrix.login.jwt`, as `m.login.jwt` is not part of the Matrix spec. ([\#7675](https://github.com/matrix-org/synapse/issues/7675)) + + +Internal Changes +---------------- + +- Refactor getting replication updates from database. ([\#7636](https://github.com/matrix-org/synapse/issues/7636)) +- Clean-up the login fallback code. ([\#7657](https://github.com/matrix-org/synapse/issues/7657)) +- Increase the default SAML session expiry time to 15 minutes. ([\#7664](https://github.com/matrix-org/synapse/issues/7664)) +- Convert the device message and pagination handlers to async/await. ([\#7678](https://github.com/matrix-org/synapse/issues/7678)) +- Convert typing handler to async/await. ([\#7679](https://github.com/matrix-org/synapse/issues/7679)) +- Require `parameterized` package version to be at least 0.7.0. ([\#7680](https://github.com/matrix-org/synapse/issues/7680)) +- Refactor handling of `listeners` configuration settings. ([\#7681](https://github.com/matrix-org/synapse/issues/7681)) +- Replace uses of `six.iterkeys`/`iteritems`/`itervalues` with `keys()`/`items()`/`values()`. ([\#7692](https://github.com/matrix-org/synapse/issues/7692)) +- Add support for using `rust-python-jaeger-reporter` library to reduce jaeger tracing overhead. ([\#7697](https://github.com/matrix-org/synapse/issues/7697)) +- Make Tox actions work on Debian 10. ([\#7703](https://github.com/matrix-org/synapse/issues/7703)) +- Replace all remaining uses of `six` with native Python 3 equivalents. Contributed by @ilmari. ([\#7704](https://github.com/matrix-org/synapse/issues/7704)) +- Fix broken link in sample config. ([\#7712](https://github.com/matrix-org/synapse/issues/7712)) +- Speed up state res v2 across large state differences. ([\#7725](https://github.com/matrix-org/synapse/issues/7725)) +- Convert directory handler to async/await. ([\#7727](https://github.com/matrix-org/synapse/issues/7727)) +- Move `flake8` to the end of `scripts-dev/lint.sh` as it takes the longest and could cause the script to exit early. ([\#7738](https://github.com/matrix-org/synapse/issues/7738)) +- Explain the "test" conditional requirement for dependencies is not all of the modules necessary to run the unit tests. ([\#7751](https://github.com/matrix-org/synapse/issues/7751)) +- Add some metrics for inbound and outbound federation latencies: `synapse_federation_server_pdu_process_time` and `synapse_event_processing_lag_by_event`. ([\#7755](https://github.com/matrix-org/synapse/issues/7755)) + + +Synapse 1.15.1 (2020-06-16) +=========================== + +Bugfixes +-------- + +- Fix a bug introduced in v1.15.0 that would crash Synapse on start when using certain password auth providers. ([\#7684](https://github.com/matrix-org/synapse/issues/7684)) +- Fix a bug introduced in v1.15.0 which meant that some 3PID management endpoints were not accessible on the correct URL. ([\#7685](https://github.com/matrix-org/synapse/issues/7685)) + + +Synapse 1.15.0 (2020-06-11) +=========================== + +No significant changes. + + +Synapse 1.15.0rc1 (2020-06-09) +============================== + +Features +-------- + +- Advertise support for Client-Server API r0.6.0 and remove related unstable feature flags. ([\#6585](https://github.com/matrix-org/synapse/issues/6585)) +- Add an option to disable autojoining rooms for guest accounts. ([\#6637](https://github.com/matrix-org/synapse/issues/6637)) +- For SAML authentication, add the ability to pass email addresses to be added to new users' accounts via SAML attributes. Contributed by Christopher Cooper. ([\#7385](https://github.com/matrix-org/synapse/issues/7385)) +- Add admin APIs to allow server admins to manage users' devices. Contributed by @dklimpel. ([\#7481](https://github.com/matrix-org/synapse/issues/7481)) +- Add support for generating thumbnails for WebP images. Previously, users would see an empty box instead of preview image. Contributed by @WGH-. ([\#7586](https://github.com/matrix-org/synapse/issues/7586)) +- Support the standardized `m.login.sso` user-interactive authentication flow. ([\#7630](https://github.com/matrix-org/synapse/issues/7630)) + + +Bugfixes +-------- + +- Allow new users to be registered via the admin API even if the monthly active user limit has been reached. Contributed by @dklimpel. ([\#7263](https://github.com/matrix-org/synapse/issues/7263)) +- Fix email notifications not being enabled for new users when created via the Admin API. ([\#7267](https://github.com/matrix-org/synapse/issues/7267)) +- Fix str placeholders in an instance of `PrepareDatabaseException`. Introduced in Synapse v1.8.0. ([\#7575](https://github.com/matrix-org/synapse/issues/7575)) +- Fix a bug in automatic user creation during first time login with `m.login.jwt`. Regression in v1.6.0. Contributed by @olof. ([\#7585](https://github.com/matrix-org/synapse/issues/7585)) +- Fix a bug causing the cross-signing keys to be ignored when resyncing a device list. ([\#7594](https://github.com/matrix-org/synapse/issues/7594)) +- Fix metrics failing when there is a large number of active background processes. ([\#7597](https://github.com/matrix-org/synapse/issues/7597)) +- Fix bug where returning rooms for a group would fail if it included a room that the server was not in. ([\#7599](https://github.com/matrix-org/synapse/issues/7599)) +- Fix duplicate key violation when persisting read markers. ([\#7607](https://github.com/matrix-org/synapse/issues/7607)) +- Prevent an entire iteration of the device list resync loop from failing if one server responds with a malformed result. ([\#7609](https://github.com/matrix-org/synapse/issues/7609)) +- Fix exceptions when fetching events from a remote host fails. ([\#7622](https://github.com/matrix-org/synapse/issues/7622)) +- Make `synctl restart` start synapse if it wasn't running. ([\#7624](https://github.com/matrix-org/synapse/issues/7624)) +- Pass device information through to the login endpoint when using the login fallback. ([\#7629](https://github.com/matrix-org/synapse/issues/7629)) +- Advertise the `m.login.token` login flow when OpenID Connect is enabled. ([\#7631](https://github.com/matrix-org/synapse/issues/7631)) +- Fix bug in account data replication stream. ([\#7656](https://github.com/matrix-org/synapse/issues/7656)) + + +Improved Documentation +---------------------- + +- Update the OpenBSD installation instructions. ([\#7587](https://github.com/matrix-org/synapse/issues/7587)) +- Advertise Python 3.8 support in `setup.py`. ([\#7602](https://github.com/matrix-org/synapse/issues/7602)) +- Add a link to `#synapse:matrix.org` in the troubleshooting section of the README. ([\#7603](https://github.com/matrix-org/synapse/issues/7603)) +- Clarifications to the admin api documentation. ([\#7647](https://github.com/matrix-org/synapse/issues/7647)) + + +Internal Changes +---------------- + +- Convert the identity handler to async/await. ([\#7561](https://github.com/matrix-org/synapse/issues/7561)) +- Improve query performance for fetching state from a PostgreSQL database. Contributed by @ilmari. ([\#7567](https://github.com/matrix-org/synapse/issues/7567)) +- Speed up processing of federation stream RDATA rows. ([\#7584](https://github.com/matrix-org/synapse/issues/7584)) +- Add comment to systemd example to show postgresql dependency. ([\#7591](https://github.com/matrix-org/synapse/issues/7591)) +- Refactor `Ratelimiter` to limit the amount of expensive config value accesses. ([\#7595](https://github.com/matrix-org/synapse/issues/7595)) +- Convert groups handlers to async/await. ([\#7600](https://github.com/matrix-org/synapse/issues/7600)) +- Clean up exception handling in `SAML2ResponseResource`. ([\#7614](https://github.com/matrix-org/synapse/issues/7614)) +- Check that all asynchronous tasks succeed and general cleanup of `MonthlyActiveUsersTestCase` and `TestMauLimit`. ([\#7619](https://github.com/matrix-org/synapse/issues/7619)) +- Convert `get_user_id_by_threepid` to async/await. ([\#7620](https://github.com/matrix-org/synapse/issues/7620)) +- Switch to upstream `dh-virtualenv` rather than our fork for Debian package builds. ([\#7621](https://github.com/matrix-org/synapse/issues/7621)) +- Update CI scripts to check the number in the newsfile fragment. ([\#7623](https://github.com/matrix-org/synapse/issues/7623)) +- Check if the localpart of a Matrix ID is reserved for guest users earlier in the registration flow, as well as when responding to requests to `/register/available`. ([\#7625](https://github.com/matrix-org/synapse/issues/7625)) +- Minor cleanups to OpenID Connect integration. ([\#7628](https://github.com/matrix-org/synapse/issues/7628)) +- Attempt to fix flaky test: `PhoneHomeStatsTestCase.test_performance_100`. ([\#7634](https://github.com/matrix-org/synapse/issues/7634)) +- Fix typos of `m.olm.curve25519-aes-sha2` and `m.megolm.v1.aes-sha2` in comments, test files. ([\#7637](https://github.com/matrix-org/synapse/issues/7637)) +- Convert user directory, state deltas, and stats handlers to async/await. ([\#7640](https://github.com/matrix-org/synapse/issues/7640)) +- Remove some unused constants. ([\#7644](https://github.com/matrix-org/synapse/issues/7644)) +- Fix type information on `assert_*_is_admin` methods. ([\#7645](https://github.com/matrix-org/synapse/issues/7645)) +- Convert registration handler to async/await. ([\#7649](https://github.com/matrix-org/synapse/issues/7649)) + + +Synapse 1.14.0 (2020-05-28) +=========================== + +No significant changes. + + +Synapse 1.14.0rc2 (2020-05-27) +============================== + +Bugfixes +-------- + +- Fix cache config to not apply cache factor to event cache. Regression in v1.14.0rc1. ([\#7578](https://github.com/matrix-org/synapse/issues/7578)) +- Fix bug where `ReplicationStreamer` was not always started when replication was enabled. Bug introduced in v1.14.0rc1. ([\#7579](https://github.com/matrix-org/synapse/issues/7579)) +- Fix specifying individual cache factors for caches with special characters in their name. Regression in v1.14.0rc1. ([\#7580](https://github.com/matrix-org/synapse/issues/7580)) + + +Improved Documentation +---------------------- + +- Fix the OIDC `client_auth_method` value in the sample config. ([\#7581](https://github.com/matrix-org/synapse/issues/7581)) + + +Synapse 1.14.0rc1 (2020-05-26) +============================== + +Features +-------- + +- Synapse's cache factor can now be configured in `homeserver.yaml` by the `caches.global_factor` setting. Additionally, `caches.per_cache_factors` controls the cache factors for individual caches. ([\#6391](https://github.com/matrix-org/synapse/issues/6391)) +- Add OpenID Connect login/registration support. Contributed by Quentin Gliech, on behalf of [les Connecteurs](https://connecteu.rs). ([\#7256](https://github.com/matrix-org/synapse/issues/7256), [\#7457](https://github.com/matrix-org/synapse/issues/7457)) +- Add room details admin endpoint. Contributed by Awesome Technologies Innovationslabor GmbH. ([\#7317](https://github.com/matrix-org/synapse/issues/7317)) +- Allow for using more than one spam checker module at once. ([\#7435](https://github.com/matrix-org/synapse/issues/7435)) +- Add additional authentication checks for `m.room.power_levels` event per [MSC2209](https://github.com/matrix-org/matrix-doc/pull/2209). ([\#7502](https://github.com/matrix-org/synapse/issues/7502)) +- Implement room version 6 per [MSC2240](https://github.com/matrix-org/matrix-doc/pull/2240). ([\#7506](https://github.com/matrix-org/synapse/issues/7506)) +- Add highly experimental option to move event persistence off master. ([\#7281](https://github.com/matrix-org/synapse/issues/7281), [\#7374](https://github.com/matrix-org/synapse/issues/7374), [\#7436](https://github.com/matrix-org/synapse/issues/7436), [\#7440](https://github.com/matrix-org/synapse/issues/7440), [\#7475](https://github.com/matrix-org/synapse/issues/7475), [\#7490](https://github.com/matrix-org/synapse/issues/7490), [\#7491](https://github.com/matrix-org/synapse/issues/7491), [\#7492](https://github.com/matrix-org/synapse/issues/7492), [\#7493](https://github.com/matrix-org/synapse/issues/7493), [\#7495](https://github.com/matrix-org/synapse/issues/7495), [\#7515](https://github.com/matrix-org/synapse/issues/7515), [\#7516](https://github.com/matrix-org/synapse/issues/7516), [\#7517](https://github.com/matrix-org/synapse/issues/7517), [\#7542](https://github.com/matrix-org/synapse/issues/7542)) + + +Bugfixes +-------- + +- Fix a bug where event updates might not be sent over replication to worker processes after the stream falls behind. ([\#7384](https://github.com/matrix-org/synapse/issues/7384)) +- Allow expired user accounts to log out their device sessions. ([\#7443](https://github.com/matrix-org/synapse/issues/7443)) +- Fix a bug that would cause Synapse not to resync out-of-sync device lists. ([\#7453](https://github.com/matrix-org/synapse/issues/7453)) +- Prevent rooms with 0 members or with invalid version strings from breaking group queries. ([\#7465](https://github.com/matrix-org/synapse/issues/7465)) +- Workaround for an upstream Twisted bug that caused Synapse to become unresponsive after startup. ([\#7473](https://github.com/matrix-org/synapse/issues/7473)) +- Fix Redis reconnection logic that can result in missed updates over replication if master reconnects to Redis without restarting. ([\#7482](https://github.com/matrix-org/synapse/issues/7482)) +- When sending `m.room.member` events, omit `displayname` and `avatar_url` if they aren't set instead of setting them to `null`. Contributed by Aaron Raimist. ([\#7497](https://github.com/matrix-org/synapse/issues/7497)) +- Fix incorrect `method` label on `synapse_http_matrixfederationclient_{requests,responses}` prometheus metrics. ([\#7503](https://github.com/matrix-org/synapse/issues/7503)) +- Ignore incoming presence events from other homeservers if presence is disabled locally. ([\#7508](https://github.com/matrix-org/synapse/issues/7508)) +- Fix a long-standing bug that broke the update remote profile background process. ([\#7511](https://github.com/matrix-org/synapse/issues/7511)) +- Hash passwords as early as possible during password reset. ([\#7538](https://github.com/matrix-org/synapse/issues/7538)) +- Fix bug where a local user leaving a room could fail under rare circumstances. ([\#7548](https://github.com/matrix-org/synapse/issues/7548)) +- Fix "Missing RelayState parameter" error when using user interactive authentication with SAML for some SAML providers. ([\#7552](https://github.com/matrix-org/synapse/issues/7552)) +- Fix exception `'GenericWorkerReplicationHandler' object has no attribute 'send_federation_ack'`, introduced in v1.13.0. ([\#7564](https://github.com/matrix-org/synapse/issues/7564)) +- `synctl` now warns if it was unable to stop Synapse and will not attempt to start Synapse if nothing was stopped. Contributed by Romain Bouyé. ([\#6598](https://github.com/matrix-org/synapse/issues/6598)) + + +Updates to the Docker image +--------------------------- + +- Update docker runtime image to Alpine v3.11. Contributed by @Starbix. ([\#7398](https://github.com/matrix-org/synapse/issues/7398)) + + +Improved Documentation +---------------------- + +- Update information about mapping providers for SAML and OpenID. ([\#7458](https://github.com/matrix-org/synapse/issues/7458)) +- Add additional reverse proxy example for Caddy v2. Contributed by Jeff Peeler. ([\#7463](https://github.com/matrix-org/synapse/issues/7463)) +- Fix copy-paste error in `ServerNoticesConfig` docstring. Contributed by @ptman. ([\#7477](https://github.com/matrix-org/synapse/issues/7477)) +- Improve the formatting of `reverse_proxy.md`. ([\#7514](https://github.com/matrix-org/synapse/issues/7514)) +- Change the systemd worker service to check that the worker config file exists instead of silently failing. Contributed by David Vo. ([\#7528](https://github.com/matrix-org/synapse/issues/7528)) +- Minor clarifications to the TURN docs. ([\#7533](https://github.com/matrix-org/synapse/issues/7533)) + + +Internal Changes +---------------- + +- Add typing annotations in `synapse.federation`. ([\#7382](https://github.com/matrix-org/synapse/issues/7382)) +- Convert the room handler to async/await. ([\#7396](https://github.com/matrix-org/synapse/issues/7396)) +- Improve performance of `get_e2e_cross_signing_key`. ([\#7428](https://github.com/matrix-org/synapse/issues/7428)) +- Improve performance of `mark_as_sent_devices_by_remote`. ([\#7429](https://github.com/matrix-org/synapse/issues/7429), [\#7562](https://github.com/matrix-org/synapse/issues/7562)) +- Add type hints to the SAML handler. ([\#7445](https://github.com/matrix-org/synapse/issues/7445)) +- Remove storage method `get_hosts_in_room` that is no longer called anywhere. ([\#7448](https://github.com/matrix-org/synapse/issues/7448)) +- Fix some typos in the `notice_expiry` templates. ([\#7449](https://github.com/matrix-org/synapse/issues/7449)) +- Convert the federation handler to async/await. ([\#7459](https://github.com/matrix-org/synapse/issues/7459)) +- Convert the search handler to async/await. ([\#7460](https://github.com/matrix-org/synapse/issues/7460)) +- Add type hints to `synapse.event_auth`. ([\#7505](https://github.com/matrix-org/synapse/issues/7505)) +- Convert the room member handler to async/await. ([\#7507](https://github.com/matrix-org/synapse/issues/7507)) +- Add type hints to room member handler. ([\#7513](https://github.com/matrix-org/synapse/issues/7513)) +- Fix typing annotations in `tests.replication`. ([\#7518](https://github.com/matrix-org/synapse/issues/7518)) +- Remove some redundant Python 2 support code. ([\#7519](https://github.com/matrix-org/synapse/issues/7519)) +- All endpoints now respond with a 200 OK for `OPTIONS` requests. ([\#7534](https://github.com/matrix-org/synapse/issues/7534), [\#7560](https://github.com/matrix-org/synapse/issues/7560)) +- Synapse now exports [detailed allocator statistics](https://doc.pypy.org/en/latest/gc_info.html#gc-get-stats) and basic GC timings as Prometheus metrics (`pypy_gc_time_seconds_total` and `pypy_memory_bytes`) when run under PyPy. Contributed by Ivan Shapovalov. ([\#7536](https://github.com/matrix-org/synapse/issues/7536)) +- Remove Ubuntu Cosmic and Disco from the list of distributions which we provide `.deb`s for, due to end-of-life. ([\#7539](https://github.com/matrix-org/synapse/issues/7539)) +- Make worker processes return a stubbed-out response to `GET /presence` requests. ([\#7545](https://github.com/matrix-org/synapse/issues/7545)) +- Optimise some references to `hs.config`. ([\#7546](https://github.com/matrix-org/synapse/issues/7546)) +- On upgrade room only send canonical alias once. ([\#7547](https://github.com/matrix-org/synapse/issues/7547)) +- Fix some indentation inconsistencies in the sample config. ([\#7550](https://github.com/matrix-org/synapse/issues/7550)) +- Include `synapse.http.site` in type checking. ([\#7553](https://github.com/matrix-org/synapse/issues/7553)) +- Fix some test code to not mangle stacktraces, to make it easier to debug errors. ([\#7554](https://github.com/matrix-org/synapse/issues/7554)) +- Refresh apt cache when building `dh_virtualenv` docker image. ([\#7555](https://github.com/matrix-org/synapse/issues/7555)) +- Stop logging some expected HTTP request errors as exceptions. ([\#7556](https://github.com/matrix-org/synapse/issues/7556), [\#7563](https://github.com/matrix-org/synapse/issues/7563)) +- Convert sending mail to async/await. ([\#7557](https://github.com/matrix-org/synapse/issues/7557)) +- Simplify `reap_monthly_active_users`. ([\#7558](https://github.com/matrix-org/synapse/issues/7558)) + + +Synapse 1.13.0 (2020-05-19) +=========================== + +This release brings some potential changes necessary for certain +configurations of Synapse: + +* If your Synapse is configured to use SSO and have a custom + `sso_redirect_confirm_template_dir` configuration option set, you will need + to duplicate the new `sso_auth_confirm.html`, `sso_auth_success.html` and + `sso_account_deactivated.html` templates into that directory. +* Synapse plugins using the `complete_sso_login` method of + `synapse.module_api.ModuleApi` should instead switch to the async/await + version, `complete_sso_login_async`, which includes additional checks. The + former version is now deprecated. +* A bug was introduced in Synapse 1.4.0 which could cause the room directory + to be incomplete or empty if Synapse was upgraded directly from v1.2.1 or + earlier, to versions between v1.4.0 and v1.12.x. + +Please review the [upgrade notes](docs/upgrade.md) for more details on these changes +and for general upgrade guidance. + + +Notice of change to the default `git` branch for Synapse +-------------------------------------------------------- + +With the release of Synapse 1.13.0, the default `git` branch for Synapse has +changed to `develop`, which is the development tip. This is more consistent with +common practice and modern `git` usage. + +The `master` branch, which tracks the latest release, is still available. It is +recommended that developers and distributors who have scripts which run builds +using the default branch of Synapse should therefore consider pinning their +scripts to `master`. + + +Internal Changes +---------------- + +- Update the version of dh-virtualenv we use to build debs, and add focal to the list of target distributions. ([\#7526](https://github.com/matrix-org/synapse/issues/7526)) + + +Synapse 1.13.0rc3 (2020-05-18) +============================== + +Bugfixes +-------- + +- Hash passwords as early as possible during registration. ([\#7523](https://github.com/matrix-org/synapse/issues/7523)) + + +Synapse 1.13.0rc2 (2020-05-14) +============================== + +Bugfixes +-------- + +- Fix a long-standing bug which could cause messages not to be sent over federation, when state events with state keys matching user IDs (such as custom user statuses) were received. ([\#7376](https://github.com/matrix-org/synapse/issues/7376)) +- Restore compatibility with non-compliant clients during the user interactive authentication process, fixing a problem introduced in v1.13.0rc1. ([\#7483](https://github.com/matrix-org/synapse/issues/7483)) + +Internal Changes +---------------- + +- Fix linting errors in new version of Flake8. ([\#7470](https://github.com/matrix-org/synapse/issues/7470)) + + +Synapse 1.13.0rc1 (2020-05-11) +============================== + +Features +-------- + +- Extend the `web_client_location` option to accept an absolute URL to use as a redirect. Adds a warning when running the web client on the same hostname as homeserver. Contributed by Martin Milata. ([\#7006](https://github.com/matrix-org/synapse/issues/7006)) +- Set `Referrer-Policy` header to `no-referrer` on media downloads. ([\#7009](https://github.com/matrix-org/synapse/issues/7009)) +- Add support for running replication over Redis when using workers. ([\#7040](https://github.com/matrix-org/synapse/issues/7040), [\#7325](https://github.com/matrix-org/synapse/issues/7325), [\#7352](https://github.com/matrix-org/synapse/issues/7352), [\#7401](https://github.com/matrix-org/synapse/issues/7401), [\#7427](https://github.com/matrix-org/synapse/issues/7427), [\#7439](https://github.com/matrix-org/synapse/issues/7439), [\#7446](https://github.com/matrix-org/synapse/issues/7446), [\#7450](https://github.com/matrix-org/synapse/issues/7450), [\#7454](https://github.com/matrix-org/synapse/issues/7454)) +- Admin API `POST /_synapse/admin/v1/join/` to join users to a room like `auto_join_rooms` for creation of users. ([\#7051](https://github.com/matrix-org/synapse/issues/7051)) +- Add options to prevent users from changing their profile or associated 3PIDs. ([\#7096](https://github.com/matrix-org/synapse/issues/7096)) +- Support SSO in the user interactive authentication workflow. ([\#7102](https://github.com/matrix-org/synapse/issues/7102), [\#7186](https://github.com/matrix-org/synapse/issues/7186), [\#7279](https://github.com/matrix-org/synapse/issues/7279), [\#7343](https://github.com/matrix-org/synapse/issues/7343)) +- Allow server admins to define and enforce a password policy ([MSC2000](https://github.com/matrix-org/matrix-doc/issues/2000)). ([\#7118](https://github.com/matrix-org/synapse/issues/7118)) +- Improve the support for SSO authentication on the login fallback page. ([\#7152](https://github.com/matrix-org/synapse/issues/7152), [\#7235](https://github.com/matrix-org/synapse/issues/7235)) +- Always whitelist the login fallback in the SSO configuration if `public_baseurl` is set. ([\#7153](https://github.com/matrix-org/synapse/issues/7153)) +- Admin users are no longer required to be in a room to create an alias for it. ([\#7191](https://github.com/matrix-org/synapse/issues/7191)) +- Require admin privileges to enable room encryption by default. This does not affect existing rooms. ([\#7230](https://github.com/matrix-org/synapse/issues/7230)) +- Add a config option for specifying the value of the Accept-Language HTTP header when generating URL previews. ([\#7265](https://github.com/matrix-org/synapse/issues/7265)) +- Allow `/requestToken` endpoints to hide the existence (or lack thereof) of 3PID associations on the homeserver. ([\#7315](https://github.com/matrix-org/synapse/issues/7315)) +- Add a configuration setting to tweak the threshold for dummy events. ([\#7422](https://github.com/matrix-org/synapse/issues/7422)) + + +Bugfixes +-------- + +- Don't attempt to use an invalid sqlite config if no database configuration is provided. Contributed by @nekatak. ([\#6573](https://github.com/matrix-org/synapse/issues/6573)) +- Fix single-sign on with CAS systems: pass the same service URL when requesting the CAS ticket and when calling the `proxyValidate` URL. Contributed by @Naugrimm. ([\#6634](https://github.com/matrix-org/synapse/issues/6634)) +- Fix missing field `default` when fetching user-defined push rules. ([\#6639](https://github.com/matrix-org/synapse/issues/6639)) +- Improve error responses when accessing remote public room lists. ([\#6899](https://github.com/matrix-org/synapse/issues/6899), [\#7368](https://github.com/matrix-org/synapse/issues/7368)) +- Transfer alias mappings on room upgrade. ([\#6946](https://github.com/matrix-org/synapse/issues/6946)) +- Ensure that a user interactive authentication session is tied to a single request. ([\#7068](https://github.com/matrix-org/synapse/issues/7068), [\#7455](https://github.com/matrix-org/synapse/issues/7455)) +- Fix a bug in the federation API which could cause occasional "Failed to get PDU" errors. ([\#7089](https://github.com/matrix-org/synapse/issues/7089)) +- Return the proper error (`M_BAD_ALIAS`) when a non-existant canonical alias is provided. ([\#7109](https://github.com/matrix-org/synapse/issues/7109)) +- Fix a bug which meant that groups updates were not correctly replicated between workers. ([\#7117](https://github.com/matrix-org/synapse/issues/7117)) +- Fix starting workers when federation sending not split out. ([\#7133](https://github.com/matrix-org/synapse/issues/7133)) +- Ensure `is_verified` is a boolean in responses to `GET /_matrix/client/r0/room_keys/keys`. Also warn the user if they forgot the `version` query param. ([\#7150](https://github.com/matrix-org/synapse/issues/7150)) +- Fix error page being shown when a custom SAML handler attempted to redirect when processing an auth response. ([\#7151](https://github.com/matrix-org/synapse/issues/7151)) +- Avoid importing `sqlite3` when using the postgres backend. Contributed by David Vo. ([\#7155](https://github.com/matrix-org/synapse/issues/7155)) +- Fix excessive CPU usage by `prune_old_outbound_device_pokes` job. ([\#7159](https://github.com/matrix-org/synapse/issues/7159)) +- Fix a bug which could cause outbound federation traffic to stop working if a client uploaded an incorrect e2e device signature. ([\#7177](https://github.com/matrix-org/synapse/issues/7177)) +- Fix a bug which could cause incorrect 'cyclic dependency' error. ([\#7178](https://github.com/matrix-org/synapse/issues/7178)) +- Fix a bug that could cause a user to be invited to a server notices (aka System Alerts) room without any notice being sent. ([\#7199](https://github.com/matrix-org/synapse/issues/7199)) +- Fix some worker-mode replication handling not being correctly recorded in CPU usage stats. ([\#7203](https://github.com/matrix-org/synapse/issues/7203)) +- Do not allow a deactivated user to login via SSO. ([\#7240](https://github.com/matrix-org/synapse/issues/7240), [\#7259](https://github.com/matrix-org/synapse/issues/7259)) +- Fix --help command-line argument. ([\#7249](https://github.com/matrix-org/synapse/issues/7249)) +- Fix room publish permissions not being checked on room creation. ([\#7260](https://github.com/matrix-org/synapse/issues/7260)) +- Reject unknown session IDs during user interactive authentication instead of silently creating a new session. ([\#7268](https://github.com/matrix-org/synapse/issues/7268)) +- Fix a SQL query introduced in Synapse 1.12.0 which could cause large amounts of logging to the postgres slow-query log. ([\#7274](https://github.com/matrix-org/synapse/issues/7274)) +- Persist user interactive authentication sessions across workers and Synapse restarts. ([\#7302](https://github.com/matrix-org/synapse/issues/7302)) +- Fixed backwards compatibility logic of the first value of `trusted_third_party_id_servers` being used for `account_threepid_delegates.email`, which occurs when the former, deprecated option is set and the latter is not. ([\#7316](https://github.com/matrix-org/synapse/issues/7316)) +- Fix a bug where event updates might not be sent over replication to worker processes after the stream falls behind. ([\#7337](https://github.com/matrix-org/synapse/issues/7337), [\#7358](https://github.com/matrix-org/synapse/issues/7358)) +- Fix bad error handling that would cause Synapse to crash if it's provided with a YAML configuration file that's either empty or doesn't parse into a key-value map. ([\#7341](https://github.com/matrix-org/synapse/issues/7341)) +- Fix incorrect metrics reporting for `renew_attestations` background task. ([\#7344](https://github.com/matrix-org/synapse/issues/7344)) +- Prevent non-federating rooms from appearing in responses to federated `POST /publicRoom` requests when a filter was included. ([\#7367](https://github.com/matrix-org/synapse/issues/7367)) +- Fix a bug which would cause the room durectory to be incorrectly populated if Synapse was upgraded directly from v1.2.1 or earlier to v1.4.0 or later. Note that this fix does not apply retrospectively; see the [upgrade notes](docs/upgrade.md#upgrading-to-v1130) for more information. ([\#7387](https://github.com/matrix-org/synapse/issues/7387)) +- Fix bug in `EventContext.deserialize`. ([\#7393](https://github.com/matrix-org/synapse/issues/7393)) + + +Improved Documentation +---------------------- + +- Update Debian installation instructions to recommend installing the `virtualenv` package instead of `python3-virtualenv`. ([\#6892](https://github.com/matrix-org/synapse/issues/6892)) +- Improve the documentation for database configuration. ([\#6988](https://github.com/matrix-org/synapse/issues/6988)) +- Improve the documentation of application service configuration files. ([\#7091](https://github.com/matrix-org/synapse/issues/7091)) +- Update pre-built package name for FreeBSD. ([\#7107](https://github.com/matrix-org/synapse/issues/7107)) +- Update postgres docs with login troubleshooting information. ([\#7119](https://github.com/matrix-org/synapse/issues/7119)) +- Clean up INSTALL.md a bit. ([\#7141](https://github.com/matrix-org/synapse/issues/7141)) +- Add documentation for running a local CAS server for testing. ([\#7147](https://github.com/matrix-org/synapse/issues/7147)) +- Improve README.md by being explicit about public IP recommendation for TURN relaying. ([\#7167](https://github.com/matrix-org/synapse/issues/7167)) +- Fix a small typo in the `metrics_flags` config option. ([\#7171](https://github.com/matrix-org/synapse/issues/7171)) +- Update the contributed documentation on managing synapse workers with systemd, and bring it into the core distribution. ([\#7234](https://github.com/matrix-org/synapse/issues/7234)) +- Add documentation to the `password_providers` config option. Add known password provider implementations to docs. ([\#7238](https://github.com/matrix-org/synapse/issues/7238), [\#7248](https://github.com/matrix-org/synapse/issues/7248)) +- Modify suggested nginx reverse proxy configuration to match Synapse's default file upload size. Contributed by @ProCycleDev. ([\#7251](https://github.com/matrix-org/synapse/issues/7251)) +- Documentation of media_storage_providers options updated to avoid misunderstandings. Contributed by Tristan Lins. ([\#7272](https://github.com/matrix-org/synapse/issues/7272)) +- Add documentation on monitoring workers with Prometheus. ([\#7357](https://github.com/matrix-org/synapse/issues/7357)) +- Clarify endpoint usage in the users admin api documentation. ([\#7361](https://github.com/matrix-org/synapse/issues/7361)) + + +Deprecations and Removals +------------------------- + +- Remove nonfunctional `captcha_bypass_secret` option from `homeserver.yaml`. ([\#7137](https://github.com/matrix-org/synapse/issues/7137)) + + +Internal Changes +---------------- + +- Add benchmarks for LruCache. ([\#6446](https://github.com/matrix-org/synapse/issues/6446)) +- Return total number of users and profile attributes in admin users endpoint. Contributed by Awesome Technologies Innovationslabor GmbH. ([\#6881](https://github.com/matrix-org/synapse/issues/6881)) +- Change device list streams to have one row per ID. ([\#7010](https://github.com/matrix-org/synapse/issues/7010)) +- Remove concept of a non-limited stream. ([\#7011](https://github.com/matrix-org/synapse/issues/7011)) +- Move catchup of replication streams logic to worker. ([\#7024](https://github.com/matrix-org/synapse/issues/7024), [\#7195](https://github.com/matrix-org/synapse/issues/7195), [\#7226](https://github.com/matrix-org/synapse/issues/7226), [\#7239](https://github.com/matrix-org/synapse/issues/7239), [\#7286](https://github.com/matrix-org/synapse/issues/7286), [\#7290](https://github.com/matrix-org/synapse/issues/7290), [\#7318](https://github.com/matrix-org/synapse/issues/7318), [\#7326](https://github.com/matrix-org/synapse/issues/7326), [\#7378](https://github.com/matrix-org/synapse/issues/7378), [\#7421](https://github.com/matrix-org/synapse/issues/7421)) +- Convert some of synapse.rest.media to async/await. ([\#7110](https://github.com/matrix-org/synapse/issues/7110), [\#7184](https://github.com/matrix-org/synapse/issues/7184), [\#7241](https://github.com/matrix-org/synapse/issues/7241)) +- De-duplicate / remove unused REST code for login and auth. ([\#7115](https://github.com/matrix-org/synapse/issues/7115)) +- Convert `*StreamRow` classes to inner classes. ([\#7116](https://github.com/matrix-org/synapse/issues/7116)) +- Clean up some LoggingContext code. ([\#7120](https://github.com/matrix-org/synapse/issues/7120), [\#7181](https://github.com/matrix-org/synapse/issues/7181), [\#7183](https://github.com/matrix-org/synapse/issues/7183), [\#7408](https://github.com/matrix-org/synapse/issues/7408), [\#7426](https://github.com/matrix-org/synapse/issues/7426)) +- Add explicit `instance_id` for USER_SYNC commands and remove implicit `conn_id` usage. ([\#7128](https://github.com/matrix-org/synapse/issues/7128)) +- Refactored the CAS authentication logic to a separate class. ([\#7136](https://github.com/matrix-org/synapse/issues/7136)) +- Run replication streamers on workers. ([\#7146](https://github.com/matrix-org/synapse/issues/7146)) +- Add tests for outbound device pokes. ([\#7157](https://github.com/matrix-org/synapse/issues/7157)) +- Fix device list update stream ids going backward. ([\#7158](https://github.com/matrix-org/synapse/issues/7158)) +- Use `stream.current_token()` and remove `stream_positions()`. ([\#7172](https://github.com/matrix-org/synapse/issues/7172)) +- Move client command handling out of TCP protocol. ([\#7185](https://github.com/matrix-org/synapse/issues/7185)) +- Move server command handling out of TCP protocol. ([\#7187](https://github.com/matrix-org/synapse/issues/7187)) +- Fix consistency of HTTP status codes reported in log lines. ([\#7188](https://github.com/matrix-org/synapse/issues/7188)) +- Only run one background database update at a time. ([\#7190](https://github.com/matrix-org/synapse/issues/7190)) +- Remove sent outbound device list pokes from the database. ([\#7192](https://github.com/matrix-org/synapse/issues/7192)) +- Add a background database update job to clear out duplicate `device_lists_outbound_pokes`. ([\#7193](https://github.com/matrix-org/synapse/issues/7193)) +- Remove some extraneous debugging log lines. ([\#7207](https://github.com/matrix-org/synapse/issues/7207)) +- Add explicit Python build tooling as dependencies for the snapcraft build. ([\#7213](https://github.com/matrix-org/synapse/issues/7213)) +- Add typing information to federation server code. ([\#7219](https://github.com/matrix-org/synapse/issues/7219)) +- Extend room admin api (`GET /_synapse/admin/v1/rooms`) with additional attributes. ([\#7225](https://github.com/matrix-org/synapse/issues/7225)) +- Unblacklist '/upgrade creates a new room' sytest for workers. ([\#7228](https://github.com/matrix-org/synapse/issues/7228)) +- Remove redundant checks on `daemonize` from synctl. ([\#7233](https://github.com/matrix-org/synapse/issues/7233)) +- Upgrade jQuery to v3.4.1 on fallback login/registration pages. ([\#7236](https://github.com/matrix-org/synapse/issues/7236)) +- Change log line that told user to implement onLogin/onRegister fallback js functions to a warning, instead of an info, so it's more visible. ([\#7237](https://github.com/matrix-org/synapse/issues/7237)) +- Correct the parameters of a test fixture. Contributed by Isaiah Singletary. ([\#7243](https://github.com/matrix-org/synapse/issues/7243)) +- Convert auth handler to async/await. ([\#7261](https://github.com/matrix-org/synapse/issues/7261)) +- Add some unit tests for replication. ([\#7278](https://github.com/matrix-org/synapse/issues/7278)) +- Improve typing annotations in `synapse.replication.tcp.streams.Stream`. ([\#7291](https://github.com/matrix-org/synapse/issues/7291)) +- Reduce log verbosity of url cache cleanup tasks. ([\#7295](https://github.com/matrix-org/synapse/issues/7295)) +- Fix sample SAML Service Provider configuration. Contributed by @frcl. ([\#7300](https://github.com/matrix-org/synapse/issues/7300)) +- Fix StreamChangeCache to work with multiple entities changing on the same stream id. ([\#7303](https://github.com/matrix-org/synapse/issues/7303)) +- Fix an incorrect import in IdentityHandler. ([\#7319](https://github.com/matrix-org/synapse/issues/7319)) +- Reduce logging verbosity for successful federation requests. ([\#7321](https://github.com/matrix-org/synapse/issues/7321)) +- Convert some federation handler code to async/await. ([\#7338](https://github.com/matrix-org/synapse/issues/7338)) +- Fix collation for postgres for unit tests. ([\#7359](https://github.com/matrix-org/synapse/issues/7359)) +- Convert RegistrationWorkerStore.is_server_admin and dependent code to async/await. ([\#7363](https://github.com/matrix-org/synapse/issues/7363)) +- Add an `instance_name` to `RDATA` and `POSITION` replication commands. ([\#7364](https://github.com/matrix-org/synapse/issues/7364)) +- Thread through instance name to replication client. ([\#7369](https://github.com/matrix-org/synapse/issues/7369)) +- Convert synapse.server_notices to async/await. ([\#7394](https://github.com/matrix-org/synapse/issues/7394)) +- Convert synapse.notifier to async/await. ([\#7395](https://github.com/matrix-org/synapse/issues/7395)) +- Fix issues with the Python package manifest. ([\#7404](https://github.com/matrix-org/synapse/issues/7404)) +- Prevent methods in `synapse.handlers.auth` from polling the homeserver config every request. ([\#7420](https://github.com/matrix-org/synapse/issues/7420)) +- Speed up fetching device lists changes when handling `/sync` requests. ([\#7423](https://github.com/matrix-org/synapse/issues/7423)) +- Run group attestation renewal in series rather than parallel for performance. ([\#7442](https://github.com/matrix-org/synapse/issues/7442)) + + +Synapse 1.12.4 (2020-04-23) +=========================== + +No significant changes. + + +Synapse 1.12.4rc1 (2020-04-22) +============================== + +Features +-------- + +- Always send users their own device updates. ([\#7160](https://github.com/matrix-org/synapse/issues/7160)) +- Add support for handling GET requests for `account_data` on a worker. ([\#7311](https://github.com/matrix-org/synapse/issues/7311)) + + +Bugfixes +-------- + +- Fix a bug that prevented cross-signing with users on worker-mode synapses. ([\#7255](https://github.com/matrix-org/synapse/issues/7255)) +- Do not treat display names as globs in push rules. ([\#7271](https://github.com/matrix-org/synapse/issues/7271)) +- Fix a bug with cross-signing devices belonging to remote users who did not share a room with any user on the local homeserver. ([\#7289](https://github.com/matrix-org/synapse/issues/7289)) + +Synapse 1.12.3 (2020-04-03) +=========================== + +- Remove the the pin to Pillow 7.0 which was introduced in Synapse 1.12.2, and +correctly fix the issue with building the Debian packages. ([\#7212](https://github.com/matrix-org/synapse/issues/7212)) + +Synapse 1.12.2 (2020-04-02) +=========================== + +This release works around [an issue](https://github.com/matrix-org/synapse/issues/7208) with building the debian packages. + +No other significant changes since 1.12.1. + +Synapse 1.12.1 (2020-04-02) +=========================== + +No significant changes since 1.12.1rc1. + + +Synapse 1.12.1rc1 (2020-03-31) +============================== + +Bugfixes +-------- + +- Fix starting workers when federation sending not split out. ([\#7133](https://github.com/matrix-org/synapse/issues/7133)). Introduced in v1.12.0. +- Avoid importing `sqlite3` when using the postgres backend. Contributed by David Vo. ([\#7155](https://github.com/matrix-org/synapse/issues/7155)). Introduced in v1.12.0rc1. +- Fix a bug which could cause outbound federation traffic to stop working if a client uploaded an incorrect e2e device signature. ([\#7177](https://github.com/matrix-org/synapse/issues/7177)). Introduced in v1.11.0. + +Synapse 1.12.0 (2020-03-23) +=========================== + +Debian packages and Docker images are rebuilt using the latest versions of +dependency libraries, including Twisted 20.3.0. **Please see security advisory +below**. + +Potential slow database update during upgrade +--------------------------------------------- + +Synapse 1.12.0 includes a database update which is run as part of the upgrade, +and which may take some time (several hours in the case of a large +server). Synapse will not respond to HTTP requests while this update is taking +place. For imformation on seeing if you are affected, and workaround if you +are, see the [upgrade notes](docs/upgrade.md#upgrading-to-v1120). + +Security advisory +----------------- + +Synapse may be vulnerable to request-smuggling attacks when it is used with a +reverse-proxy. The vulnerabilties are fixed in Twisted 20.3.0, and are +described in +[CVE-2020-10108](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-10108) +and +[CVE-2020-10109](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-10109). +For a good introduction to this class of request-smuggling attacks, see +https://portswigger.net/research/http-desync-attacks-request-smuggling-reborn. + +We are not aware of these vulnerabilities being exploited in the wild, and +do not believe that they are exploitable with current versions of any reverse +proxies. Nevertheless, we recommend that all Synapse administrators ensure that +they have the latest versions of the Twisted library to ensure that their +installation remains secure. + +* Administrators using the [`matrix.org` Docker + image](https://hub.docker.com/r/matrixdotorg/synapse/) or the [Debian/Ubuntu + packages from + `matrix.org`](https://matrix-org.github.io/synapse/latest/setup/installation.html#matrixorg-packages) + should ensure that they have version 1.12.0 installed: these images include + Twisted 20.3.0. +* Administrators who have [installed Synapse from + source](https://matrix-org.github.io/synapse/latest/setup/installation.html#installing-from-source) + should upgrade Twisted within their virtualenv by running: + ```sh + /bin/pip install 'Twisted>=20.3.0' + ``` +* Administrators who have installed Synapse from distribution packages should + consult the information from their distributions. + +The `matrix.org` Synapse instance was not vulnerable to these vulnerabilities. + +Advance notice of change to the default `git` branch for Synapse +---------------------------------------------------------------- + +Currently, the default `git` branch for Synapse is `master`, which tracks the +latest release. + +After the release of Synapse 1.13.0, we intend to change this default to +`develop`, which is the development tip. This is more consistent with common +practice and modern `git` usage. + +Although we try to keep `develop` in a stable state, there may be occasions +where regressions creep in. Developers and distributors who have scripts which +run builds using the default branch of `Synapse` should therefore consider +pinning their scripts to `master`. + + +Synapse 1.12.0rc1 (2020-03-19) +============================== + +Features +-------- + +- Changes related to room alias management ([MSC2432](https://github.com/matrix-org/matrix-doc/pull/2432)): + - Publishing/removing a room from the room directory now requires the user to have a power level capable of modifying the canonical alias, instead of the room aliases. ([\#6965](https://github.com/matrix-org/synapse/issues/6965)) + - Validate the `alt_aliases` property of canonical alias events. ([\#6971](https://github.com/matrix-org/synapse/issues/6971)) + - Users with a power level sufficient to modify the canonical alias of a room can now delete room aliases. ([\#6986](https://github.com/matrix-org/synapse/issues/6986)) + - Implement updated authorization rules and redaction rules for aliases events, from [MSC2261](https://github.com/matrix-org/matrix-doc/pull/2261) and [MSC2432](https://github.com/matrix-org/matrix-doc/pull/2432). ([\#7037](https://github.com/matrix-org/synapse/issues/7037)) + - Stop sending m.room.aliases events during room creation and upgrade. ([\#6941](https://github.com/matrix-org/synapse/issues/6941)) + - Synapse no longer uses room alias events to calculate room names for push notifications. ([\#6966](https://github.com/matrix-org/synapse/issues/6966)) + - The room list endpoint no longer returns a list of aliases. ([\#6970](https://github.com/matrix-org/synapse/issues/6970)) + - Remove special handling of aliases events from [MSC2260](https://github.com/matrix-org/matrix-doc/pull/2260) added in v1.10.0rc1. ([\#7034](https://github.com/matrix-org/synapse/issues/7034)) +- Expose the `synctl`, `hash_password` and `generate_config` commands in the snapcraft package. Contributed by @devec0. ([\#6315](https://github.com/matrix-org/synapse/issues/6315)) +- Check that server_name is correctly set before running database updates. ([\#6982](https://github.com/matrix-org/synapse/issues/6982)) +- Break down monthly active users by `appservice_id` and emit via Prometheus. ([\#7030](https://github.com/matrix-org/synapse/issues/7030)) +- Render a configurable and comprehensible error page if something goes wrong during the SAML2 authentication process. ([\#7058](https://github.com/matrix-org/synapse/issues/7058), [\#7067](https://github.com/matrix-org/synapse/issues/7067)) +- Add an optional parameter to control whether other sessions are logged out when a user's password is modified. ([\#7085](https://github.com/matrix-org/synapse/issues/7085)) +- Add prometheus metrics for the number of active pushers. ([\#7103](https://github.com/matrix-org/synapse/issues/7103), [\#7106](https://github.com/matrix-org/synapse/issues/7106)) +- Improve performance when making HTTPS requests to sygnal, sydent, etc, by sharing the SSL context object between connections. ([\#7094](https://github.com/matrix-org/synapse/issues/7094)) + + +Bugfixes +-------- + +- When a user's profile is updated via the admin API, also generate a displayname/avatar update for that user in each room. ([\#6572](https://github.com/matrix-org/synapse/issues/6572)) +- Fix a couple of bugs in email configuration handling. ([\#6962](https://github.com/matrix-org/synapse/issues/6962)) +- Fix an issue affecting worker-based deployments where replication would stop working, necessitating a full restart, after joining a large room. ([\#6967](https://github.com/matrix-org/synapse/issues/6967)) +- Fix `duplicate key` error which was logged when rejoining a room over federation. ([\#6968](https://github.com/matrix-org/synapse/issues/6968)) +- Prevent user from setting 'deactivated' to anything other than a bool on the v2 PUT /users Admin API. ([\#6990](https://github.com/matrix-org/synapse/issues/6990)) +- Fix py35-old CI by using native tox package. ([\#7018](https://github.com/matrix-org/synapse/issues/7018)) +- Fix a bug causing `org.matrix.dummy_event` to be included in responses from `/sync`. ([\#7035](https://github.com/matrix-org/synapse/issues/7035)) +- Fix a bug that renders UTF-8 text files incorrectly when loaded from media. Contributed by @TheStranjer. ([\#7044](https://github.com/matrix-org/synapse/issues/7044)) +- Fix a bug that would cause Synapse to respond with an error about event visibility if a client tried to request the state of a room at a given token. ([\#7066](https://github.com/matrix-org/synapse/issues/7066)) +- Repair a data-corruption issue which was introduced in Synapse 1.10, and fixed in Synapse 1.11, and which could cause `/sync` to return with 404 errors about missing events and unknown rooms. ([\#7070](https://github.com/matrix-org/synapse/issues/7070)) +- Fix a bug causing account validity renewal emails to be sent even if the feature is turned off in some cases. ([\#7074](https://github.com/matrix-org/synapse/issues/7074)) + + +Improved Documentation +---------------------- + +- Updated CentOS8 install instructions. Contributed by Richard Kellner. ([\#6925](https://github.com/matrix-org/synapse/issues/6925)) +- Fix `POSTGRES_INITDB_ARGS` in the `contrib/docker/docker-compose.yml` example docker-compose configuration. ([\#6984](https://github.com/matrix-org/synapse/issues/6984)) +- Change date in [INSTALL.md](./INSTALL.md#tls-certificates) for last date of getting TLS certificates to November 2019. ([\#7015](https://github.com/matrix-org/synapse/issues/7015)) +- Document that the fallback auth endpoints must be routed to the same worker node as the register endpoints. ([\#7048](https://github.com/matrix-org/synapse/issues/7048)) + + +Deprecations and Removals +------------------------- + +- Remove the unused query_auth federation endpoint per [MSC2451](https://github.com/matrix-org/matrix-doc/pull/2451). ([\#7026](https://github.com/matrix-org/synapse/issues/7026)) + + +Internal Changes +---------------- + +- Add type hints to `logging/context.py`. ([\#6309](https://github.com/matrix-org/synapse/issues/6309)) +- Add some clarifications to `README.md` in the database schema directory. ([\#6615](https://github.com/matrix-org/synapse/issues/6615)) +- Refactoring work in preparation for changing the event redaction algorithm. ([\#6874](https://github.com/matrix-org/synapse/issues/6874), [\#6875](https://github.com/matrix-org/synapse/issues/6875), [\#6983](https://github.com/matrix-org/synapse/issues/6983), [\#7003](https://github.com/matrix-org/synapse/issues/7003)) +- Improve performance of v2 state resolution for large rooms. ([\#6952](https://github.com/matrix-org/synapse/issues/6952), [\#7095](https://github.com/matrix-org/synapse/issues/7095)) +- Reduce time spent doing GC, by freezing objects on startup. ([\#6953](https://github.com/matrix-org/synapse/issues/6953)) +- Minor perfermance fixes to `get_auth_chain_ids`. ([\#6954](https://github.com/matrix-org/synapse/issues/6954)) +- Don't record remote cross-signing keys in the `devices` table. ([\#6956](https://github.com/matrix-org/synapse/issues/6956)) +- Use flake8-comprehensions to enforce good hygiene of list/set/dict comprehensions. ([\#6957](https://github.com/matrix-org/synapse/issues/6957)) +- Merge worker apps together. ([\#6964](https://github.com/matrix-org/synapse/issues/6964), [\#7002](https://github.com/matrix-org/synapse/issues/7002), [\#7055](https://github.com/matrix-org/synapse/issues/7055), [\#7104](https://github.com/matrix-org/synapse/issues/7104)) +- Remove redundant `store_room` call from `FederationHandler._process_received_pdu`. ([\#6979](https://github.com/matrix-org/synapse/issues/6979)) +- Update warning for incorrect database collation/ctype to include link to documentation. ([\#6985](https://github.com/matrix-org/synapse/issues/6985)) +- Add some type annotations to the database storage classes. ([\#6987](https://github.com/matrix-org/synapse/issues/6987)) +- Port `synapse.handlers.presence` to async/await. ([\#6991](https://github.com/matrix-org/synapse/issues/6991), [\#7019](https://github.com/matrix-org/synapse/issues/7019)) +- Add some type annotations to the federation base & client classes. ([\#6995](https://github.com/matrix-org/synapse/issues/6995)) +- Port `synapse.rest.keys` to async/await. ([\#7020](https://github.com/matrix-org/synapse/issues/7020)) +- Add a type check to `is_verified` when processing room keys. ([\#7045](https://github.com/matrix-org/synapse/issues/7045)) +- Add type annotations and comments to the auth handler. ([\#7063](https://github.com/matrix-org/synapse/issues/7063)) + + +Synapse 1.11.1 (2020-03-03) +=========================== + +This release includes a security fix impacting installations using Single Sign-On (i.e. SAML2 or CAS) for authentication. Administrators of such installations are encouraged to upgrade as soon as possible. + +The release also includes fixes for a couple of other bugs. + +Bugfixes +-------- + +- Add a confirmation step to the SSO login flow before redirecting users to the redirect URL. ([b2bd54a2](https://github.com/matrix-org/synapse/commit/b2bd54a2e31d9a248f73fadb184ae9b4cbdb49f9), [65c73cdf](https://github.com/matrix-org/synapse/commit/65c73cdfec1876a9fec2fd2c3a74923cd146fe0b), [a0178df1](https://github.com/matrix-org/synapse/commit/a0178df10422a76fd403b82d2b2a4ed28a9a9d1e)) +- Fixed set a user as an admin with the admin API `PUT /_synapse/admin/v2/users/`. Contributed by @dklimpel. ([\#6910](https://github.com/matrix-org/synapse/issues/6910)) +- Fix bug introduced in Synapse 1.11.0 which sometimes caused errors when joining rooms over federation, with `'coroutine' object has no attribute 'event_id'`. ([\#6996](https://github.com/matrix-org/synapse/issues/6996)) + + +Synapse 1.11.0 (2020-02-21) +=========================== + +Improved Documentation +---------------------- + +- Small grammatical fixes to the ACME v1 deprecation notice. ([\#6944](https://github.com/matrix-org/synapse/issues/6944)) + + +Synapse 1.11.0rc1 (2020-02-19) +============================== + +Features +-------- + +- Admin API to add or modify threepids of user accounts. ([\#6769](https://github.com/matrix-org/synapse/issues/6769)) +- Limit the number of events that can be requested by the backfill federation API to 100. ([\#6864](https://github.com/matrix-org/synapse/issues/6864)) +- Add ability to run some group APIs on workers. ([\#6866](https://github.com/matrix-org/synapse/issues/6866)) +- Reject device display names over 100 characters in length to prevent abuse. ([\#6882](https://github.com/matrix-org/synapse/issues/6882)) +- Add ability to route federation user device queries to workers. ([\#6873](https://github.com/matrix-org/synapse/issues/6873)) +- The result of a user directory search can now be filtered via the spam checker. ([\#6888](https://github.com/matrix-org/synapse/issues/6888)) +- Implement new `GET /_matrix/client/unstable/org.matrix.msc2432/rooms/{roomId}/aliases` endpoint as per [MSC2432](https://github.com/matrix-org/matrix-doc/pull/2432). ([\#6939](https://github.com/matrix-org/synapse/issues/6939), [\#6948](https://github.com/matrix-org/synapse/issues/6948), [\#6949](https://github.com/matrix-org/synapse/issues/6949)) +- Stop sending `m.room.alias` events wheng adding / removing aliases. Check `alt_aliases` in the latest `m.room.canonical_alias` event when deleting an alias. ([\#6904](https://github.com/matrix-org/synapse/issues/6904)) +- Change the default power levels of invites, tombstones and server ACLs for new rooms. ([\#6834](https://github.com/matrix-org/synapse/issues/6834)) + +Bugfixes +-------- + +- Fixed third party event rules function `on_create_room`'s return value being ignored. ([\#6781](https://github.com/matrix-org/synapse/issues/6781)) +- Allow URL-encoded User IDs on `/_synapse/admin/v2/users/[/admin]` endpoints. Thanks to @NHAS for reporting. ([\#6825](https://github.com/matrix-org/synapse/issues/6825)) +- Fix Synapse refusing to start if `federation_certificate_verification_whitelist` option is blank. ([\#6849](https://github.com/matrix-org/synapse/issues/6849)) +- Fix errors from logging in the purge jobs related to the message retention policies support. ([\#6945](https://github.com/matrix-org/synapse/issues/6945)) +- Return a 404 instead of 200 for querying information of a non-existant user through the admin API. ([\#6901](https://github.com/matrix-org/synapse/issues/6901)) + + +Updates to the Docker image +--------------------------- + +- The deprecated "generate-config-on-the-fly" mode is no longer supported. ([\#6918](https://github.com/matrix-org/synapse/issues/6918)) + + +Improved Documentation +---------------------- + +- Add details of PR merge strategy to contributing docs. ([\#6846](https://github.com/matrix-org/synapse/issues/6846)) +- Spell out that the last event sent to a room won't be deleted by a purge. ([\#6891](https://github.com/matrix-org/synapse/issues/6891)) +- Update Synapse's documentation to warn about the deprecation of ACME v1. ([\#6905](https://github.com/matrix-org/synapse/issues/6905), [\#6907](https://github.com/matrix-org/synapse/issues/6907), [\#6909](https://github.com/matrix-org/synapse/issues/6909)) +- Add documentation for the spam checker. ([\#6906](https://github.com/matrix-org/synapse/issues/6906)) +- Fix worker docs to point `/publicised_groups` API correctly. ([\#6938](https://github.com/matrix-org/synapse/issues/6938)) +- Clean up and update docs on setting up federation. ([\#6940](https://github.com/matrix-org/synapse/issues/6940)) +- Add a warning about indentation to generated configuration files. ([\#6920](https://github.com/matrix-org/synapse/issues/6920)) +- Databases created using the compose file in contrib/docker will now always have correct encoding and locale settings. Contributed by Fridtjof Mund. ([\#6921](https://github.com/matrix-org/synapse/issues/6921)) +- Update pip install directions in readme to avoid error when using zsh. ([\#6855](https://github.com/matrix-org/synapse/issues/6855)) + + +Deprecations and Removals +------------------------- + +- Remove `m.lazy_load_members` from `unstable_features` since lazy loading is in the stable Client-Server API version r0.5.0. ([\#6877](https://github.com/matrix-org/synapse/issues/6877)) + + +Internal Changes +---------------- + +- Add type hints to `SyncHandler`. ([\#6821](https://github.com/matrix-org/synapse/issues/6821)) +- Refactoring work in preparation for changing the event redaction algorithm. ([\#6823](https://github.com/matrix-org/synapse/issues/6823), [\#6827](https://github.com/matrix-org/synapse/issues/6827), [\#6854](https://github.com/matrix-org/synapse/issues/6854), [\#6856](https://github.com/matrix-org/synapse/issues/6856), [\#6857](https://github.com/matrix-org/synapse/issues/6857), [\#6858](https://github.com/matrix-org/synapse/issues/6858)) +- Fix stacktraces when using `ObservableDeferred` and async/await. ([\#6836](https://github.com/matrix-org/synapse/issues/6836)) +- Port much of `synapse.handlers.federation` to async/await. ([\#6837](https://github.com/matrix-org/synapse/issues/6837), [\#6840](https://github.com/matrix-org/synapse/issues/6840)) +- Populate `rooms.room_version` database column at startup, rather than in a background update. ([\#6847](https://github.com/matrix-org/synapse/issues/6847)) +- Reduce amount we log at `INFO` level. ([\#6833](https://github.com/matrix-org/synapse/issues/6833), [\#6862](https://github.com/matrix-org/synapse/issues/6862)) +- Remove unused `get_room_stats_state` method. ([\#6869](https://github.com/matrix-org/synapse/issues/6869)) +- Add typing to `synapse.federation.sender` and port to async/await. ([\#6871](https://github.com/matrix-org/synapse/issues/6871)) +- Refactor `_EventInternalMetadata` object to improve type safety. ([\#6872](https://github.com/matrix-org/synapse/issues/6872)) +- Add an additional entry to the SyTest blacklist for worker mode. ([\#6883](https://github.com/matrix-org/synapse/issues/6883)) +- Fix the use of sed in the linting scripts when using BSD sed. ([\#6887](https://github.com/matrix-org/synapse/issues/6887)) +- Add type hints to the spam checker module. ([\#6915](https://github.com/matrix-org/synapse/issues/6915)) +- Convert the directory handler tests to use HomeserverTestCase. ([\#6919](https://github.com/matrix-org/synapse/issues/6919)) +- Increase DB/CPU perf of `_is_server_still_joined` check. ([\#6936](https://github.com/matrix-org/synapse/issues/6936)) +- Tiny optimisation for incoming HTTP request dispatch. ([\#6950](https://github.com/matrix-org/synapse/issues/6950)) + + +Synapse 1.10.1 (2020-02-17) +=========================== + +Bugfixes +-------- + +- Fix a bug introduced in Synapse 1.10.0 which would cause room state to be cleared in the database if Synapse was upgraded direct from 1.2.1 or earlier to 1.10.0. ([\#6924](https://github.com/matrix-org/synapse/issues/6924)) + + +Synapse 1.10.0 (2020-02-12) +=========================== + +**WARNING to client developers**: As of this release Synapse validates `client_secret` parameters in the Client-Server API as per the spec. See [\#6766](https://github.com/matrix-org/synapse/issues/6766) for details. + +Updates to the Docker image +--------------------------- + +- Update the docker images to Alpine Linux 3.11. ([\#6897](https://github.com/matrix-org/synapse/issues/6897)) + + +Synapse 1.10.0rc5 (2020-02-11) +============================== + +Bugfixes +-------- + +- Fix the filtering introduced in 1.10.0rc3 to also apply to the state blocks returned by `/sync`. ([\#6884](https://github.com/matrix-org/synapse/issues/6884)) + +Synapse 1.10.0rc4 (2020-02-11) +============================== + +This release candidate was built incorrectly and is superceded by 1.10.0rc5. + +Synapse 1.10.0rc3 (2020-02-10) +============================== + +Features +-------- + +- Filter out `m.room.aliases` from the CS API to mitigate abuse while a better solution is specced. ([\#6878](https://github.com/matrix-org/synapse/issues/6878)) + + +Internal Changes +---------------- + +- Fix continuous integration failures with old versions of `pip`, which were introduced by a release of the `zipp` library. ([\#6880](https://github.com/matrix-org/synapse/issues/6880)) + + +Synapse 1.10.0rc2 (2020-02-06) +============================== + +Bugfixes +-------- + +- Fix an issue with cross-signing where device signatures were not sent to remote servers. ([\#6844](https://github.com/matrix-org/synapse/issues/6844)) +- Fix to the unknown remote device detection which was introduced in 1.10.rc1. ([\#6848](https://github.com/matrix-org/synapse/issues/6848)) + + +Internal Changes +---------------- + +- Detect unexpected sender keys on remote encrypted events and resync device lists. ([\#6850](https://github.com/matrix-org/synapse/issues/6850)) + + +Synapse 1.10.0rc1 (2020-01-31) +============================== + +Features +-------- + +- Add experimental support for updated authorization rules for aliases events, from [MSC2260](https://github.com/matrix-org/matrix-doc/pull/2260). ([\#6787](https://github.com/matrix-org/synapse/issues/6787), [\#6790](https://github.com/matrix-org/synapse/issues/6790), [\#6794](https://github.com/matrix-org/synapse/issues/6794)) + + +Bugfixes +-------- + +- Warn if postgres database has a non-C locale, as that can cause issues when upgrading locales (e.g. due to upgrading OS). ([\#6734](https://github.com/matrix-org/synapse/issues/6734)) +- Minor fixes to `PUT /_synapse/admin/v2/users` admin api. ([\#6761](https://github.com/matrix-org/synapse/issues/6761)) +- Validate `client_secret` parameter using the regex provided by the Client-Server API, temporarily allowing `:` characters for older clients. The `:` character will be removed in a future release. ([\#6767](https://github.com/matrix-org/synapse/issues/6767)) +- Fix persisting redaction events that have been redacted (or otherwise don't have a redacts key). ([\#6771](https://github.com/matrix-org/synapse/issues/6771)) +- Fix outbound federation request metrics. ([\#6795](https://github.com/matrix-org/synapse/issues/6795)) +- Fix bug where querying a remote user's device keys that weren't cached resulted in only returning a single device. ([\#6796](https://github.com/matrix-org/synapse/issues/6796)) +- Fix race in federation sender worker that delayed sending of device updates. ([\#6799](https://github.com/matrix-org/synapse/issues/6799), [\#6800](https://github.com/matrix-org/synapse/issues/6800)) +- Fix bug where Synapse didn't invalidate cache of remote users' devices when Synapse left a room. ([\#6801](https://github.com/matrix-org/synapse/issues/6801)) +- Fix waking up other workers when remote server is detected to have come back online. ([\#6811](https://github.com/matrix-org/synapse/issues/6811)) + + +Improved Documentation +---------------------- + +- Clarify documentation related to `user_dir` and `federation_reader` workers. ([\#6775](https://github.com/matrix-org/synapse/issues/6775)) + + +Internal Changes +---------------- + +- Record room versions in the `rooms` table. ([\#6729](https://github.com/matrix-org/synapse/issues/6729), [\#6788](https://github.com/matrix-org/synapse/issues/6788), [\#6810](https://github.com/matrix-org/synapse/issues/6810)) +- Propagate cache invalidates from workers to other workers. ([\#6748](https://github.com/matrix-org/synapse/issues/6748)) +- Remove some unnecessary admin handler abstraction methods. ([\#6751](https://github.com/matrix-org/synapse/issues/6751)) +- Add some debugging for media storage providers. ([\#6757](https://github.com/matrix-org/synapse/issues/6757)) +- Detect unknown remote devices and mark cache as stale. ([\#6776](https://github.com/matrix-org/synapse/issues/6776), [\#6819](https://github.com/matrix-org/synapse/issues/6819)) +- Attempt to resync remote users' devices when detected as stale. ([\#6786](https://github.com/matrix-org/synapse/issues/6786)) +- Delete current state from the database when server leaves a room. ([\#6792](https://github.com/matrix-org/synapse/issues/6792)) +- When a client asks for a remote user's device keys check if the local cache for that user has been marked as potentially stale. ([\#6797](https://github.com/matrix-org/synapse/issues/6797)) +- Add background update to clean out left rooms from current state. ([\#6802](https://github.com/matrix-org/synapse/issues/6802), [\#6816](https://github.com/matrix-org/synapse/issues/6816)) +- Refactoring work in preparation for changing the event redaction algorithm. ([\#6803](https://github.com/matrix-org/synapse/issues/6803), [\#6805](https://github.com/matrix-org/synapse/issues/6805), [\#6806](https://github.com/matrix-org/synapse/issues/6806), [\#6807](https://github.com/matrix-org/synapse/issues/6807), [\#6820](https://github.com/matrix-org/synapse/issues/6820)) + + +Synapse 1.9.1 (2020-01-28) +========================== + +Bugfixes +-------- + +- Fix bug where setting `mau_limit_reserved_threepids` config would cause Synapse to refuse to start. ([\#6793](https://github.com/matrix-org/synapse/issues/6793)) + + +Synapse 1.9.0 (2020-01-23) +========================== + +**WARNING**: As of this release, Synapse no longer supports versions of SQLite before 3.11, and will refuse to start when configured to use an older version. Administrators are recommended to migrate their database to Postgres (see instructions [here](docs/postgres.md)). + +If your Synapse deployment uses workers, note that the reverse-proxy configurations for the `synapse.app.media_repository`, `synapse.app.federation_reader` and `synapse.app.event_creator` workers have changed, with the addition of a few paths (see the updated configurations [here](docs/workers.md#available-worker-applications)). Existing configurations will continue to work. + + +Improved Documentation +---------------------- + +- Fix endpoint documentation for the List Rooms admin API. ([\#6770](https://github.com/matrix-org/synapse/issues/6770)) + + +Synapse 1.9.0rc1 (2020-01-22) +============================= + +Features +-------- + +- Allow admin to create or modify a user. Contributed by Awesome Technologies Innovationslabor GmbH. ([\#5742](https://github.com/matrix-org/synapse/issues/5742)) +- Add new quarantine media admin APIs to quarantine by media ID or by user who uploaded the media. ([\#6681](https://github.com/matrix-org/synapse/issues/6681), [\#6756](https://github.com/matrix-org/synapse/issues/6756)) +- Add `org.matrix.e2e_cross_signing` to `unstable_features` in `/versions` as per [MSC1756](https://github.com/matrix-org/matrix-doc/pull/1756). ([\#6712](https://github.com/matrix-org/synapse/issues/6712)) +- Add a new admin API to list and filter rooms on the server. ([\#6720](https://github.com/matrix-org/synapse/issues/6720)) + + +Bugfixes +-------- + +- Correctly proxy HTTP errors due to API calls to remote group servers. ([\#6654](https://github.com/matrix-org/synapse/issues/6654)) +- Fix media repo admin APIs when using a media worker. ([\#6664](https://github.com/matrix-org/synapse/issues/6664)) +- Fix "CRITICAL" errors being logged when a request is received for a uri containing non-ascii characters. ([\#6682](https://github.com/matrix-org/synapse/issues/6682)) +- Fix a bug where we would assign a numeric user ID if somebody tried registering with an empty username. ([\#6690](https://github.com/matrix-org/synapse/issues/6690)) +- Fix `purge_room` admin API. ([\#6711](https://github.com/matrix-org/synapse/issues/6711)) +- Fix a bug causing Synapse to not always purge quiet rooms with a low `max_lifetime` in their message retention policies when running the automated purge jobs. ([\#6714](https://github.com/matrix-org/synapse/issues/6714)) +- Fix the `synapse_port_db` not correctly running background updates. Thanks @tadzik for reporting. ([\#6718](https://github.com/matrix-org/synapse/issues/6718)) +- Fix changing password via user admin API. ([\#6730](https://github.com/matrix-org/synapse/issues/6730)) +- Fix `/events/:event_id` deprecated API. ([\#6731](https://github.com/matrix-org/synapse/issues/6731)) +- Fix monthly active user limiting support for worker mode, fixes [#4639](https://github.com/matrix-org/synapse/issues/4639). ([\#6742](https://github.com/matrix-org/synapse/issues/6742)) +- Fix bug when setting `account_validity` to an empty block in the config. Thanks to @Sorunome for reporting. ([\#6747](https://github.com/matrix-org/synapse/issues/6747)) +- Fix `AttributeError: 'NoneType' object has no attribute 'get'` in `hash_password` when configuration has an empty `password_config`. Contributed by @ivilata. ([\#6753](https://github.com/matrix-org/synapse/issues/6753)) +- Fix the `docker-compose.yaml` overriding the entire `/etc` folder of the container. Contributed by Fabian Meyer. ([\#6656](https://github.com/matrix-org/synapse/issues/6656)) + + +Improved Documentation +---------------------- + +- Fix a typo in the configuration example for purge jobs in the sample configuration file. ([\#6621](https://github.com/matrix-org/synapse/issues/6621)) +- Add complete documentation of the message retention policies support. ([\#6624](https://github.com/matrix-org/synapse/issues/6624), [\#6665](https://github.com/matrix-org/synapse/issues/6665)) +- Add some helpful tips about changelog entries to the GitHub pull request template. ([\#6663](https://github.com/matrix-org/synapse/issues/6663)) +- Clarify the `account_validity` and `email` sections of the sample configuration. ([\#6685](https://github.com/matrix-org/synapse/issues/6685)) +- Add more endpoints to the documentation for Synapse workers. ([\#6698](https://github.com/matrix-org/synapse/issues/6698)) + + +Deprecations and Removals +------------------------- + +- Synapse no longer supports versions of SQLite before 3.11, and will refuse to start when configured to use an older version. Administrators are recommended to migrate their database to Postgres (see instructions [here](docs/postgres.md)). ([\#6675](https://github.com/matrix-org/synapse/issues/6675)) + + +Internal Changes +---------------- + +- Add `local_current_membership` table for tracking local user membership state in rooms. ([\#6655](https://github.com/matrix-org/synapse/issues/6655), [\#6728](https://github.com/matrix-org/synapse/issues/6728)) +- Port `synapse.replication.tcp` to async/await. ([\#6666](https://github.com/matrix-org/synapse/issues/6666)) +- Fixup `synapse.replication` to pass mypy checks. ([\#6667](https://github.com/matrix-org/synapse/issues/6667)) +- Allow `additional_resources` to implement `IResource` directly. ([\#6686](https://github.com/matrix-org/synapse/issues/6686)) +- Allow REST endpoint implementations to raise a `RedirectException`, which will redirect the user's browser to a given location. ([\#6687](https://github.com/matrix-org/synapse/issues/6687)) +- Updates and extensions to the module API. ([\#6688](https://github.com/matrix-org/synapse/issues/6688)) +- Updates to the SAML mapping provider API. ([\#6689](https://github.com/matrix-org/synapse/issues/6689), [\#6723](https://github.com/matrix-org/synapse/issues/6723)) +- Remove redundant `RegistrationError` class. ([\#6691](https://github.com/matrix-org/synapse/issues/6691)) +- Don't block processing of incoming EDUs behind processing PDUs in the same transaction. ([\#6697](https://github.com/matrix-org/synapse/issues/6697)) +- Remove duplicate check for the `session` query parameter on the `/auth/xxx/fallback/web` Client-Server endpoint. ([\#6702](https://github.com/matrix-org/synapse/issues/6702)) +- Attempt to retry sending a transaction when we detect a remote server has come back online, rather than waiting for a transaction to be triggered by new data. ([\#6706](https://github.com/matrix-org/synapse/issues/6706)) +- Add `StateMap` type alias to simplify types. ([\#6715](https://github.com/matrix-org/synapse/issues/6715)) +- Add a `DeltaState` to track changes to be made to current state during event persistence. ([\#6716](https://github.com/matrix-org/synapse/issues/6716)) +- Add more logging around message retention policies support. ([\#6717](https://github.com/matrix-org/synapse/issues/6717)) +- When processing a SAML response, log the assertions for easier configuration. ([\#6724](https://github.com/matrix-org/synapse/issues/6724)) +- Fixup `synapse.rest` to pass mypy. ([\#6732](https://github.com/matrix-org/synapse/issues/6732), [\#6764](https://github.com/matrix-org/synapse/issues/6764)) +- Fixup `synapse.api` to pass mypy. ([\#6733](https://github.com/matrix-org/synapse/issues/6733)) +- Allow streaming cache 'invalidate all' to workers. ([\#6749](https://github.com/matrix-org/synapse/issues/6749)) +- Remove unused CI docker compose files. ([\#6754](https://github.com/matrix-org/synapse/issues/6754)) + + +Synapse 1.8.0 (2020-01-09) +========================== + +**WARNING**: As of this release Synapse will refuse to start if the `log_file` config option is specified. Support for the option was removed in v1.3.0. + + +Bugfixes +-------- + +- Fix `GET` request on `/_synapse/admin/v2/users` endpoint. Contributed by Awesome Technologies Innovationslabor GmbH. ([\#6563](https://github.com/matrix-org/synapse/issues/6563)) +- Fix incorrect signing of responses from the key server implementation. ([\#6657](https://github.com/matrix-org/synapse/issues/6657)) + + +Synapse 1.8.0rc1 (2020-01-07) +============================= + +Features +-------- + +- Add v2 APIs for the `send_join` and `send_leave` federation endpoints (as described in [MSC1802](https://github.com/matrix-org/matrix-doc/pull/1802)). ([\#6349](https://github.com/matrix-org/synapse/issues/6349)) +- Add a develop script to generate full SQL schemas. ([\#6394](https://github.com/matrix-org/synapse/issues/6394)) +- Add custom SAML username mapping functionality through an external provider plugin. ([\#6411](https://github.com/matrix-org/synapse/issues/6411)) +- Automatically delete empty groups/communities. ([\#6453](https://github.com/matrix-org/synapse/issues/6453)) +- Add option `limit_profile_requests_to_users_who_share_rooms` to prevent requirement of a local user sharing a room with another user to query their profile information. ([\#6523](https://github.com/matrix-org/synapse/issues/6523)) +- Add an `export_signing_key` script to extract the public part of signing keys when rotating them. ([\#6546](https://github.com/matrix-org/synapse/issues/6546)) +- Add experimental config option to specify multiple databases. ([\#6580](https://github.com/matrix-org/synapse/issues/6580)) +- Raise an error if someone tries to use the `log_file` config option. ([\#6626](https://github.com/matrix-org/synapse/issues/6626)) + + +Bugfixes +-------- + +- Prevent redacted events from being returned during message search. ([\#6377](https://github.com/matrix-org/synapse/issues/6377), [\#6522](https://github.com/matrix-org/synapse/issues/6522)) +- Prevent error on trying to search a upgraded room when the server is not in the predecessor room. ([\#6385](https://github.com/matrix-org/synapse/issues/6385)) +- Improve performance of looking up cross-signing keys. ([\#6486](https://github.com/matrix-org/synapse/issues/6486)) +- Fix race which occasionally caused deleted devices to reappear. ([\#6514](https://github.com/matrix-org/synapse/issues/6514)) +- Fix missing row in `device_max_stream_id` that could cause unable to decrypt errors after server restart. ([\#6555](https://github.com/matrix-org/synapse/issues/6555)) +- Fix a bug which meant that we did not send systemd notifications on startup if acme was enabled. ([\#6571](https://github.com/matrix-org/synapse/issues/6571)) +- Fix exception when fetching the `matrix.org:ed25519:auto` key. ([\#6625](https://github.com/matrix-org/synapse/issues/6625)) +- Fix bug where a moderator upgraded a room and became an admin in the new room. ([\#6633](https://github.com/matrix-org/synapse/issues/6633)) +- Fix an error which was thrown by the `PresenceHandler` `_on_shutdown` handler. ([\#6640](https://github.com/matrix-org/synapse/issues/6640)) +- Fix exceptions in the synchrotron worker log when events are rejected. ([\#6645](https://github.com/matrix-org/synapse/issues/6645)) +- Ensure that upgraded rooms are removed from the directory. ([\#6648](https://github.com/matrix-org/synapse/issues/6648)) +- Fix a bug causing Synapse not to fetch missing events when it believes it has every event in the room. ([\#6652](https://github.com/matrix-org/synapse/issues/6652)) + + +Improved Documentation +---------------------- + +- Document the Room Shutdown Admin API. ([\#6541](https://github.com/matrix-org/synapse/issues/6541)) +- Reword sections of [docs/federate.md](docs/federate.md) that explained delegation at time of Synapse 1.0 transition. ([\#6601](https://github.com/matrix-org/synapse/issues/6601)) +- Added the section 'Configuration' in [docs/turn-howto.md](docs/turn-howto.md). ([\#6614](https://github.com/matrix-org/synapse/issues/6614)) + + +Deprecations and Removals +------------------------- + +- Remove redundant code from event authorisation implementation. ([\#6502](https://github.com/matrix-org/synapse/issues/6502)) +- Remove unused, undocumented `/_matrix/content` API. ([\#6628](https://github.com/matrix-org/synapse/issues/6628)) + + +Internal Changes +---------------- + +- Add *experimental* support for multiple physical databases and split out state storage to separate data store. ([\#6245](https://github.com/matrix-org/synapse/issues/6245), [\#6510](https://github.com/matrix-org/synapse/issues/6510), [\#6511](https://github.com/matrix-org/synapse/issues/6511), [\#6513](https://github.com/matrix-org/synapse/issues/6513), [\#6564](https://github.com/matrix-org/synapse/issues/6564), [\#6565](https://github.com/matrix-org/synapse/issues/6565)) +- Port sections of code base to async/await. ([\#6496](https://github.com/matrix-org/synapse/issues/6496), [\#6504](https://github.com/matrix-org/synapse/issues/6504), [\#6505](https://github.com/matrix-org/synapse/issues/6505), [\#6517](https://github.com/matrix-org/synapse/issues/6517), [\#6559](https://github.com/matrix-org/synapse/issues/6559), [\#6647](https://github.com/matrix-org/synapse/issues/6647), [\#6653](https://github.com/matrix-org/synapse/issues/6653)) +- Remove `SnapshotCache` in favour of `ResponseCache`. ([\#6506](https://github.com/matrix-org/synapse/issues/6506)) +- Silence mypy errors for files outside those specified. ([\#6512](https://github.com/matrix-org/synapse/issues/6512)) +- Clean up some logging when handling incoming events over federation. ([\#6515](https://github.com/matrix-org/synapse/issues/6515)) +- Test more folders against mypy. ([\#6534](https://github.com/matrix-org/synapse/issues/6534)) +- Update `mypy` to new version. ([\#6537](https://github.com/matrix-org/synapse/issues/6537)) +- Adjust the sytest blacklist for worker mode. ([\#6538](https://github.com/matrix-org/synapse/issues/6538)) +- Remove unused `get_pagination_rows` methods from `EventSource` classes. ([\#6557](https://github.com/matrix-org/synapse/issues/6557)) +- Clean up logs from the push notifier at startup. ([\#6558](https://github.com/matrix-org/synapse/issues/6558)) +- Improve diagnostics on database upgrade failure. ([\#6570](https://github.com/matrix-org/synapse/issues/6570)) +- Reduce the reconnect time when worker replication fails, to make it easier to catch up. ([\#6617](https://github.com/matrix-org/synapse/issues/6617)) +- Simplify http handling by removing redundant `SynapseRequestFactory`. ([\#6619](https://github.com/matrix-org/synapse/issues/6619)) +- Add a workaround for synapse raising exceptions when fetching the notary's own key from the notary. ([\#6620](https://github.com/matrix-org/synapse/issues/6620)) +- Automate generation of the sample log config. ([\#6627](https://github.com/matrix-org/synapse/issues/6627)) +- Simplify event creation code by removing redundant queries on the `event_reference_hashes` table. ([\#6629](https://github.com/matrix-org/synapse/issues/6629)) +- Fix errors when `frozen_dicts` are enabled. ([\#6642](https://github.com/matrix-org/synapse/issues/6642)) + + +**Changelogs for older versions can be found [here](CHANGES-2019.md).** diff --git a/docs/changelogs/CHANGES-2021.md b/docs/changelogs/CHANGES-2021.md new file mode 100644 index 0000000000..8e349504d5 --- /dev/null +++ b/docs/changelogs/CHANGES-2021.md @@ -0,0 +1,2573 @@ + +Synapse 1.49.2 (2021-12-21) +=========================== + +This release fixes a regression introduced in Synapse 1.49.0 which could cause `/sync` requests to take significantly longer. This would particularly affect "initial" syncs for users participating in a large number of rooms, and in extreme cases, could make it impossible for such users to log in on a new client. + +**Note:** in line with our [deprecation policy](https://matrix-org.github.io/synapse/latest/deprecation_policy.html) for platform dependencies, this will be the last release to support Python 3.6 and PostgreSQL 9.6, both of which have now reached upstream end-of-life. Synapse will require Python 3.7+ and PostgreSQL 10+. + +**Note:** We will also stop producing packages for Ubuntu 18.04 (Bionic Beaver) after this release, as it uses Python 3.6. + +Bugfixes +-------- + +- Fix a performance regression in `/sync` handling, introduced in 1.49.0. ([\#11583](https://github.com/matrix-org/synapse/issues/11583)) + +Internal Changes +---------------- + +- Work around a build problem on Debian Buster. ([\#11625](https://github.com/matrix-org/synapse/issues/11625)) + + +Synapse 1.49.1 (2021-12-21) +=========================== + +Not released due to problems building the debian packages. + + +Synapse 1.49.0 (2021-12-14) +=========================== + +No significant changes since version 1.49.0rc1. + + +Support for Ubuntu 21.04 ends next month on the 20th of January +--------------------------------------------------------------- + +For users of Ubuntu 21.04 (Hirsute Hippo), please be aware that [upstream support for this version of Ubuntu will end next month][Ubuntu2104EOL]. +We will stop producing packages for Ubuntu 21.04 after upstream support ends. + +[Ubuntu2104EOL]: https://lists.ubuntu.com/archives/ubuntu-announce/2021-December/000275.html + + +The wiki has been migrated to the documentation website +------------------------------------------------------- + +We've decided to move the existing, somewhat stagnant pages from the GitHub wiki +to the [documentation website](https://matrix-org.github.io/synapse/latest/). + +This was done for two reasons. The first was to ensure that changes are checked by +multiple authors before being committed (everyone makes mistakes!) and the second +was visibility of the documentation. Not everyone knows that Synapse has some very +useful information hidden away in its GitHub wiki pages. Bringing them to the +documentation website should help with visibility, as well as keep all Synapse documentation +in one, easily-searchable location. + +Note that contributions to the documentation website happen through [GitHub pull +requests](https://github.com/matrix-org/synapse/pulls). Please visit [#synapse-dev:matrix.org](https://matrix.to/#/#synapse-dev:matrix.org) +if you need help with the process! + + +Synapse 1.49.0rc1 (2021-12-07) +============================== + +Features +-------- + +- Add [MSC3030](https://github.com/matrix-org/matrix-doc/pull/3030) experimental client and federation API endpoints to get the closest event to a given timestamp. ([\#9445](https://github.com/matrix-org/synapse/issues/9445)) +- Include bundled relation aggregations during a limited `/sync` request and `/relations` request, per [MSC2675](https://github.com/matrix-org/matrix-doc/pull/2675). ([\#11284](https://github.com/matrix-org/synapse/issues/11284), [\#11478](https://github.com/matrix-org/synapse/issues/11478)) +- Add plugin support for controlling database background updates. ([\#11306](https://github.com/matrix-org/synapse/issues/11306), [\#11475](https://github.com/matrix-org/synapse/issues/11475), [\#11479](https://github.com/matrix-org/synapse/issues/11479)) +- Support the stable API endpoints for [MSC2946](https://github.com/matrix-org/matrix-doc/pull/2946): the room `/hierarchy` endpoint. ([\#11329](https://github.com/matrix-org/synapse/issues/11329)) +- Add admin API to get some information about federation status with remote servers. ([\#11407](https://github.com/matrix-org/synapse/issues/11407)) +- Support expiry of refresh tokens and expiry of the overall session when refresh tokens are in use. ([\#11425](https://github.com/matrix-org/synapse/issues/11425)) +- Stabilise support for [MSC2918](https://github.com/matrix-org/matrix-doc/blob/main/proposals/2918-refreshtokens.md#msc2918-refresh-tokens) refresh tokens as they have now been merged into the Matrix specification. ([\#11435](https://github.com/matrix-org/synapse/issues/11435), [\#11522](https://github.com/matrix-org/synapse/issues/11522)) +- Update [MSC2918 refresh token](https://github.com/matrix-org/matrix-doc/blob/main/proposals/2918-refreshtokens.md#msc2918-refresh-tokens) support to confirm with the latest revision: accept the `refresh_tokens` parameter in the request body rather than in the URL parameters. ([\#11430](https://github.com/matrix-org/synapse/issues/11430)) +- Support configuring the lifetime of non-refreshable access tokens separately to refreshable access tokens. ([\#11445](https://github.com/matrix-org/synapse/issues/11445)) +- Expose `synapse_homeserver` and `synapse_worker` commands as entry points to run Synapse's main process and worker processes, respectively. Contributed by @Ma27. ([\#11449](https://github.com/matrix-org/synapse/issues/11449)) +- `synctl stop` will now wait for Synapse to exit before returning. ([\#11459](https://github.com/matrix-org/synapse/issues/11459), [\#11490](https://github.com/matrix-org/synapse/issues/11490)) +- Extend the "delete room" admin api to work correctly on rooms which have previously been partially deleted. ([\#11523](https://github.com/matrix-org/synapse/issues/11523)) +- Add support for the `/_matrix/client/v3/login/sso/redirect/{idpId}` API from Matrix v1.1. This endpoint was overlooked when support for v3 endpoints was added in Synapse 1.48.0rc1. ([\#11451](https://github.com/matrix-org/synapse/issues/11451)) + + +Bugfixes +-------- + +- Fix using [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) batch sending in combination with event persistence workers. Contributed by @tulir at Beeper. ([\#11220](https://github.com/matrix-org/synapse/issues/11220)) +- Fix a long-standing bug where all requests that read events from the database could get stuck as a result of losing the database connection, properly this time. Also fix a race condition introduced in the previous insufficient fix in Synapse 1.47.0. ([\#11376](https://github.com/matrix-org/synapse/issues/11376)) +- The `/send_join` response now includes the stable `event` field instead of the unstable field from [MSC3083](https://github.com/matrix-org/matrix-doc/pull/3083). ([\#11413](https://github.com/matrix-org/synapse/issues/11413)) +- Fix a bug introduced in Synapse 1.47.0 where `send_join` could fail due to an outdated `ijson` version. ([\#11439](https://github.com/matrix-org/synapse/issues/11439), [\#11441](https://github.com/matrix-org/synapse/issues/11441), [\#11460](https://github.com/matrix-org/synapse/issues/11460)) +- Fix a bug introduced in Synapse 1.36.0 which could cause problems fetching event-signing keys from trusted key servers. ([\#11440](https://github.com/matrix-org/synapse/issues/11440)) +- Fix a bug introduced in Synapse 1.47.1 where the media repository would fail to work if the media store path contained any symbolic links. ([\#11446](https://github.com/matrix-org/synapse/issues/11446)) +- Fix an `LruCache` corruption bug, introduced in Synapse 1.38.0, that would cause certain requests to fail until the next Synapse restart. ([\#11454](https://github.com/matrix-org/synapse/issues/11454)) +- Fix a long-standing bug where invites from ignored users were included in incremental syncs. ([\#11511](https://github.com/matrix-org/synapse/issues/11511)) +- Fix a regression in Synapse 1.48.0 where presence workers would not clear their presence updates over replication on shutdown. ([\#11518](https://github.com/matrix-org/synapse/issues/11518)) +- Fix a regression in Synapse 1.48.0 where the module API's `looping_background_call` method would spam errors to the logs when given a non-async function. ([\#11524](https://github.com/matrix-org/synapse/issues/11524)) + + +Updates to the Docker image +--------------------------- + +- Update `Dockerfile-workers` to healthcheck all workers in the container. ([\#11429](https://github.com/matrix-org/synapse/issues/11429)) + + +Improved Documentation +---------------------- + +- Update the media repository documentation. ([\#11415](https://github.com/matrix-org/synapse/issues/11415)) +- Update section about backward extremities in the room DAG concepts doc to correct the misconception about backward extremities indicating whether we have fetched an events' `prev_events`. ([\#11469](https://github.com/matrix-org/synapse/issues/11469)) + + +Internal Changes +---------------- + +- Add `Final` annotation to string constants in `synapse.api.constants` so that they get typed as `Literal`s. ([\#11356](https://github.com/matrix-org/synapse/issues/11356)) +- Add a check to ensure that users cannot start the Synapse master process when `worker_app` is set. ([\#11416](https://github.com/matrix-org/synapse/issues/11416)) +- Add a note about postgres memory management and hugepages to postgres doc. ([\#11467](https://github.com/matrix-org/synapse/issues/11467)) +- Add missing type hints to `synapse.config` module. ([\#11465](https://github.com/matrix-org/synapse/issues/11465)) +- Add missing type hints to `synapse.federation`. ([\#11483](https://github.com/matrix-org/synapse/issues/11483)) +- Add type annotations to `tests.storage.test_appservice`. ([\#11488](https://github.com/matrix-org/synapse/issues/11488), [\#11492](https://github.com/matrix-org/synapse/issues/11492)) +- Add type annotations to some of the configuration surrounding refresh tokens. ([\#11428](https://github.com/matrix-org/synapse/issues/11428)) +- Add type hints to `synapse/tests/rest/admin`. ([\#11501](https://github.com/matrix-org/synapse/issues/11501)) +- Add type hints to storage classes. ([\#11411](https://github.com/matrix-org/synapse/issues/11411)) +- Add wiki pages to documentation website. ([\#11402](https://github.com/matrix-org/synapse/issues/11402)) +- Clean up `tests.storage.test_main` to remove use of legacy code. ([\#11493](https://github.com/matrix-org/synapse/issues/11493)) +- Clean up `tests.test_visibility` to remove legacy code. ([\#11495](https://github.com/matrix-org/synapse/issues/11495)) +- Convert status codes to `HTTPStatus` in `synapse.rest.admin`. ([\#11452](https://github.com/matrix-org/synapse/issues/11452), [\#11455](https://github.com/matrix-org/synapse/issues/11455)) +- Extend the `scripts-dev/sign_json` script to support signing events. ([\#11486](https://github.com/matrix-org/synapse/issues/11486)) +- Improve internal types in push code. ([\#11409](https://github.com/matrix-org/synapse/issues/11409)) +- Improve type annotations in `synapse.module_api`. ([\#11029](https://github.com/matrix-org/synapse/issues/11029)) +- Improve type hints for `LruCache`. ([\#11453](https://github.com/matrix-org/synapse/issues/11453)) +- Preparation for database schema simplifications: disambiguate queries on `state_key`. ([\#11497](https://github.com/matrix-org/synapse/issues/11497)) +- Refactor `backfilled` into specific behavior function arguments (`_persist_events_and_state_updates` and downstream calls). ([\#11417](https://github.com/matrix-org/synapse/issues/11417)) +- Refactor `get_version_string` to fix-up types and duplicated code. ([\#11468](https://github.com/matrix-org/synapse/issues/11468)) +- Refactor various parts of the `/sync` handler. ([\#11494](https://github.com/matrix-org/synapse/issues/11494), [\#11515](https://github.com/matrix-org/synapse/issues/11515)) +- Remove unnecessary `json.dumps` from `tests.rest.admin`. ([\#11461](https://github.com/matrix-org/synapse/issues/11461)) +- Save the OpenID Connect session ID on login. ([\#11482](https://github.com/matrix-org/synapse/issues/11482)) +- Update and clean up recently ported documentation pages. ([\#11466](https://github.com/matrix-org/synapse/issues/11466)) + + +Synapse 1.48.0 (2021-11-30) +=========================== + +This release removes support for the long-deprecated `trust_identity_server_for_password_resets` configuration flag. + +This release also fixes some performance issues with some background database updates introduced in Synapse 1.47.0. + +No significant changes since 1.48.0rc1. + +Synapse 1.48.0rc1 (2021-11-25) +============================== + +Features +-------- + +- Experimental support for the thread relation defined in [MSC3440](https://github.com/matrix-org/matrix-doc/pull/3440). ([\#11161](https://github.com/matrix-org/synapse/issues/11161)) +- Support filtering by relation senders & types per [MSC3440](https://github.com/matrix-org/matrix-doc/pull/3440). ([\#11236](https://github.com/matrix-org/synapse/issues/11236)) +- Add support for the `/_matrix/client/v3` and `/_matrix/media/v3` APIs from Matrix v1.1. ([\#11318](https://github.com/matrix-org/synapse/issues/11318), [\#11371](https://github.com/matrix-org/synapse/issues/11371)) +- Support the stable version of [MSC2778](https://github.com/matrix-org/matrix-doc/pull/2778): the `m.login.application_service` login type. Contributed by @tulir. ([\#11335](https://github.com/matrix-org/synapse/issues/11335)) +- Add a new version of delete room admin API `DELETE /_synapse/admin/v2/rooms/` to run it in the background. Contributed by @dklimpel. ([\#11223](https://github.com/matrix-org/synapse/issues/11223)) +- Allow the admin [Delete Room API](https://matrix-org.github.io/synapse/latest/admin_api/rooms.html#delete-room-api) to block a room without the need to join it. ([\#11228](https://github.com/matrix-org/synapse/issues/11228)) +- Add an admin API to un-shadow-ban a user. ([\#11347](https://github.com/matrix-org/synapse/issues/11347)) +- Add an admin API to run background database schema updates. ([\#11352](https://github.com/matrix-org/synapse/issues/11352)) +- Add an admin API for blocking a room. ([\#11324](https://github.com/matrix-org/synapse/issues/11324)) +- Update the JWT login type to support custom a `sub` claim. ([\#11361](https://github.com/matrix-org/synapse/issues/11361)) +- Store and allow querying of arbitrary event relations. ([\#11391](https://github.com/matrix-org/synapse/issues/11391)) + + +Bugfixes +-------- + +- Fix a long-standing bug wherein display names or avatar URLs containing null bytes cause an internal server error when stored in the DB. ([\#11230](https://github.com/matrix-org/synapse/issues/11230)) +- Prevent [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) historical state events from being pushed to an application service via `/transactions`. ([\#11265](https://github.com/matrix-org/synapse/issues/11265)) +- Fix a long-standing bug where uploading extremely thin images (e.g. 1000x1) would fail. Contributed by @Neeeflix. ([\#11288](https://github.com/matrix-org/synapse/issues/11288)) +- Fix a bug, introduced in Synapse 1.46.0, which caused the `check_3pid_auth` and `on_logged_out` callbacks in legacy password authentication provider modules to not be registered. Modules using the generic module interface were not affected. ([\#11340](https://github.com/matrix-org/synapse/issues/11340)) +- Fix a bug introduced in 1.41.0 where space hierarchy responses would be incorrectly reused if multiple users were to make the same request at the same time. ([\#11355](https://github.com/matrix-org/synapse/issues/11355)) +- Fix a bug introduced in 1.45.0 where the `read_templates` method of the module API would error. ([\#11377](https://github.com/matrix-org/synapse/issues/11377)) +- Fix an issue introduced in 1.47.0 which prevented servers re-joining rooms they had previously left, if their signing keys were replaced. ([\#11379](https://github.com/matrix-org/synapse/issues/11379)) +- Fix a bug introduced in 1.13.0 where creating and publishing a room could cause errors if `room_list_publication_rules` is configured. ([\#11392](https://github.com/matrix-org/synapse/issues/11392)) +- Improve performance of various background database updates. ([\#11421](https://github.com/matrix-org/synapse/issues/11421), [\#11422](https://github.com/matrix-org/synapse/issues/11422)) + + +Improved Documentation +---------------------- + +- Suggest users of the Debian packages add configuration to `/etc/matrix-synapse/conf.d/` to prevent, upon upgrade, being asked to choose between their configuration and the maintainer's. ([\#11281](https://github.com/matrix-org/synapse/issues/11281)) +- Fix typos in the documentation for the `username_available` admin API. Contributed by Stanislav Motylkov. ([\#11286](https://github.com/matrix-org/synapse/issues/11286)) +- Add Single Sign-On, SAML and CAS pages to the documentation. ([\#11298](https://github.com/matrix-org/synapse/issues/11298)) +- Change the word 'Home server' as one word 'homeserver' in documentation. ([\#11320](https://github.com/matrix-org/synapse/issues/11320)) +- Fix missing quotes for wildcard domains in `federation_certificate_verification_whitelist`. ([\#11381](https://github.com/matrix-org/synapse/issues/11381)) + + +Deprecations and Removals +------------------------- + +- Remove deprecated `trust_identity_server_for_password_resets` configuration flag. ([\#11333](https://github.com/matrix-org/synapse/issues/11333), [\#11395](https://github.com/matrix-org/synapse/issues/11395)) + + +Internal Changes +---------------- + +- Add type annotations to `synapse.metrics`. ([\#10847](https://github.com/matrix-org/synapse/issues/10847)) +- Split out federated PDU retrieval function into a non-cached version. ([\#11242](https://github.com/matrix-org/synapse/issues/11242)) +- Clean up code relating to to-device messages and sending ephemeral events to application services. ([\#11247](https://github.com/matrix-org/synapse/issues/11247)) +- Fix a small typo in the error response when a relation type other than 'm.annotation' is passed to `GET /rooms/{room_id}/aggregations/{event_id}`. ([\#11278](https://github.com/matrix-org/synapse/issues/11278)) +- Drop unused database tables `room_stats_historical` and `user_stats_historical`. ([\#11280](https://github.com/matrix-org/synapse/issues/11280)) +- Require all files in synapse/ and tests/ to pass mypy unless specifically excluded. ([\#11282](https://github.com/matrix-org/synapse/issues/11282), [\#11285](https://github.com/matrix-org/synapse/issues/11285), [\#11359](https://github.com/matrix-org/synapse/issues/11359)) +- Add missing type hints to `synapse.app`. ([\#11287](https://github.com/matrix-org/synapse/issues/11287)) +- Remove unused parameters on `FederationEventHandler._check_event_auth`. ([\#11292](https://github.com/matrix-org/synapse/issues/11292)) +- Add type hints to `synapse._scripts`. ([\#11297](https://github.com/matrix-org/synapse/issues/11297)) +- Fix an issue which prevented the `remove_deleted_devices_from_device_inbox` background database schema update from running when updating from a recent Synapse version. ([\#11303](https://github.com/matrix-org/synapse/issues/11303)) +- Add type hints to storage classes. ([\#11307](https://github.com/matrix-org/synapse/issues/11307), [\#11310](https://github.com/matrix-org/synapse/issues/11310), [\#11311](https://github.com/matrix-org/synapse/issues/11311), [\#11312](https://github.com/matrix-org/synapse/issues/11312), [\#11313](https://github.com/matrix-org/synapse/issues/11313), [\#11314](https://github.com/matrix-org/synapse/issues/11314), [\#11316](https://github.com/matrix-org/synapse/issues/11316), [\#11322](https://github.com/matrix-org/synapse/issues/11322), [\#11332](https://github.com/matrix-org/synapse/issues/11332), [\#11339](https://github.com/matrix-org/synapse/issues/11339), [\#11342](https://github.com/matrix-org/synapse/issues/11342)) +- Add type hints to `synapse.util`. ([\#11321](https://github.com/matrix-org/synapse/issues/11321), [\#11328](https://github.com/matrix-org/synapse/issues/11328)) +- Improve type annotations in Synapse's test suite. ([\#11323](https://github.com/matrix-org/synapse/issues/11323), [\#11330](https://github.com/matrix-org/synapse/issues/11330)) +- Test that room alias deletion works as intended. ([\#11327](https://github.com/matrix-org/synapse/issues/11327)) +- Add type annotations for some methods and properties in the module API. ([\#11341](https://github.com/matrix-org/synapse/issues/11341)) +- Fix running `scripts-dev/complement.sh`, which was broken in v1.47.0rc1. ([\#11368](https://github.com/matrix-org/synapse/issues/11368)) +- Rename internal functions for token generation to better reflect what they do. ([\#11369](https://github.com/matrix-org/synapse/issues/11369), [\#11370](https://github.com/matrix-org/synapse/issues/11370)) +- Add type hints to configuration classes. ([\#11377](https://github.com/matrix-org/synapse/issues/11377)) +- Publish a `develop` image to Docker Hub. ([\#11380](https://github.com/matrix-org/synapse/issues/11380)) +- Keep fallback key marked as used if it's re-uploaded. ([\#11382](https://github.com/matrix-org/synapse/issues/11382)) +- Use `auto_attribs` on the `attrs` class `RefreshTokenLookupResult`. ([\#11386](https://github.com/matrix-org/synapse/issues/11386)) +- Rename unstable `access_token_lifetime` configuration option to `refreshable_access_token_lifetime` to make it clear it only concerns refreshable access tokens. ([\#11388](https://github.com/matrix-org/synapse/issues/11388)) +- Do not run the broken MSC2716 tests when running `scripts-dev/complement.sh`. ([\#11389](https://github.com/matrix-org/synapse/issues/11389)) +- Remove dead code from supporting ACME. ([\#11393](https://github.com/matrix-org/synapse/issues/11393)) +- Refactor including the bundled relations when serializing an event. ([\#11408](https://github.com/matrix-org/synapse/issues/11408)) + + +Synapse 1.47.1 (2021-11-23) +=========================== + +This release fixes a security issue in the media store, affecting all prior releases of Synapse. Server administrators are encouraged to update Synapse as soon as possible. We are not aware of these vulnerabilities being exploited in the wild. + +Server administrators who are unable to update Synapse may use the workarounds described in the linked GitHub Security Advisory below. + +Security advisory +----------------- + +The following issue is fixed in 1.47.1. + +- **[GHSA-3hfw-x7gx-437c](https://github.com/matrix-org/synapse/security/advisories/GHSA-3hfw-x7gx-437c) / [CVE-2021-41281](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-41281): Path traversal when downloading remote media.** + + Synapse instances with the media repository enabled can be tricked into downloading a file from a remote server into an arbitrary directory, potentially outside the media store directory. + + The last two directories and file name of the path are chosen randomly by Synapse and cannot be controlled by an attacker, which limits the impact. + + Homeservers with the media repository disabled are unaffected. Homeservers configured with a federation whitelist are also unaffected. + + Fixed by [91f2bd090](https://github.com/matrix-org/synapse/commit/91f2bd090). + + +Synapse 1.47.0 (2021-11-17) +=========================== + +No significant changes since 1.47.0rc3. + + +Synapse 1.47.0rc3 (2021-11-16) +============================== + +Bugfixes +-------- + +- Fix a bug introduced in 1.47.0rc1 which caused worker processes to not halt startup in the presence of outstanding database migrations. ([\#11346](https://github.com/matrix-org/synapse/issues/11346)) +- Fix a bug introduced in 1.47.0rc1 which prevented the 'remove deleted devices from `device_inbox` column' background process from running when updating from a recent Synapse version. ([\#11303](https://github.com/matrix-org/synapse/issues/11303), [\#11353](https://github.com/matrix-org/synapse/issues/11353)) + + +Synapse 1.47.0rc2 (2021-11-10) +============================== + +This fixes an issue with publishing the Debian packages for 1.47.0rc1. +It is otherwise identical to 1.47.0rc1. + + +Synapse 1.47.0rc1 (2021-11-09) +============================== + +Deprecations and Removals +------------------------- + +- The `user_may_create_room_with_invites` module callback is now deprecated. Please refer to the [upgrade notes](https://matrix-org.github.io/synapse/develop/upgrade#upgrading-to-v1470) for more information. ([\#11206](https://github.com/matrix-org/synapse/issues/11206)) +- Remove deprecated admin API to delete rooms (`POST /_synapse/admin/v1/rooms//delete`). ([\#11213](https://github.com/matrix-org/synapse/issues/11213)) + + +Features +-------- + +- Advertise support for Client-Server API r0.6.1. ([\#11097](https://github.com/matrix-org/synapse/issues/11097)) +- Add search by room ID and room alias to the List Room admin API. ([\#11099](https://github.com/matrix-org/synapse/issues/11099)) +- Add an `on_new_event` third-party rules callback to allow Synapse modules to act after an event has been sent into a room. ([\#11126](https://github.com/matrix-org/synapse/issues/11126)) +- Add a module API method to update a user's membership in a room. ([\#11147](https://github.com/matrix-org/synapse/issues/11147)) +- Add metrics for thread pool usage. ([\#11178](https://github.com/matrix-org/synapse/issues/11178)) +- Support the stable room type field for [MSC3288](https://github.com/matrix-org/matrix-doc/pull/3288). ([\#11187](https://github.com/matrix-org/synapse/issues/11187)) +- Add a module API method to retrieve the current state of a room. ([\#11204](https://github.com/matrix-org/synapse/issues/11204)) +- Calculate a default value for `public_baseurl` based on `server_name`. ([\#11210](https://github.com/matrix-org/synapse/issues/11210)) +- Add support for serving `/.well-known/matrix/server` files, to redirect federation traffic to port 443. ([\#11211](https://github.com/matrix-org/synapse/issues/11211)) +- Add admin APIs to pause, start and check the status of background updates. ([\#11263](https://github.com/matrix-org/synapse/issues/11263)) + + +Bugfixes +-------- + +- Fix a long-standing bug which allowed hidden devices to receive to-device messages, resulting in unnecessary database bloat. ([\#10097](https://github.com/matrix-org/synapse/issues/10097)) +- Fix a long-standing bug where messages in the `device_inbox` table for deleted devices would persist indefinitely. Contributed by @dklimpel and @JohannesKleine. ([\#10969](https://github.com/matrix-org/synapse/issues/10969), [\#11212](https://github.com/matrix-org/synapse/issues/11212)) +- Do not accept events if a third-party rule `check_event_allowed` callback raises an exception. ([\#11033](https://github.com/matrix-org/synapse/issues/11033)) +- Fix long-standing bug where verification requests could fail in certain cases if a federation whitelist was in place but did not include your own homeserver. ([\#11129](https://github.com/matrix-org/synapse/issues/11129)) +- Allow an empty list of `state_events_at_start` to be sent when using the [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) `/batch_send` endpoint and the author of the historical messages is already part of the current room state at the given `?prev_event_id`. ([\#11188](https://github.com/matrix-org/synapse/issues/11188)) +- Fix a bug introduced in Synapse 1.45.0 which prevented the `synapse_review_recent_signups` script from running. Contributed by @samuel-p. ([\#11191](https://github.com/matrix-org/synapse/issues/11191)) +- Delete `to_device` messages for hidden devices that will never be read, reducing database size. ([\#11199](https://github.com/matrix-org/synapse/issues/11199)) +- Fix a long-standing bug wherein a missing `Content-Type` header when downloading remote media would cause Synapse to throw an error. ([\#11200](https://github.com/matrix-org/synapse/issues/11200)) +- Fix a long-standing bug which could result in serialization errors and potentially duplicate transaction data when sending ephemeral events to application services. Contributed by @Fizzadar at Beeper. ([\#11207](https://github.com/matrix-org/synapse/issues/11207)) +- Fix a bug introduced in Synapse 1.35.0 which made it impossible to join rooms that return a `send_join` response containing floats. ([\#11217](https://github.com/matrix-org/synapse/issues/11217)) +- Fix long-standing bug where cross signing keys were not included in the response to `/r0/keys/query` the first time a remote user was queried. ([\#11234](https://github.com/matrix-org/synapse/issues/11234)) +- Fix a long-standing bug where all requests that read events from the database could get stuck as a result of losing the database connection. ([\#11240](https://github.com/matrix-org/synapse/issues/11240)) +- Fix a bug preventing Synapse from being rolled back to an earlier version when using workers. ([\#11255](https://github.com/matrix-org/synapse/issues/11255), [\#11276](https://github.com/matrix-org/synapse/issues/11276)) +- Fix a bug introduced in Synapse 1.37.1 which caused a remote event being processed by a worker to not get processed on restart if the worker was killed. ([\#11262](https://github.com/matrix-org/synapse/issues/11262)) +- Only allow old Element/Riot Android clients to send read receipts without a request body. All other clients must include a request body as required by the specification. Contributed by @rogersheu. ([\#11157](https://github.com/matrix-org/synapse/issues/11157)) + + +Updates to the Docker image +--------------------------- + +- Avoid changing user ID when started as a non-root user, and no explicit `UID` is set. ([\#11209](https://github.com/matrix-org/synapse/issues/11209)) + + +Improved Documentation +---------------------- + +- Improve example HAProxy config in the docs to properly handle HTTP `Host` headers with port information. This is required for federation over port 443 to work correctly. ([\#11128](https://github.com/matrix-org/synapse/issues/11128)) +- Add documentation for using Authentik as an OpenID Connect Identity Provider. Contributed by @samip5. ([\#11151](https://github.com/matrix-org/synapse/issues/11151)) +- Clarify lack of support for Windows. ([\#11198](https://github.com/matrix-org/synapse/issues/11198)) +- Improve code formatting and fix a few typos in docs. Contributed by @sumnerevans at Beeper. ([\#11221](https://github.com/matrix-org/synapse/issues/11221)) +- Add documentation for using LemonLDAP as an OpenID Connect Identity Provider. Contributed by @l00ptr. ([\#11257](https://github.com/matrix-org/synapse/issues/11257)) + + +Internal Changes +---------------- + +- Add type annotations for the `log_function` decorator. ([\#10943](https://github.com/matrix-org/synapse/issues/10943)) +- Add type hints to `synapse.events`. ([\#11098](https://github.com/matrix-org/synapse/issues/11098)) +- Remove and document unnecessary `RoomStreamToken` checks in application service ephemeral event code. ([\#11137](https://github.com/matrix-org/synapse/issues/11137)) +- Add type hints so that `synapse.http` passes `mypy` checks. ([\#11164](https://github.com/matrix-org/synapse/issues/11164)) +- Update scripts to pass Shellcheck lints. ([\#11166](https://github.com/matrix-org/synapse/issues/11166)) +- Add knock information in admin export. Contributed by Rafael Gonçalves. ([\#11171](https://github.com/matrix-org/synapse/issues/11171)) +- Add tests to check that `ClientIpStore.get_last_client_ip_by_device` and `get_user_ip_and_agents` combine database and in-memory data correctly. ([\#11179](https://github.com/matrix-org/synapse/issues/11179)) +- Refactor `Filter` to check different fields depending on the data type. ([\#11194](https://github.com/matrix-org/synapse/issues/11194)) +- Improve type hints for the relations datastore. ([\#11205](https://github.com/matrix-org/synapse/issues/11205)) +- Replace outdated links in the pull request checklist with links to the rendered documentation. ([\#11225](https://github.com/matrix-org/synapse/issues/11225)) +- Fix a bug in unit test `test_block_room_and_not_purge`. ([\#11226](https://github.com/matrix-org/synapse/issues/11226)) +- In `ObservableDeferred`, run observers in the order they were registered. ([\#11229](https://github.com/matrix-org/synapse/issues/11229)) +- Minor speed up to start up times and getting updates for groups by adding missing index to `local_group_updates.stream_id`. ([\#11231](https://github.com/matrix-org/synapse/issues/11231)) +- Add `twine` and `towncrier` as dev dependencies, as they're used by the release script. ([\#11233](https://github.com/matrix-org/synapse/issues/11233)) +- Allow `stream_writers.typing` config to be a list of one worker. ([\#11237](https://github.com/matrix-org/synapse/issues/11237)) +- Remove debugging statement in tests. ([\#11239](https://github.com/matrix-org/synapse/issues/11239)) +- Fix [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) historical messages backfilling in random order on remote homeservers. ([\#11244](https://github.com/matrix-org/synapse/issues/11244)) +- Add an additional test for the `cachedList` method decorator. ([\#11246](https://github.com/matrix-org/synapse/issues/11246)) +- Make minor correction to the type of `auth_checkers` callbacks. ([\#11253](https://github.com/matrix-org/synapse/issues/11253)) +- Clean up trivial aspects of the Debian package build tooling. ([\#11269](https://github.com/matrix-org/synapse/issues/11269), [\#11273](https://github.com/matrix-org/synapse/issues/11273)) +- Blacklist new SyTest that checks that key uploads are valid pending the validation being implemented in Synapse. ([\#11270](https://github.com/matrix-org/synapse/issues/11270)) + + +Synapse 1.46.0 (2021-11-02) +=========================== + +The cause of the [performance regression affecting Synapse 1.44](https://github.com/matrix-org/synapse/issues/11049) has been identified and fixed. ([\#11177](https://github.com/matrix-org/synapse/issues/11177)) + +Bugfixes +-------- + +- Fix a bug introduced in v1.46.0rc1 where URL previews of some XML documents would fail. ([\#11196](https://github.com/matrix-org/synapse/issues/11196)) + + +Synapse 1.46.0rc1 (2021-10-27) +============================== + +Features +-------- + +- Add support for Ubuntu 21.10 "Impish Indri". ([\#11024](https://github.com/matrix-org/synapse/issues/11024)) +- Port the Password Auth Providers module interface to the new generic interface. ([\#10548](https://github.com/matrix-org/synapse/issues/10548), [\#11180](https://github.com/matrix-org/synapse/issues/11180)) +- Experimental support for the thread relation defined in [MSC3440](https://github.com/matrix-org/matrix-doc/pull/3440). ([\#11088](https://github.com/matrix-org/synapse/issues/11088), [\#11181](https://github.com/matrix-org/synapse/issues/11181), [\#11192](https://github.com/matrix-org/synapse/issues/11192)) +- Users admin API can now also modify user type in addition to allowing it to be set on user creation. ([\#11174](https://github.com/matrix-org/synapse/issues/11174)) + + +Bugfixes +-------- + +- Newly-created public rooms are now only assigned an alias if the room's creation has not been blocked by permission settings. Contributed by @AndrewFerr. ([\#10930](https://github.com/matrix-org/synapse/issues/10930)) +- Fix a long-standing bug which meant that events received over federation were sometimes incorrectly accepted into the room state. ([\#11001](https://github.com/matrix-org/synapse/issues/11001), [\#11009](https://github.com/matrix-org/synapse/issues/11009), [\#11012](https://github.com/matrix-org/synapse/issues/11012)) +- Fix 500 error on `/messages` when the server accumulates more than 5 backwards extremities at a given depth for a room. ([\#11027](https://github.com/matrix-org/synapse/issues/11027)) +- Fix a bug where setting a user's `external_id` via the admin API returns 500 and deletes user's existing external mappings if that external ID is already mapped. ([\#11051](https://github.com/matrix-org/synapse/issues/11051)) +- Fix a long-standing bug where users excluded from the user directory were added into the directory if they belonged to a room which became public or private. ([\#11075](https://github.com/matrix-org/synapse/issues/11075)) +- Fix a long-standing bug when attempting to preview URLs which are in the `windows-1252` character encoding. ([\#11077](https://github.com/matrix-org/synapse/issues/11077), [\#11089](https://github.com/matrix-org/synapse/issues/11089)) +- Fix broken export-data admin command and add test script checking the command to CI. ([\#11078](https://github.com/matrix-org/synapse/issues/11078)) +- Show an error when timestamp in seconds is provided to the `/purge_media_cache` Admin API. ([\#11101](https://github.com/matrix-org/synapse/issues/11101)) +- Fix local users who left all their rooms being removed from the user directory, even if the `search_all_users` config option was enabled. ([\#11103](https://github.com/matrix-org/synapse/issues/11103)) +- Fix a bug which caused the module API's `get_user_ip_and_agents` function to always fail on workers. `get_user_ip_and_agents` was introduced in 1.44.0 and did not function correctly on worker processes at the time. ([\#11112](https://github.com/matrix-org/synapse/issues/11112)) +- Identity server connection is no longer ignoring `ip_range_whitelist`. ([\#11120](https://github.com/matrix-org/synapse/issues/11120)) +- Fix a bug introduced in Synapse 1.45.0 breaking the configuration file parsing script. ([\#11145](https://github.com/matrix-org/synapse/issues/11145)) +- Fix a performance regression introduced in 1.44.0 which could cause client requests to time out when making large numbers of outbound requests. ([\#11177](https://github.com/matrix-org/synapse/issues/11177), [\#11190](https://github.com/matrix-org/synapse/issues/11190)) +- Resolve and share `state_groups` for all [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) historical events in batch. ([\#10975](https://github.com/matrix-org/synapse/issues/10975)) + + +Improved Documentation +---------------------- + +- Fix broken links relating to module API deprecation in the upgrade notes. ([\#11069](https://github.com/matrix-org/synapse/issues/11069)) +- Add more information about what happens when a user is deactivated. ([\#11083](https://github.com/matrix-org/synapse/issues/11083)) +- Clarify the the sample log config can be copied from the documentation without issue. ([\#11092](https://github.com/matrix-org/synapse/issues/11092)) +- Update the admin API documentation with an updated list of the characters allowed in registration tokens. ([\#11093](https://github.com/matrix-org/synapse/issues/11093)) +- Document Synapse's behaviour when dealing with multiple modules registering the same callbacks and/or handlers for the same HTTP endpoints. ([\#11096](https://github.com/matrix-org/synapse/issues/11096)) +- Fix instances of `[example]{.title-ref}` in the upgrade documentation as a result of prior RST to Markdown conversion. ([\#11118](https://github.com/matrix-org/synapse/issues/11118)) +- Document the version of Synapse each module callback was introduced in. ([\#11132](https://github.com/matrix-org/synapse/issues/11132)) +- Document the version of Synapse that introduced each module API method. ([\#11183](https://github.com/matrix-org/synapse/issues/11183)) + + +Internal Changes +---------------- +- Fix spurious warnings about losing the logging context on the `ReplicationCommandHandler` when losing the replication connection. ([\#10984](https://github.com/matrix-org/synapse/issues/10984)) +- Include rejected status when we log events. ([\#11008](https://github.com/matrix-org/synapse/issues/11008)) +- Add some extra logging to the event persistence code. ([\#11014](https://github.com/matrix-org/synapse/issues/11014)) +- Rearrange the internal workings of the incremental user directory updates. ([\#11035](https://github.com/matrix-org/synapse/issues/11035)) +- Fix a long-standing bug where users excluded from the directory could still be added to the `users_who_share_private_rooms` table after a regular user joins a private room. ([\#11143](https://github.com/matrix-org/synapse/issues/11143)) +- Add and improve type hints. ([\#10972](https://github.com/matrix-org/synapse/issues/10972), [\#11055](https://github.com/matrix-org/synapse/issues/11055), [\#11066](https://github.com/matrix-org/synapse/issues/11066), [\#11076](https://github.com/matrix-org/synapse/issues/11076), [\#11095](https://github.com/matrix-org/synapse/issues/11095), [\#11109](https://github.com/matrix-org/synapse/issues/11109), [\#11121](https://github.com/matrix-org/synapse/issues/11121), [\#11146](https://github.com/matrix-org/synapse/issues/11146)) +- Mark the Synapse package as containing type annotations and fix export declarations so that Synapse pluggable modules may be type checked against Synapse. ([\#11054](https://github.com/matrix-org/synapse/issues/11054)) +- Remove dead code from `MediaFilePaths`. ([\#11056](https://github.com/matrix-org/synapse/issues/11056)) +- Be more lenient when parsing oEmbed response versions. ([\#11065](https://github.com/matrix-org/synapse/issues/11065)) +- Create a separate module for the retention configuration. ([\#11070](https://github.com/matrix-org/synapse/issues/11070)) +- Clean up some of the federation event authentication code for clarity. ([\#11115](https://github.com/matrix-org/synapse/issues/11115), [\#11116](https://github.com/matrix-org/synapse/issues/11116), [\#11122](https://github.com/matrix-org/synapse/issues/11122)) +- Add docstrings and comments to the application service ephemeral event sending code. ([\#11138](https://github.com/matrix-org/synapse/issues/11138)) +- Update the `sign_json` script to support inline configuration of the signing key. ([\#11139](https://github.com/matrix-org/synapse/issues/11139)) +- Fix broken link in the docker image README. ([\#11144](https://github.com/matrix-org/synapse/issues/11144)) +- Always dump logs from unit tests during CI runs. ([\#11068](https://github.com/matrix-org/synapse/issues/11068)) +- Add tests for `MediaFilePaths` class. ([\#11057](https://github.com/matrix-org/synapse/issues/11057)) +- Simplify the user admin API tests. ([\#11048](https://github.com/matrix-org/synapse/issues/11048)) +- Add a test for the workaround introduced in [\#11042](https://github.com/matrix-org/synapse/pull/11042) concerning the behaviour of third-party rule modules and `SynapseError`s. ([\#11071](https://github.com/matrix-org/synapse/issues/11071)) + + +Synapse 1.45.1 (2021-10-20) +=========================== + +Bugfixes +-------- + +- Revert change to counting of deactivated users towards the monthly active users limit, introduced in 1.45.0rc1. ([\#11127](https://github.com/matrix-org/synapse/issues/11127)) + + +Synapse 1.45.0 (2021-10-19) +=========================== + +No functional changes since Synapse 1.45.0rc2. + +Known Issues +------------ + +- A suspected [performance regression](https://github.com/matrix-org/synapse/issues/11049) which was first reported after the release of 1.44.0 remains unresolved. + + We have not been able to identify a probable cause. Affected users report that setting up a federation sender worker appears to alleviate symptoms of the regression. + +Improved Documentation +---------------------- + +- Reword changelog to clarify concerns about a suspected performance regression in 1.44.0. ([\#11117](https://github.com/matrix-org/synapse/issues/11117)) + + +Synapse 1.45.0rc2 (2021-10-14) +============================== + +This release candidate [fixes](https://github.com/matrix-org/synapse/issues/11053) a user directory [bug](https://github.com/matrix-org/synapse/issues/11025) present in 1.45.0rc1. + +Known Issues +------------ + +- A suspected [performance regression](https://github.com/matrix-org/synapse/issues/11049) which was first reported after the release of 1.44.0 remains unresolved. + + We have not been able to identify a probable cause. Affected users report that setting up a federation sender worker appears to alleviate symptoms of the regression. + +Bugfixes +-------- + +- Fix a long-standing bug when using multiple event persister workers where events were not correctly sent down `/sync` due to a race. ([\#11045](https://github.com/matrix-org/synapse/issues/11045)) +- Fix a bug introduced in Synapse 1.45.0rc1 where the user directory would stop updating if it processed an event from a + user not in the `users` table. ([\#11053](https://github.com/matrix-org/synapse/issues/11053)) +- Fix a bug introduced in Synapse 1.44.0 when logging errors during oEmbed processing. ([\#11061](https://github.com/matrix-org/synapse/issues/11061)) + + +Internal Changes +---------------- + +- Add an 'approximate difference' method to `StateFilter`. ([\#10825](https://github.com/matrix-org/synapse/issues/10825)) +- Fix inconsistent behavior of `get_last_client_by_ip` when reporting data that has not been stored in the database yet. ([\#10970](https://github.com/matrix-org/synapse/issues/10970)) +- Fix a bug introduced in Synapse 1.21.0 that causes opentracing and Prometheus metrics for replication requests to be measured incorrectly. ([\#10996](https://github.com/matrix-org/synapse/issues/10996)) +- Ensure that cache config tests do not share state. ([\#11036](https://github.com/matrix-org/synapse/issues/11036)) + + +Synapse 1.45.0rc1 (2021-10-12) +============================== + +**Note:** Media storage providers module that read from Synapse's configuration need changes as of this version, see the [upgrade notes](https://matrix-org.github.io/synapse/develop/upgrade#upgrading-to-v1450) for more information. + +Known Issues +------------ + +- We are investigating [a performance issue](https://github.com/matrix-org/synapse/issues/11049) which was reported after the release of 1.44.0. +- We are aware of [a bug](https://github.com/matrix-org/synapse/issues/11025) with the user directory when using application services. A second release candidate is expected which will resolve this. + +Features +-------- + +- Add [MSC3069](https://github.com/matrix-org/matrix-doc/pull/3069) support to `/account/whoami`. ([\#9655](https://github.com/matrix-org/synapse/issues/9655)) +- Support autodiscovery of oEmbed previews. ([\#10822](https://github.com/matrix-org/synapse/issues/10822)) +- Add a `user_may_send_3pid_invite` spam checker callback for modules to allow or deny 3PID invites. ([\#10894](https://github.com/matrix-org/synapse/issues/10894)) +- Add a spam checker callback to allow or deny room joins. ([\#10910](https://github.com/matrix-org/synapse/issues/10910)) +- Include an `update_synapse_database` script in the distribution. Contributed by @Fizzadar at Beeper. ([\#10954](https://github.com/matrix-org/synapse/issues/10954)) +- Include exception information in JSON logging output. Contributed by @Fizzadar at Beeper. ([\#11028](https://github.com/matrix-org/synapse/issues/11028)) + + +Bugfixes +-------- + +- Fix a minor bug in the response to `/_matrix/client/r0/voip/turnServer`. Contributed by @lukaslihotzki. ([\#10922](https://github.com/matrix-org/synapse/issues/10922)) +- Fix a bug where empty `yyyy-mm-dd/` directories would be left behind in the media store's `url_cache_thumbnails/` directory. ([\#10924](https://github.com/matrix-org/synapse/issues/10924)) +- Fix a bug introduced in Synapse v1.40.0 where the signature checks for room version 8 and 9 could be applied to earlier room versions in some situations. ([\#10927](https://github.com/matrix-org/synapse/issues/10927)) +- Fix a long-standing bug wherein deactivated users still count towards the monthly active users limit. ([\#10947](https://github.com/matrix-org/synapse/issues/10947)) +- Fix a long-standing bug which meant that events received over federation were sometimes incorrectly accepted into the room state. ([\#10956](https://github.com/matrix-org/synapse/issues/10956)) +- Fix a long-standing bug where rebuilding the user directory wouldn't exclude support and deactivated users. ([\#10960](https://github.com/matrix-org/synapse/issues/10960)) +- Fix [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) `/batch_send` endpoint rejecting subsequent batches with unknown batch ID error in existing room versions from the room creator. ([\#10962](https://github.com/matrix-org/synapse/issues/10962)) +- Fix a bug that could leak local users' per-room nicknames and avatars when the user directory is rebuilt. ([\#10981](https://github.com/matrix-org/synapse/issues/10981)) +- Fix a long-standing bug where the remainder of a batch of user directory changes would be silently dropped if the server left a room early in the batch. ([\#10982](https://github.com/matrix-org/synapse/issues/10982)) +- Correct a bugfix introduced in Synapse v1.44.0 that would catch the wrong error if a connection is lost before a response could be written to it. ([\#10995](https://github.com/matrix-org/synapse/issues/10995)) +- Fix a long-standing bug where local users' per-room nicknames/avatars were visible to anyone who could see you in the user directory. ([\#11002](https://github.com/matrix-org/synapse/issues/11002)) +- Fix a long-standing bug where a user's per-room nickname/avatar would overwrite their profile in the user directory when a room was made public. ([\#11003](https://github.com/matrix-org/synapse/issues/11003)) +- Work around a regression, introduced in Synapse v1.39.0, that caused `SynapseError`s raised by the experimental third-party rules module callback `check_event_allowed` to be ignored. ([\#11042](https://github.com/matrix-org/synapse/issues/11042)) +- Fix a bug in [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) insertion events in rooms that could cause cross-talk/conflicts between batches. ([\#10877](https://github.com/matrix-org/synapse/issues/10877)) + + +Improved Documentation +---------------------- + +- Change wording ("reference homeserver") in Synapse repository documentation. Contributed by @maxkratz. ([\#10971](https://github.com/matrix-org/synapse/issues/10971)) +- Fix a dead URL in development documentation (SAML) and change wording from "Riot" to "Element". Contributed by @maxkratz. ([\#10973](https://github.com/matrix-org/synapse/issues/10973)) +- Add additional content to the Welcome and Overview page of the documentation. ([\#10990](https://github.com/matrix-org/synapse/issues/10990)) +- Update links to MSCs in documentation. Contributed by @dklimpel. ([\#10991](https://github.com/matrix-org/synapse/issues/10991)) + + +Internal Changes +---------------- + +- Improve type hinting in `synapse.util`. ([\#10888](https://github.com/matrix-org/synapse/issues/10888)) +- Add further type hints to `synapse.storage.util`. ([\#10892](https://github.com/matrix-org/synapse/issues/10892)) +- Fix type hints to be compatible with an upcoming change to Twisted. ([\#10895](https://github.com/matrix-org/synapse/issues/10895)) +- Update utility code to handle C implementations of frozendict. ([\#10902](https://github.com/matrix-org/synapse/issues/10902)) +- Drop old functionality which maintained database compatibility with Synapse versions before v1.31. ([\#10903](https://github.com/matrix-org/synapse/issues/10903)) +- Clean-up configuration helper classes for the `ServerConfig` class. ([\#10915](https://github.com/matrix-org/synapse/issues/10915)) +- Use direct references to config flags. ([\#10916](https://github.com/matrix-org/synapse/issues/10916), [\#10959](https://github.com/matrix-org/synapse/issues/10959), [\#10985](https://github.com/matrix-org/synapse/issues/10985)) +- Clean up some of the federation event authentication code for clarity. ([\#10926](https://github.com/matrix-org/synapse/issues/10926), [\#10940](https://github.com/matrix-org/synapse/issues/10940), [\#10986](https://github.com/matrix-org/synapse/issues/10986), [\#10987](https://github.com/matrix-org/synapse/issues/10987), [\#10988](https://github.com/matrix-org/synapse/issues/10988), [\#11010](https://github.com/matrix-org/synapse/issues/11010), [\#11011](https://github.com/matrix-org/synapse/issues/11011)) +- Refactor various parts of the codebase to use `RoomVersion` objects instead of room version identifier strings. ([\#10934](https://github.com/matrix-org/synapse/issues/10934)) +- Refactor user directory tests in preparation for upcoming changes. ([\#10935](https://github.com/matrix-org/synapse/issues/10935)) +- Include the event id in the logcontext when handling PDUs received over federation. ([\#10936](https://github.com/matrix-org/synapse/issues/10936)) +- Fix logged errors in unit tests. ([\#10939](https://github.com/matrix-org/synapse/issues/10939)) +- Fix a broken test to ensure that consent configuration works during registration. ([\#10945](https://github.com/matrix-org/synapse/issues/10945)) +- Add type hints to filtering classes. ([\#10958](https://github.com/matrix-org/synapse/issues/10958)) +- Add type-hint to `HomeserverTestcase.setup_test_homeserver`. ([\#10961](https://github.com/matrix-org/synapse/issues/10961)) +- Fix the test utility function `create_room_as` so that `is_public=True` will explicitly set the `visibility` parameter of room creation requests to `public`. Contributed by @AndrewFerr. ([\#10963](https://github.com/matrix-org/synapse/issues/10963)) +- Make the release script more robust and transparent. ([\#10966](https://github.com/matrix-org/synapse/issues/10966)) +- Refactor [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) `/batch_send` mega function into smaller handler functions. ([\#10974](https://github.com/matrix-org/synapse/issues/10974)) +- Log stack traces when a missing opentracing span is detected. ([\#10983](https://github.com/matrix-org/synapse/issues/10983)) +- Update GHA config to run tests against Python 3.10 and PostgreSQL 14. ([\#10992](https://github.com/matrix-org/synapse/issues/10992)) +- Fix a long-standing bug where `ReadWriteLock`s could drop logging contexts on exit. ([\#10993](https://github.com/matrix-org/synapse/issues/10993)) +- Add a `CODEOWNERS` file to automatically request reviews from the `@matrix-org/synapse-core` team on new pull requests. ([\#10994](https://github.com/matrix-org/synapse/issues/10994)) +- Add further type hints to `synapse.state`. ([\#11004](https://github.com/matrix-org/synapse/issues/11004)) +- Remove the deprecated `BaseHandler` object. ([\#11005](https://github.com/matrix-org/synapse/issues/11005)) +- Bump mypy version for CI to 0.910, and pull in new type stubs for dependencies. ([\#11006](https://github.com/matrix-org/synapse/issues/11006)) +- Fix CI to run the unit tests without optional deps. ([\#11017](https://github.com/matrix-org/synapse/issues/11017)) +- Ensure that cache config tests do not share state. ([\#11019](https://github.com/matrix-org/synapse/issues/11019)) +- Add additional type hints to `synapse.server_notices`. ([\#11021](https://github.com/matrix-org/synapse/issues/11021)) +- Add additional type hints for `synapse.push`. ([\#11023](https://github.com/matrix-org/synapse/issues/11023)) +- When installing the optional developer dependencies, also include the dependencies needed for type-checking and unit testing. ([\#11034](https://github.com/matrix-org/synapse/issues/11034)) +- Remove unnecessary list comprehension from `synapse_port_db` to satisfy code style requirements. ([\#11043](https://github.com/matrix-org/synapse/issues/11043)) + + +Synapse 1.44.0 (2021-10-05) +=========================== + +No significant changes since 1.44.0rc3. + + +Synapse 1.44.0rc3 (2021-10-04) +============================== + +Bugfixes +-------- + +- Fix a bug introduced in Synapse v1.40.0 where changing a user's display name or avatar in a restricted room would cause an authentication error. ([\#10933](https://github.com/matrix-org/synapse/issues/10933)) +- Fix `/admin/whois/{user_id}` endpoint, which was broken in v1.44.0rc1. ([\#10968](https://github.com/matrix-org/synapse/issues/10968)) + + +Synapse 1.44.0rc2 (2021-09-30) +============================== + +Bugfixes +-------- + +- Fix a bug introduced in v1.44.0rc1 which caused the experimental [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) `/batch_send` endpoint to return a 500 error. ([\#10938](https://github.com/matrix-org/synapse/issues/10938)) +- Fix a bug introduced in v1.44.0rc1 which prevented sending presence events to application services. ([\#10944](https://github.com/matrix-org/synapse/issues/10944)) + + +Improved Documentation +---------------------- + +- Minor updates to the installation instructions. ([\#10919](https://github.com/matrix-org/synapse/issues/10919)) + + +Synapse 1.44.0rc1 (2021-09-29) +============================== + +Features +-------- + +- Only allow the [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) `/batch_send?chunk_id=xxx` endpoint to connect to an already existing insertion event. ([\#10776](https://github.com/matrix-org/synapse/issues/10776)) +- Improve oEmbed URL previews by processing the author name, photo, and video information. ([\#10814](https://github.com/matrix-org/synapse/issues/10814), [\#10819](https://github.com/matrix-org/synapse/issues/10819)) +- Speed up responding with large JSON objects to requests. ([\#10868](https://github.com/matrix-org/synapse/issues/10868), [\#10905](https://github.com/matrix-org/synapse/issues/10905)) +- Add a `user_may_create_room_with_invites` spam checker callback to allow modules to allow or deny a room creation request based on the invites and/or 3PID invites it includes. ([\#10898](https://github.com/matrix-org/synapse/issues/10898)) + + +Bugfixes +-------- + +- Fix a long-standing bug that caused an `AssertionError` when purging history in certain rooms. Contributed by @Kokokokoka. ([\#10690](https://github.com/matrix-org/synapse/issues/10690)) +- Fix a long-standing bug which caused deactivated users that were later reactivated to be missing from the user directory. ([\#10782](https://github.com/matrix-org/synapse/issues/10782)) +- Fix a long-standing bug that caused unbanning a user by sending a membership event to fail. Contributed by @aaronraimist. ([\#10807](https://github.com/matrix-org/synapse/issues/10807)) +- Fix a long-standing bug where logging contexts would go missing when federation requests time out. ([\#10810](https://github.com/matrix-org/synapse/issues/10810)) +- Fix a long-standing bug causing an error in the deprecated `/initialSync` endpoint when using the undocumented `from` and `to` parameters. ([\#10827](https://github.com/matrix-org/synapse/issues/10827)) +- Fix a bug causing the `remove_stale_pushers` background job to repeatedly fail and log errors. This bug affected Synapse servers that had been upgraded from version 1.28 or older and are using SQLite. ([\#10843](https://github.com/matrix-org/synapse/issues/10843)) +- Fix a long-standing bug in Unicode support of the room search admin API breaking search for rooms with non-ASCII characters. ([\#10859](https://github.com/matrix-org/synapse/issues/10859)) +- Fix a bug introduced in Synapse 1.37.0 which caused `knock` membership events which we sent to remote servers to be incorrectly stored in the local database. ([\#10873](https://github.com/matrix-org/synapse/issues/10873)) +- Fix invalidating one-time key count cache after claiming keys. The bug was introduced in Synapse v1.41.0. Contributed by Tulir at Beeper. ([\#10875](https://github.com/matrix-org/synapse/issues/10875)) +- Fix a long-standing bug causing application service users to be subject to MAU blocking if the MAU limit had been reached, even if configured not to be blocked. ([\#10881](https://github.com/matrix-org/synapse/issues/10881)) +- Fix a long-standing bug which could cause events pulled over federation to be incorrectly rejected. ([\#10907](https://github.com/matrix-org/synapse/issues/10907)) +- Fix a long-standing bug causing URL cache files to be stored in storage providers. Server admins may safely delete the `url_cache/` and `url_cache_thumbnails/` directories from any configured storage providers to reclaim space. ([\#10911](https://github.com/matrix-org/synapse/issues/10911)) +- Fix a long-standing bug leading to race conditions when creating media store and config directories. ([\#10913](https://github.com/matrix-org/synapse/issues/10913)) + + +Improved Documentation +---------------------- + +- Fix some crashes in the Module API example code, by adding JSON encoding/decoding. ([\#10845](https://github.com/matrix-org/synapse/issues/10845)) +- Add developer documentation about experimental configuration flags. ([\#10865](https://github.com/matrix-org/synapse/issues/10865)) +- Properly remove deleted files from GitHub pages when generating the documentation. ([\#10869](https://github.com/matrix-org/synapse/issues/10869)) + + +Internal Changes +---------------- + +- Fix GitHub Actions config so we can run sytest on synapse from parallel branches. ([\#10659](https://github.com/matrix-org/synapse/issues/10659)) +- Split out [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) meta events to their own fields in the `/batch_send` response. ([\#10777](https://github.com/matrix-org/synapse/issues/10777)) +- Add missing type hints to REST servlets. ([\#10785](https://github.com/matrix-org/synapse/issues/10785), [\#10817](https://github.com/matrix-org/synapse/issues/10817)) +- Simplify the internal logic which maintains the user directory database tables. ([\#10796](https://github.com/matrix-org/synapse/issues/10796)) +- Use direct references to config flags. ([\#10812](https://github.com/matrix-org/synapse/issues/10812), [\#10885](https://github.com/matrix-org/synapse/issues/10885), [\#10893](https://github.com/matrix-org/synapse/issues/10893), [\#10897](https://github.com/matrix-org/synapse/issues/10897)) +- Specify the type of token in generic "Invalid token" error messages. ([\#10815](https://github.com/matrix-org/synapse/issues/10815)) +- Make `StateFilter` frozen so it is hashable. ([\#10816](https://github.com/matrix-org/synapse/issues/10816)) +- Fix a long-standing bug where an `m.room.message` event containing a null byte would cause an internal server error. ([\#10820](https://github.com/matrix-org/synapse/issues/10820)) +- Add type hints to the state database. ([\#10823](https://github.com/matrix-org/synapse/issues/10823)) +- Opt out of cache expiry for `get_users_who_share_room_with_user`, to hopefully improve `/sync` performance when you + haven't synced recently. ([\#10826](https://github.com/matrix-org/synapse/issues/10826)) +- Track cache eviction rates more finely in Prometheus's monitoring. ([\#10829](https://github.com/matrix-org/synapse/issues/10829)) +- Add missing type hints to `synapse.handlers`. ([\#10831](https://github.com/matrix-org/synapse/issues/10831), [\#10856](https://github.com/matrix-org/synapse/issues/10856)) +- Extend the Module API to let plug-ins check whether an ID is local and to access IP + User Agent data. ([\#10833](https://github.com/matrix-org/synapse/issues/10833)) +- Factor out PNG image data to a constant to be used in several tests. ([\#10834](https://github.com/matrix-org/synapse/issues/10834)) +- Add a test to ensure state events sent by modules get persisted correctly. ([\#10835](https://github.com/matrix-org/synapse/issues/10835)) +- Rename [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) fields and event types from `chunk` to `batch` to match the `/batch_send` endpoint. ([\#10838](https://github.com/matrix-org/synapse/issues/10838)) +- Rename [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) `/batch_send` query parameter from `?prev_event` to more obvious usage with `?prev_event_id`. ([\#10839](https://github.com/matrix-org/synapse/issues/10839)) +- Add type hints to `synapse.http.site`. ([\#10867](https://github.com/matrix-org/synapse/issues/10867)) +- Include outlier status when we log V2 or V3 events. ([\#10879](https://github.com/matrix-org/synapse/issues/10879)) +- Break down Grafana's cache expiry time series based on reason for eviction, c.f. [\#10829](https://github.com/matrix-org/synapse/issues/10829). ([\#10880](https://github.com/matrix-org/synapse/issues/10880)) +- Clean up some of the federation event authentication code for clarity. ([\#10883](https://github.com/matrix-org/synapse/issues/10883), [\#10884](https://github.com/matrix-org/synapse/issues/10884), [\#10896](https://github.com/matrix-org/synapse/issues/10896), [\#10901](https://github.com/matrix-org/synapse/issues/10901)) +- Allow the `.` and `~` characters when creating registration tokens as per the change to [MSC3231](https://github.com/matrix-org/matrix-doc/pull/3231). ([\#10887](https://github.com/matrix-org/synapse/issues/10887)) +- Clean up some unnecessary parentheses in places around the codebase. ([\#10889](https://github.com/matrix-org/synapse/issues/10889)) +- Improve type hinting in the user directory code. ([\#10891](https://github.com/matrix-org/synapse/issues/10891)) +- Update development testing script `test_postgresql.sh` to use a supported Python version and make re-runs quicker. ([\#10906](https://github.com/matrix-org/synapse/issues/10906)) +- Document and summarize changes in schema version `61` – `64`. ([\#10917](https://github.com/matrix-org/synapse/issues/10917)) +- Update release script to sign the newly created git tags. ([\#10925](https://github.com/matrix-org/synapse/issues/10925)) +- Fix Debian builds due to `dh-virtualenv` no longer being able to build their docs. ([\#10931](https://github.com/matrix-org/synapse/issues/10931)) + + +Synapse 1.43.0 (2021-09-21) +=========================== + +This release drops support for the deprecated, unstable API for [MSC2858 (Multiple SSO Identity Providers)](https://github.com/matrix-org/matrix-doc/blob/master/proposals/2858-Multiple-SSO-Identity-Providers.md#unstable-prefix), as well as the undocumented `experimental.msc2858_enabled` config option. Client authors should update their clients to use the stable API, available since Synapse 1.30. + +The documentation has been updated with configuration for routing `/spaces`, `/hierarchy` and `/summary` to workers. See [the upgrade notes](https://github.com/matrix-org/synapse/blob/release-v1.43/docs/upgrade.md#upgrading-to-v1430) for more details. + +No significant changes since 1.43.0rc2. + +Synapse 1.43.0rc2 (2021-09-17) +============================== + +Bugfixes +-------- + +- Added opentracing logging to help debug [\#9424](https://github.com/matrix-org/synapse/issues/9424). ([\#10828](https://github.com/matrix-org/synapse/issues/10828)) + + +Synapse 1.43.0rc1 (2021-09-14) +============================== + +Features +-------- + +- Allow room creators to send historical events specified by [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) in existing room versions. ([\#10566](https://github.com/matrix-org/synapse/issues/10566)) +- Add config option to use non-default manhole password and keys. ([\#10643](https://github.com/matrix-org/synapse/issues/10643)) +- Skip final GC at shutdown to improve restart performance. ([\#10712](https://github.com/matrix-org/synapse/issues/10712)) +- Allow configuration of the oEmbed URLs used for URL previews. ([\#10714](https://github.com/matrix-org/synapse/issues/10714), [\#10759](https://github.com/matrix-org/synapse/issues/10759)) +- Prefer [room version 9](https://github.com/matrix-org/matrix-doc/pull/3375) for restricted rooms per the [room version capabilities](https://github.com/matrix-org/matrix-doc/pull/3244) API. ([\#10772](https://github.com/matrix-org/synapse/issues/10772)) + + +Bugfixes +-------- + +- Fix a long-standing bug where room avatars were not included in email notifications. ([\#10658](https://github.com/matrix-org/synapse/issues/10658)) +- Fix a bug where the ordering algorithm was skipping the `origin_server_ts` step in the spaces summary resulting in unstable room orderings. ([\#10730](https://github.com/matrix-org/synapse/issues/10730)) +- Fix edge case when persisting events into a room where there are multiple events we previously hadn't calculated auth chains for (and hadn't marked as needing to be calculated). ([\#10743](https://github.com/matrix-org/synapse/issues/10743)) +- Fix a bug which prevented calls to `/createRoom` that included the `room_alias_name` parameter from being handled by worker processes. ([\#10757](https://github.com/matrix-org/synapse/issues/10757)) +- Fix a bug which prevented user registration via SSO to require consent tracking for SSO mapping providers that don't prompt for Matrix ID selection. Contributed by @AndrewFerr. ([\#10733](https://github.com/matrix-org/synapse/issues/10733)) +- Only return the stripped state events for the `m.space.child` events in a room for the spaces summary from [MSC2946](https://github.com/matrix-org/matrix-doc/pull/2946). ([\#10760](https://github.com/matrix-org/synapse/issues/10760)) +- Properly handle room upgrades of spaces. ([\#10774](https://github.com/matrix-org/synapse/issues/10774)) +- Fix a bug which generated invalid homeserver config when the `frontend_proxy` worker type was passed to the Synapse Worker-based Complement image. ([\#10783](https://github.com/matrix-org/synapse/issues/10783)) + + +Improved Documentation +---------------------- + +- Minor fix to the `media_repository` developer documentation. Contributed by @cuttingedge1109. ([\#10556](https://github.com/matrix-org/synapse/issues/10556)) +- Update the documentation to note that the `/spaces` and `/hierarchy` endpoints can be routed to workers. ([\#10648](https://github.com/matrix-org/synapse/issues/10648)) +- Clarify admin API documentation on undoing room deletions. ([\#10735](https://github.com/matrix-org/synapse/issues/10735)) +- Split up the modules documentation and add examples for module developers. ([\#10758](https://github.com/matrix-org/synapse/issues/10758)) +- Correct 2 typographical errors in the [Log Contexts documentation](https://matrix-org.github.io/synapse/latest/log_contexts.html). ([\#10795](https://github.com/matrix-org/synapse/issues/10795)) +- Fix a wording mistake in the sample configuration. Contributed by @bramvdnheuvel:nltrix.net. ([\#10804](https://github.com/matrix-org/synapse/issues/10804)) + + +Deprecations and Removals +------------------------- + +- Remove the [unstable MSC2858 API](https://github.com/matrix-org/matrix-doc/blob/master/proposals/2858-Multiple-SSO-Identity-Providers.md#unstable-prefix), including the undocumented `experimental.msc2858_enabled` config option. The unstable API has been deprecated since Synapse 1.35. Client authors should update their clients to use the stable API introduced in Synapse 1.30 if they have not already done so. ([\#10693](https://github.com/matrix-org/synapse/issues/10693)) + + +Internal Changes +---------------- + +- Add OpenTracing logging to help debug stuck messages (as described by issue [#9424](https://github.com/matrix-org/synapse/issues/9424)). ([\#10704](https://github.com/matrix-org/synapse/issues/10704)) +- Add type annotations to the `synapse.util` package. ([\#10601](https://github.com/matrix-org/synapse/issues/10601)) +- Ensure `rooms.creator` field is always populated for easy lookup in [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) usage later. ([\#10697](https://github.com/matrix-org/synapse/issues/10697)) +- Add missing type hints to REST servlets. ([\#10707](https://github.com/matrix-org/synapse/issues/10707), [\#10728](https://github.com/matrix-org/synapse/issues/10728), [\#10736](https://github.com/matrix-org/synapse/issues/10736)) +- Do not include rooms with unknown room versions in the spaces summary results. ([\#10727](https://github.com/matrix-org/synapse/issues/10727)) +- Additional error checking for the `preset` field when creating a room. ([\#10738](https://github.com/matrix-org/synapse/issues/10738)) +- Clean up some of the federation event authentication code for clarity. ([\#10744](https://github.com/matrix-org/synapse/issues/10744), [\#10745](https://github.com/matrix-org/synapse/issues/10745), [\#10746](https://github.com/matrix-org/synapse/issues/10746), [\#10771](https://github.com/matrix-org/synapse/issues/10771), [\#10773](https://github.com/matrix-org/synapse/issues/10773), [\#10781](https://github.com/matrix-org/synapse/issues/10781)) +- Add an index to `presence_stream` to hopefully speed up startups a little. ([\#10748](https://github.com/matrix-org/synapse/issues/10748)) +- Refactor event size checking code to simplify searching the codebase for the origins of certain error strings that are occasionally emitted. ([\#10750](https://github.com/matrix-org/synapse/issues/10750)) +- Move tests relating to rooms having encryption out of the user directory tests. ([\#10752](https://github.com/matrix-org/synapse/issues/10752)) +- Use `attrs` internally for the URL preview code & update documentation. ([\#10753](https://github.com/matrix-org/synapse/issues/10753)) +- Minor speed ups when joining large rooms over federation. ([\#10754](https://github.com/matrix-org/synapse/issues/10754), [\#10755](https://github.com/matrix-org/synapse/issues/10755), [\#10756](https://github.com/matrix-org/synapse/issues/10756), [\#10780](https://github.com/matrix-org/synapse/issues/10780), [\#10784](https://github.com/matrix-org/synapse/issues/10784)) +- Add a constant for `m.federate`. ([\#10775](https://github.com/matrix-org/synapse/issues/10775)) +- Add a script to update the Debian changelog in a Docker container for systems that are not Debian-based. ([\#10778](https://github.com/matrix-org/synapse/issues/10778)) +- Change the format of authenticated users in logs when a user is being puppeted by and admin user. ([\#10779](https://github.com/matrix-org/synapse/issues/10779)) +- Remove fixed and flakey tests from the Sytest blacklist. ([\#10788](https://github.com/matrix-org/synapse/issues/10788)) +- Improve internal details of the user directory code. ([\#10789](https://github.com/matrix-org/synapse/issues/10789)) +- Use direct references to config flags. ([\#10798](https://github.com/matrix-org/synapse/issues/10798)) +- Ensure the Rust reporter passes type checking with jaeger-client 4.7's type annotations. ([\#10799](https://github.com/matrix-org/synapse/issues/10799)) + + +Synapse 1.42.0 (2021-09-07) +=========================== + +This version of Synapse removes deprecated room-management admin APIs, removes out-of-date email pushers, and improves error handling for fallback templates for user-interactive authentication. For more information on these points, server administrators are encouraged to read [the upgrade notes](docs/upgrade.md#upgrading-to-v1420). + +No significant changes since 1.42.0rc2. + + +Synapse 1.42.0rc2 (2021-09-06) +============================== + +Features +-------- + +- Support room version 9 from [MSC3375](https://github.com/matrix-org/matrix-doc/pull/3375). ([\#10747](https://github.com/matrix-org/synapse/issues/10747)) + + +Internal Changes +---------------- + +- Print a warning when using one of the deprecated `template_dir` settings. ([\#10768](https://github.com/matrix-org/synapse/issues/10768)) + + +Synapse 1.42.0rc1 (2021-09-01) +============================== + +Features +-------- + +- Add support for [MSC3231](https://github.com/matrix-org/matrix-doc/pull/3231): Token authenticated registration. Users can be required to submit a token during registration to authenticate themselves. Contributed by Callum Brown. ([\#10142](https://github.com/matrix-org/synapse/issues/10142)) +- Add support for [MSC3283](https://github.com/matrix-org/matrix-doc/pull/3283): Expose `enable_set_displayname` in capabilities. ([\#10452](https://github.com/matrix-org/synapse/issues/10452)) +- Port the `PresenceRouter` module interface to the new generic interface. ([\#10524](https://github.com/matrix-org/synapse/issues/10524)) +- Add pagination to the spaces summary based on updates to [MSC2946](https://github.com/matrix-org/matrix-doc/pull/2946). ([\#10613](https://github.com/matrix-org/synapse/issues/10613), [\#10725](https://github.com/matrix-org/synapse/issues/10725)) + + +Bugfixes +-------- + +- Validate new `m.room.power_levels` events. Contributed by @aaronraimist. ([\#10232](https://github.com/matrix-org/synapse/issues/10232)) +- Display an error on User-Interactive Authentication fallback pages when authentication fails. Contributed by Callum Brown. ([\#10561](https://github.com/matrix-org/synapse/issues/10561)) +- Remove pushers when deleting an e-mail address from an account. Pushers for old unlinked emails will also be deleted. ([\#10581](https://github.com/matrix-org/synapse/issues/10581), [\#10734](https://github.com/matrix-org/synapse/issues/10734)) +- Reject Client-Server `/keys/query` requests which provide `device_ids` incorrectly. ([\#10593](https://github.com/matrix-org/synapse/issues/10593)) +- Rooms with unsupported room versions are no longer returned via `/sync`. ([\#10644](https://github.com/matrix-org/synapse/issues/10644)) +- Enforce the maximum length for per-room display names and avatar URLs. ([\#10654](https://github.com/matrix-org/synapse/issues/10654)) +- Fix a bug which caused the `synapse_user_logins_total` Prometheus metric not to be correctly initialised on restart. ([\#10677](https://github.com/matrix-org/synapse/issues/10677)) +- Improve `ServerNoticeServlet` to avoid duplicate requests and add unit tests. ([\#10679](https://github.com/matrix-org/synapse/issues/10679)) +- Fix long-standing issue which caused an error when a thumbnail is requested and there are multiple thumbnails with the same quality rating. ([\#10684](https://github.com/matrix-org/synapse/issues/10684)) +- Fix a regression introduced in v1.41.0 which affected the performance of concurrent fetches of large sets of events, in extreme cases causing the process to hang. ([\#10703](https://github.com/matrix-org/synapse/issues/10703)) +- Fix a regression introduced in Synapse 1.41 which broke email transmission on Systems using older versions of the Twisted library. ([\#10713](https://github.com/matrix-org/synapse/issues/10713)) + + +Improved Documentation +---------------------- + +- Add documentation on how to connect Django with Synapse using OpenID Connect and django-oauth-toolkit. Contributed by @HugoDelval. ([\#10192](https://github.com/matrix-org/synapse/issues/10192)) +- Advertise https://matrix-org.github.io/synapse documentation in the `README` and `CONTRIBUTING` files. ([\#10595](https://github.com/matrix-org/synapse/issues/10595)) +- Fix some of the titles not rendering in the OpenID Connect documentation. ([\#10639](https://github.com/matrix-org/synapse/issues/10639)) +- Minor clarifications to the documentation for reverse proxies. ([\#10708](https://github.com/matrix-org/synapse/issues/10708)) +- Remove table of contents from the top of installation and contributing documentation pages. ([\#10711](https://github.com/matrix-org/synapse/issues/10711)) + + +Deprecations and Removals +------------------------- + +- Remove deprecated Shutdown Room and Purge Room Admin API. ([\#8830](https://github.com/matrix-org/synapse/issues/8830)) + + +Internal Changes +---------------- + +- Improve type hints for the proxy agent and SRV resolver modules. Contributed by @dklimpel. ([\#10608](https://github.com/matrix-org/synapse/issues/10608)) +- Clean up some of the federation event authentication code for clarity. ([\#10614](https://github.com/matrix-org/synapse/issues/10614), [\#10615](https://github.com/matrix-org/synapse/issues/10615), [\#10624](https://github.com/matrix-org/synapse/issues/10624), [\#10640](https://github.com/matrix-org/synapse/issues/10640)) +- Add a comment asking developers to leave a reason when bumping the database schema version. ([\#10621](https://github.com/matrix-org/synapse/issues/10621)) +- Remove not needed database updates in modify user admin API. ([\#10627](https://github.com/matrix-org/synapse/issues/10627)) +- Convert room member storage tuples to `attrs` classes. ([\#10629](https://github.com/matrix-org/synapse/issues/10629), [\#10642](https://github.com/matrix-org/synapse/issues/10642)) +- Use auto-attribs for the attrs classes used in sync. ([\#10630](https://github.com/matrix-org/synapse/issues/10630)) +- Make `backfill` and `get_missing_events` use the same codepath. ([\#10645](https://github.com/matrix-org/synapse/issues/10645)) +- Improve the performance of the `/hierarchy` API (from [MSC2946](https://github.com/matrix-org/matrix-doc/pull/2946)) by caching responses received over federation. ([\#10647](https://github.com/matrix-org/synapse/issues/10647)) +- Run a nightly CI build against Twisted trunk. ([\#10651](https://github.com/matrix-org/synapse/issues/10651), [\#10672](https://github.com/matrix-org/synapse/issues/10672)) +- Do not print out stack traces for network errors when fetching data over federation. ([\#10662](https://github.com/matrix-org/synapse/issues/10662)) +- Simplify tests for device admin rest API. ([\#10664](https://github.com/matrix-org/synapse/issues/10664)) +- Add missing type hints to REST servlets. ([\#10665](https://github.com/matrix-org/synapse/issues/10665), [\#10666](https://github.com/matrix-org/synapse/issues/10666), [\#10674](https://github.com/matrix-org/synapse/issues/10674)) +- Flatten the `tests.synapse.rests` package by moving the contents of `v1` and `v2_alpha` into the parent. ([\#10667](https://github.com/matrix-org/synapse/issues/10667)) +- Update `complement.sh` to rebuild the base Docker image when run with workers. ([\#10686](https://github.com/matrix-org/synapse/issues/10686)) +- Split the event-processing methods in `FederationHandler` into a separate `FederationEventHandler`. ([\#10692](https://github.com/matrix-org/synapse/issues/10692)) +- Remove unused `compare_digest` function. ([\#10706](https://github.com/matrix-org/synapse/issues/10706)) + + +Synapse 1.41.1 (2021-08-31) +=========================== + +Due to the two security issues highlighted below, server administrators are encouraged to update Synapse. We are not aware of these vulnerabilities being exploited in the wild. + +Security advisory +----------------- + +The following issues are fixed in v1.41.1. + +- **[GHSA-3x4c-pq33-4w3q](https://github.com/matrix-org/synapse/security/advisories/GHSA-3x4c-pq33-4w3q) / [CVE-2021-39164](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-39164): Enumerating a private room's list of members and their display names.** + + If an unauthorized user both knows the Room ID of a private room *and* that room's history visibility is set to `shared`, then they may be able to enumerate the room's members, including their display names. + + The unauthorized user must be on the same homeserver as a user who is a member of the target room. + + Fixed by [52c7a51cf](https://github.com/matrix-org/synapse/commit/52c7a51cf). + +- **[GHSA-jj53-8fmw-f2w2](https://github.com/matrix-org/synapse/security/advisories/GHSA-jj53-8fmw-f2w2) / [CVE-2021-39163](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-39163): Disclosing a private room's name, avatar, topic, and number of members.** + + If an unauthorized user knows the Room ID of a private room, then its name, avatar, topic, and number of members may be disclosed through Group / Community features. + + The unauthorized user must be on the same homeserver as a user who is a member of the target room, and their homeserver must allow non-administrators to create groups (`enable_group_creation` in the Synapse configuration; off by default). + + Fixed by [cb35df940a](https://github.com/matrix-org/synapse/commit/cb35df940a), [\#10723](https://github.com/matrix-org/synapse/issues/10723). + +Bugfixes +-------- + +- Fix a regression introduced in Synapse 1.41 which broke email transmission on systems using older versions of the Twisted library. ([\#10713](https://github.com/matrix-org/synapse/issues/10713)) + +Synapse 1.41.0 (2021-08-24) +=========================== + +This release adds support for Debian 12 (Bookworm), but **removes support for Ubuntu 20.10 (Groovy Gorilla)**, which reached End of Life last month. + +Note that when using workers the `/_synapse/admin/v1/users/{userId}/media` must now be handled by media workers. See the [upgrade notes](https://matrix-org.github.io/synapse/latest/upgrade.html) for more information. + + +Features +-------- + +- Enable room capabilities ([MSC3244](https://github.com/matrix-org/matrix-doc/pull/3244)) by default and set room version 8 as the preferred room version when creating restricted rooms. ([\#10571](https://github.com/matrix-org/synapse/issues/10571)) + + +Synapse 1.41.0rc1 (2021-08-18) +============================== + +Features +-------- + +- Add `get_userinfo_by_id` method to ModuleApi. ([\#9581](https://github.com/matrix-org/synapse/issues/9581)) +- Initial local support for [MSC3266](https://github.com/matrix-org/synapse/pull/10394), Room Summary over the unstable `/rooms/{roomIdOrAlias}/summary` API. ([\#10394](https://github.com/matrix-org/synapse/issues/10394)) +- Experimental support for [MSC3288](https://github.com/matrix-org/matrix-doc/pull/3288), sending `room_type` to the identity server for 3pid invites over the `/store-invite` API. ([\#10435](https://github.com/matrix-org/synapse/issues/10435)) +- Add support for sending federation requests through a proxy. Contributed by @Bubu and @dklimpel. See the [upgrade notes](https://matrix-org.github.io/synapse/latest/upgrade.html) for more information. ([\#10596](https://github.com/matrix-org/synapse/issues/10596)). ([\#10475](https://github.com/matrix-org/synapse/issues/10475)) +- Add support for "marker" events which makes historical events discoverable for servers that already have all of the scrollback history (part of [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716)). ([\#10498](https://github.com/matrix-org/synapse/issues/10498)) +- Add a configuration setting for the time a `/sync` response is cached for. ([\#10513](https://github.com/matrix-org/synapse/issues/10513)) +- The default logging handler for new installations is now `PeriodicallyFlushingMemoryHandler`, a buffered logging handler which periodically flushes itself. ([\#10518](https://github.com/matrix-org/synapse/issues/10518)) +- Add support for new redaction rules for historical events specified in [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716). ([\#10538](https://github.com/matrix-org/synapse/issues/10538)) +- Add a setting to disable TLS when sending email. ([\#10546](https://github.com/matrix-org/synapse/issues/10546)) +- Add pagination to the spaces summary based on updates to [MSC2946](https://github.com/matrix-org/matrix-doc/pull/2946). ([\#10549](https://github.com/matrix-org/synapse/issues/10549), [\#10560](https://github.com/matrix-org/synapse/issues/10560), [\#10569](https://github.com/matrix-org/synapse/issues/10569), [\#10574](https://github.com/matrix-org/synapse/issues/10574), [\#10575](https://github.com/matrix-org/synapse/issues/10575), [\#10579](https://github.com/matrix-org/synapse/issues/10579), [\#10583](https://github.com/matrix-org/synapse/issues/10583)) +- Admin API to delete several media for a specific user. Contributed by @dklimpel. ([\#10558](https://github.com/matrix-org/synapse/issues/10558), [\#10628](https://github.com/matrix-org/synapse/issues/10628)) +- Add support for routing `/createRoom` to workers. ([\#10564](https://github.com/matrix-org/synapse/issues/10564)) +- Update the Synapse Grafana dashboard. ([\#10570](https://github.com/matrix-org/synapse/issues/10570)) +- Add an admin API (`GET /_synapse/admin/username_available`) to check if a username is available (regardless of registration settings). ([\#10578](https://github.com/matrix-org/synapse/issues/10578)) +- Allow editing a user's `external_ids` via the "Edit User" admin API. Contributed by @dklimpel. ([\#10598](https://github.com/matrix-org/synapse/issues/10598)) +- The Synapse manhole no longer needs coroutines to be wrapped in `defer.ensureDeferred`. ([\#10602](https://github.com/matrix-org/synapse/issues/10602)) +- Add option to allow modules to run periodic tasks on all instances, rather than just the one configured to run background tasks. ([\#10638](https://github.com/matrix-org/synapse/issues/10638)) + + +Bugfixes +-------- + +- Add some clarification to the sample config file. Contributed by @Kentokamoto. ([\#10129](https://github.com/matrix-org/synapse/issues/10129)) +- Fix a long-standing bug where protocols which are not implemented by any appservices were incorrectly returned via `GET /_matrix/client/r0/thirdparty/protocols`. ([\#10532](https://github.com/matrix-org/synapse/issues/10532)) +- Fix exceptions in logs when failing to get remote room list. ([\#10541](https://github.com/matrix-org/synapse/issues/10541)) +- Fix longstanding bug which caused the user's presence "status message" to be reset when the user went offline. Contributed by @dklimpel. ([\#10550](https://github.com/matrix-org/synapse/issues/10550)) +- Allow public rooms to be previewed in the spaces summary APIs from [MSC2946](https://github.com/matrix-org/matrix-doc/pull/2946). ([\#10580](https://github.com/matrix-org/synapse/issues/10580)) +- Fix a bug introduced in v1.37.1 where an error could occur in the asynchronous processing of PDUs when the queue was empty. ([\#10592](https://github.com/matrix-org/synapse/issues/10592)) +- Fix errors on /sync when read receipt data is a string. Only affects homeservers with the experimental flag for [MSC2285](https://github.com/matrix-org/matrix-doc/pull/2285) enabled. Contributed by @SimonBrandner. ([\#10606](https://github.com/matrix-org/synapse/issues/10606)) +- Additional validation for the spaces summary API to avoid errors like `ValueError: Stop argument for islice() must be None or an integer`. The missing validation has existed since v1.31.0. ([\#10611](https://github.com/matrix-org/synapse/issues/10611)) +- Revert behaviour introduced in v1.38.0 that strips `org.matrix.msc2732.device_unused_fallback_key_types` from `/sync` when its value is empty. This field should instead always be present according to [MSC2732](https://github.com/matrix-org/matrix-doc/blob/master/proposals/2732-olm-fallback-keys.md). ([\#10623](https://github.com/matrix-org/synapse/issues/10623)) + + +Improved Documentation +---------------------- + +- Add documentation for configuring a forward proxy. ([\#10443](https://github.com/matrix-org/synapse/issues/10443)) +- Updated the reverse proxy documentation to highlight the homserver configuration that is needed to make Synapse aware that is is intentionally reverse proxied. ([\#10551](https://github.com/matrix-org/synapse/issues/10551)) +- Update CONTRIBUTING.md to fix index links and the instructions for SyTest in docker. ([\#10599](https://github.com/matrix-org/synapse/issues/10599)) + + +Deprecations and Removals +------------------------- + +- No longer build `.deb` packages for Ubuntu 20.10 Groovy Gorilla, which has now EOLed. ([\#10588](https://github.com/matrix-org/synapse/issues/10588)) +- The `template_dir` configuration settings in the `sso`, `account_validity` and `email` sections of the configuration file are now deprecated in favour of the global `templates.custom_template_directory` setting. See the [upgrade notes](https://matrix-org.github.io/synapse/latest/upgrade.html) for more information. ([\#10596](https://github.com/matrix-org/synapse/issues/10596)) + + +Internal Changes +---------------- + +- Improve event caching mechanism to avoid having multiple copies of an event in memory at a time. ([\#10119](https://github.com/matrix-org/synapse/issues/10119)) +- Reduce errors in PostgreSQL logs due to concurrent serialization errors. ([\#10504](https://github.com/matrix-org/synapse/issues/10504)) +- Include room ID in ignored EDU log messages. Contributed by @ilmari. ([\#10507](https://github.com/matrix-org/synapse/issues/10507)) +- Add pagination to the spaces summary based on updates to [MSC2946](https://github.com/matrix-org/matrix-doc/pull/2946). ([\#10527](https://github.com/matrix-org/synapse/issues/10527), [\#10530](https://github.com/matrix-org/synapse/issues/10530)) +- Fix CI to not break when run against branches rather than pull requests. ([\#10529](https://github.com/matrix-org/synapse/issues/10529)) +- Mark all events stemming from the [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) `/batch_send` endpoint as historical. ([\#10537](https://github.com/matrix-org/synapse/issues/10537)) +- Clean up some of the federation event authentication code for clarity. ([\#10539](https://github.com/matrix-org/synapse/issues/10539), [\#10591](https://github.com/matrix-org/synapse/issues/10591)) +- Convert `Transaction` and `Edu` objects to attrs. ([\#10542](https://github.com/matrix-org/synapse/issues/10542)) +- Update `/batch_send` endpoint to only return `state_events` created by the `state_events_from_before` passed in. ([\#10552](https://github.com/matrix-org/synapse/issues/10552)) +- Update contributing.md to warn against rebasing an open PR. ([\#10563](https://github.com/matrix-org/synapse/issues/10563)) +- Remove the unused public rooms replication stream. ([\#10565](https://github.com/matrix-org/synapse/issues/10565)) +- Clarify error message when failing to join a restricted room. ([\#10572](https://github.com/matrix-org/synapse/issues/10572)) +- Remove references to BuildKite in favour of GitHub Actions. ([\#10573](https://github.com/matrix-org/synapse/issues/10573)) +- Move `/batch_send` endpoint defined by [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) to the `/v2_alpha` directory. ([\#10576](https://github.com/matrix-org/synapse/issues/10576)) +- Allow multiple custom directories in `read_templates`. ([\#10587](https://github.com/matrix-org/synapse/issues/10587)) +- Re-organize the `synapse.federation.transport.server` module to create smaller files. ([\#10590](https://github.com/matrix-org/synapse/issues/10590)) +- Flatten the `synapse.rest.client` package by moving the contents of `v1` and `v2_alpha` into the parent. ([\#10600](https://github.com/matrix-org/synapse/issues/10600)) +- Build Debian packages for Debian 12 (Bookworm). ([\#10612](https://github.com/matrix-org/synapse/issues/10612)) +- Fix up a couple of links to the database schema documentation. ([\#10620](https://github.com/matrix-org/synapse/issues/10620)) +- Fix a broken link to the upgrade notes. ([\#10631](https://github.com/matrix-org/synapse/issues/10631)) + + +Synapse 1.40.0 (2021-08-10) +=========================== + +No significant changes. + + +Synapse 1.40.0rc3 (2021-08-09) +============================== + +Features +-------- + +- Support [MSC3289: room version 8](https://github.com/matrix-org/matrix-doc/pull/3289). ([\#10449](https://github.com/matrix-org/synapse/issues/10449)) + + +Bugfixes +-------- + +- Mark the experimental room version from [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) as unstable. ([\#10449](https://github.com/matrix-org/synapse/issues/10449)) + + +Improved Documentation +---------------------- + +- Fix broken links in `upgrade.md`. Contributed by @dklimpel. ([\#10543](https://github.com/matrix-org/synapse/issues/10543)) + + +Synapse 1.40.0rc2 (2021-08-04) +============================== + +Bugfixes +-------- + +- Fix the `PeriodicallyFlushingMemoryHandler` inhibiting application shutdown because of its background thread. ([\#10517](https://github.com/matrix-org/synapse/issues/10517)) +- Fix a bug introduced in Synapse v1.40.0rc1 that could cause Synapse to respond with an error when clients would update read receipts. ([\#10531](https://github.com/matrix-org/synapse/issues/10531)) + + +Internal Changes +---------------- + +- Fix release script to open the correct URL for the release. ([\#10516](https://github.com/matrix-org/synapse/issues/10516)) + + +Synapse 1.40.0rc1 (2021-08-03) +============================== + +Features +-------- + +- Add support for [MSC2033](https://github.com/matrix-org/matrix-doc/pull/2033): `device_id` on `/account/whoami`. ([\#9918](https://github.com/matrix-org/synapse/issues/9918)) +- Update support for [MSC2716 - Incrementally importing history into existing rooms](https://github.com/matrix-org/matrix-doc/pull/2716). ([\#10245](https://github.com/matrix-org/synapse/issues/10245), [\#10432](https://github.com/matrix-org/synapse/issues/10432), [\#10463](https://github.com/matrix-org/synapse/issues/10463)) +- Update support for [MSC3083](https://github.com/matrix-org/matrix-doc/pull/3083) to consider changes in the MSC around which servers can issue join events. ([\#10254](https://github.com/matrix-org/synapse/issues/10254), [\#10447](https://github.com/matrix-org/synapse/issues/10447), [\#10489](https://github.com/matrix-org/synapse/issues/10489)) +- Initial support for [MSC3244](https://github.com/matrix-org/matrix-doc/pull/3244), Room version capabilities over the /capabilities API. ([\#10283](https://github.com/matrix-org/synapse/issues/10283)) +- Add a buffered logging handler which periodically flushes itself. ([\#10407](https://github.com/matrix-org/synapse/issues/10407), [\#10515](https://github.com/matrix-org/synapse/issues/10515)) +- Add support for https connections to a proxy server. Contributed by @Bubu and @dklimpel. ([\#10411](https://github.com/matrix-org/synapse/issues/10411)) +- Support for [MSC2285 (hidden read receipts)](https://github.com/matrix-org/matrix-doc/pull/2285). Contributed by @SimonBrandner. ([\#10413](https://github.com/matrix-org/synapse/issues/10413)) +- Email notifications now state whether an invitation is to a room or a space. ([\#10426](https://github.com/matrix-org/synapse/issues/10426)) +- Allow setting transaction limit for database connections. ([\#10440](https://github.com/matrix-org/synapse/issues/10440), [\#10511](https://github.com/matrix-org/synapse/issues/10511)) +- Add `creation_ts` to "list users" admin API. ([\#10448](https://github.com/matrix-org/synapse/issues/10448)) + + +Bugfixes +-------- + +- Improve character set detection in URL previews by supporting underscores (in addition to hyphens). Contributed by @srividyut. ([\#10410](https://github.com/matrix-org/synapse/issues/10410)) +- Fix events being incorrectly rejected over federation if they reference auth events that the server needed to fetch. ([\#10439](https://github.com/matrix-org/synapse/issues/10439)) +- Fix `synapse_federation_server_oldest_inbound_pdu_in_staging` Prometheus metric to not report a max age of 51 years when the queue is empty. ([\#10455](https://github.com/matrix-org/synapse/issues/10455)) +- Fix a bug which caused an explicit assignment of power-level 0 to a user to be misinterpreted in rare circumstances. ([\#10499](https://github.com/matrix-org/synapse/issues/10499)) + + +Improved Documentation +---------------------- + +- Fix hierarchy of providers on the OpenID page. ([\#10445](https://github.com/matrix-org/synapse/issues/10445)) +- Consolidate development documentation to `docs/development/`. ([\#10453](https://github.com/matrix-org/synapse/issues/10453)) +- Add some developer docs to explain room DAG concepts like `outliers`, `state_groups`, `depth`, etc. ([\#10464](https://github.com/matrix-org/synapse/issues/10464)) +- Document how to use Complement while developing a new Synapse feature. ([\#10483](https://github.com/matrix-org/synapse/issues/10483)) + + +Internal Changes +---------------- + +- Prune inbound federation queues for a room if they get too large. ([\#10390](https://github.com/matrix-org/synapse/issues/10390)) +- Add type hints to `synapse.federation.transport.client` module. ([\#10408](https://github.com/matrix-org/synapse/issues/10408)) +- Remove shebang line from module files. ([\#10415](https://github.com/matrix-org/synapse/issues/10415)) +- Drop backwards-compatibility code that was required to support Ubuntu Xenial. ([\#10429](https://github.com/matrix-org/synapse/issues/10429)) +- Use a docker image cache for the prerequisites for the debian package build. ([\#10431](https://github.com/matrix-org/synapse/issues/10431)) +- Improve servlet type hints. ([\#10437](https://github.com/matrix-org/synapse/issues/10437), [\#10438](https://github.com/matrix-org/synapse/issues/10438)) +- Replace usage of `or_ignore` in `simple_insert` with `simple_upsert` usage, to stop spamming postgres logs with spurious ERROR messages. ([\#10442](https://github.com/matrix-org/synapse/issues/10442)) +- Update the `tests-done` Github Actions status. ([\#10444](https://github.com/matrix-org/synapse/issues/10444), [\#10512](https://github.com/matrix-org/synapse/issues/10512)) +- Update type annotations to work with forthcoming Twisted 21.7.0 release. ([\#10446](https://github.com/matrix-org/synapse/issues/10446), [\#10450](https://github.com/matrix-org/synapse/issues/10450)) +- Cancel redundant GHA workflows when a new commit is pushed. ([\#10451](https://github.com/matrix-org/synapse/issues/10451)) +- Mitigate media repo XSS attacks on IE11 via the non-standard X-Content-Security-Policy header. ([\#10468](https://github.com/matrix-org/synapse/issues/10468)) +- Additional type hints in the state handler. ([\#10482](https://github.com/matrix-org/synapse/issues/10482)) +- Update syntax used to run complement tests. ([\#10488](https://github.com/matrix-org/synapse/issues/10488)) +- Fix up type annotations to work with Twisted 21.7. ([\#10490](https://github.com/matrix-org/synapse/issues/10490)) +- Improve type annotations for `ObservableDeferred`. ([\#10491](https://github.com/matrix-org/synapse/issues/10491)) +- Extend release script to also tag and create GitHub releases. ([\#10496](https://github.com/matrix-org/synapse/issues/10496)) +- Fix a bug which caused production debian packages to be incorrectly marked as 'prerelease'. ([\#10500](https://github.com/matrix-org/synapse/issues/10500)) + + +Synapse 1.39.0 (2021-07-29) +=========================== + +No significant changes. + + +Synapse 1.39.0rc3 (2021-07-28) +============================== + +Bugfixes +-------- + +- Fix a bug introduced in Synapse 1.38 which caused an exception at startup when SAML authentication was enabled. ([\#10477](https://github.com/matrix-org/synapse/issues/10477)) +- Fix a long-standing bug where Synapse would not inform clients that a device had exhausted its one-time-key pool, potentially causing problems decrypting events. ([\#10485](https://github.com/matrix-org/synapse/issues/10485)) +- Fix reporting old R30 stats as R30v2 stats. Introduced in v1.39.0rc1. ([\#10486](https://github.com/matrix-org/synapse/issues/10486)) + + +Internal Changes +---------------- + +- Fix an error which prevented the Github Actions workflow to build the docker images from running. ([\#10461](https://github.com/matrix-org/synapse/issues/10461)) +- Fix release script to correctly version debian changelog when doing RCs. ([\#10465](https://github.com/matrix-org/synapse/issues/10465)) + + +Synapse 1.39.0rc2 (2021-07-22) +============================== + +This release also includes the changes in v1.38.1. + + +Internal Changes +---------------- + +- Move docker image build to Github Actions. ([\#10416](https://github.com/matrix-org/synapse/issues/10416)) + + +Synapse 1.38.1 (2021-07-22) +=========================== + +Bugfixes +-------- + +- Always include `device_one_time_keys_count` key in `/sync` response to work around a bug in Element Android that broke encryption for new devices. ([\#10457](https://github.com/matrix-org/synapse/issues/10457)) + + +Synapse 1.39.0rc1 (2021-07-20) +============================== + +The Third-Party Event Rules module interface has been deprecated in favour of the generic module interface introduced in Synapse v1.37.0. Support for the old interface is planned to be removed in September 2021. See the [upgrade notes](https://matrix-org.github.io/synapse/latest/upgrade.html#upgrading-to-v1390) for more information. + +Features +-------- + +- Add the ability to override the account validity feature with a module. ([\#9884](https://github.com/matrix-org/synapse/issues/9884)) +- The spaces summary API now returns any joinable rooms, not only rooms which are world-readable. ([\#10298](https://github.com/matrix-org/synapse/issues/10298), [\#10305](https://github.com/matrix-org/synapse/issues/10305)) +- Add a new version of the R30 phone-home metric, which removes a false impression of retention given by the old R30 metric. ([\#10332](https://github.com/matrix-org/synapse/issues/10332), [\#10427](https://github.com/matrix-org/synapse/issues/10427)) +- Allow providing credentials to `http_proxy`. ([\#10360](https://github.com/matrix-org/synapse/issues/10360)) + + +Bugfixes +-------- + +- Fix error while dropping locks on shutdown. Introduced in v1.38.0. ([\#10433](https://github.com/matrix-org/synapse/issues/10433)) +- Add base starting insertion event when no chunk ID is specified in the historical batch send API. ([\#10250](https://github.com/matrix-org/synapse/issues/10250)) +- Fix historical batch send endpoint (MSC2716) rejecting batches with messages from multiple senders. ([\#10276](https://github.com/matrix-org/synapse/issues/10276)) +- Fix purging rooms that other homeservers are still sending events for. Contributed by @ilmari. ([\#10317](https://github.com/matrix-org/synapse/issues/10317)) +- Fix errors during backfill caused by previously purged redaction events. Contributed by Andreas Rammhold (@andir). ([\#10343](https://github.com/matrix-org/synapse/issues/10343)) +- Fix the user directory becoming broken (and noisy errors being logged) when knocking and room statistics are in use. ([\#10344](https://github.com/matrix-org/synapse/issues/10344)) +- Fix newly added `synapse_federation_server_oldest_inbound_pdu_in_staging` prometheus metric to measure age rather than timestamp. ([\#10355](https://github.com/matrix-org/synapse/issues/10355)) +- Fix PostgreSQL sometimes using table scans for queries against `state_groups_state` table, taking a long time and a large amount of IO. ([\#10359](https://github.com/matrix-org/synapse/issues/10359)) +- Fix `make_room_admin` failing for users that have left a private room. ([\#10367](https://github.com/matrix-org/synapse/issues/10367)) +- Fix a number of logged errors caused by remote servers being down. ([\#10400](https://github.com/matrix-org/synapse/issues/10400), [\#10414](https://github.com/matrix-org/synapse/issues/10414)) +- Responses from `/make_{join,leave,knock}` no longer include signatures, which will turn out to be invalid after events are returned to `/send_{join,leave,knock}`. ([\#10404](https://github.com/matrix-org/synapse/issues/10404)) + + +Improved Documentation +---------------------- + +- Updated installation dependencies for newer macOS versions and ARM Macs. Contributed by Luke Walsh. ([\#9971](https://github.com/matrix-org/synapse/issues/9971)) +- Simplify structure of room admin API. ([\#10313](https://github.com/matrix-org/synapse/issues/10313)) +- Refresh the logcontext dev documentation. ([\#10353](https://github.com/matrix-org/synapse/issues/10353)), ([\#10337](https://github.com/matrix-org/synapse/issues/10337)) +- Add delegation example for caddy in the reverse proxy documentation. Contributed by @moritzdietz. ([\#10368](https://github.com/matrix-org/synapse/issues/10368)) +- Fix and clarify some links in `docs` and `contrib`. ([\#10370](https://github.com/matrix-org/synapse/issues/10370)), ([\#10322](https://github.com/matrix-org/synapse/issues/10322)), ([\#10399](https://github.com/matrix-org/synapse/issues/10399)) +- Make deprecation notice of the spam checker doc more obvious. ([\#10395](https://github.com/matrix-org/synapse/issues/10395)) +- Add instructions on installing Debian packages for release candidates. ([\#10396](https://github.com/matrix-org/synapse/issues/10396)) + + +Deprecations and Removals +------------------------- + +- Remove functionality associated with the unused `room_stats_historical` and `user_stats_historical` tables. Contributed by @xmunoz. ([\#9721](https://github.com/matrix-org/synapse/issues/9721)) +- The third-party event rules module interface is deprecated in favour of the generic module interface introduced in Synapse v1.37.0. See the [upgrade notes](https://matrix-org.github.io/synapse/latest/upgrade.html#upgrading-to-v1390) for more information. ([\#10386](https://github.com/matrix-org/synapse/issues/10386)) + + +Internal Changes +---------------- + +- Convert `room_depth.min_depth` column to a `BIGINT`. ([\#10289](https://github.com/matrix-org/synapse/issues/10289)) +- Add tests to characterise the current behaviour of R30 phone-home metrics. ([\#10315](https://github.com/matrix-org/synapse/issues/10315)) +- Rebuild event context and auth when processing specific results from `ThirdPartyEventRules` modules. ([\#10316](https://github.com/matrix-org/synapse/issues/10316)) +- Minor change to the code that populates `user_daily_visits`. ([\#10324](https://github.com/matrix-org/synapse/issues/10324)) +- Re-enable Sytests that were disabled for the 1.37.1 release. ([\#10345](https://github.com/matrix-org/synapse/issues/10345), [\#10357](https://github.com/matrix-org/synapse/issues/10357)) +- Run `pyupgrade` on the codebase. ([\#10347](https://github.com/matrix-org/synapse/issues/10347), [\#10348](https://github.com/matrix-org/synapse/issues/10348)) +- Switch `application_services_txns.txn_id` database column to `BIGINT`. ([\#10349](https://github.com/matrix-org/synapse/issues/10349)) +- Convert internal type variable syntax to reflect wider ecosystem use. ([\#10350](https://github.com/matrix-org/synapse/issues/10350), [\#10380](https://github.com/matrix-org/synapse/issues/10380), [\#10381](https://github.com/matrix-org/synapse/issues/10381), [\#10382](https://github.com/matrix-org/synapse/issues/10382), [\#10418](https://github.com/matrix-org/synapse/issues/10418)) +- Make the Github Actions workflow configuration more efficient. ([\#10383](https://github.com/matrix-org/synapse/issues/10383)) +- Add type hints to `get_{domain,localpart}_from_id`. ([\#10385](https://github.com/matrix-org/synapse/issues/10385)) +- When building Debian packages for prerelease versions, set the Section accordingly. ([\#10391](https://github.com/matrix-org/synapse/issues/10391)) +- Add type hints and comments to event auth code. ([\#10393](https://github.com/matrix-org/synapse/issues/10393)) +- Stagger sending of presence update to remote servers, reducing CPU spikes caused by starting many connections to remote servers at once. ([\#10398](https://github.com/matrix-org/synapse/issues/10398)) +- Remove unused `events_by_room` code (tech debt). ([\#10421](https://github.com/matrix-org/synapse/issues/10421)) +- Add a github actions job which records success of other jobs. ([\#10430](https://github.com/matrix-org/synapse/issues/10430)) + + +Synapse 1.38.0 (2021-07-13) +=========================== + +This release includes a database schema update which could result in elevated disk usage. See the [upgrade notes](https://matrix-org.github.io/synapse/develop/upgrade#upgrading-to-v1380) for more information. + +No significant changes since 1.38.0rc3. + + +Synapse 1.38.0rc3 (2021-07-13) +============================== + +Internal Changes +---------------- + +- Build the Debian packages in CI. ([\#10247](https://github.com/matrix-org/synapse/issues/10247), [\#10379](https://github.com/matrix-org/synapse/issues/10379)) + + +Synapse 1.38.0rc2 (2021-07-09) +============================== + +Bugfixes +-------- + +- Fix bug where inbound federation in a room could be delayed due to not correctly dropping a lock. Introduced in v1.37.1. ([\#10336](https://github.com/matrix-org/synapse/issues/10336)) + + +Improved Documentation +---------------------- + +- Update links to documentation in the sample config. Contributed by @dklimpel. ([\#10287](https://github.com/matrix-org/synapse/issues/10287)) +- Fix broken links in [INSTALL.md](INSTALL.md). Contributed by @dklimpel. ([\#10331](https://github.com/matrix-org/synapse/issues/10331)) + + +Synapse 1.38.0rc1 (2021-07-06) +============================== + +Features +-------- + +- Implement refresh tokens as specified by [MSC2918](https://github.com/matrix-org/matrix-doc/pull/2918). ([\#9450](https://github.com/matrix-org/synapse/issues/9450)) +- Add support for evicting cache entries based on last access time. ([\#10205](https://github.com/matrix-org/synapse/issues/10205)) +- Omit empty fields from the `/sync` response. Contributed by @deepbluev7. ([\#10214](https://github.com/matrix-org/synapse/issues/10214)) +- Improve validation on federation `send_{join,leave,knock}` endpoints. ([\#10225](https://github.com/matrix-org/synapse/issues/10225), [\#10243](https://github.com/matrix-org/synapse/issues/10243)) +- Add SSO `external_ids` to the Query User Account admin API. ([\#10261](https://github.com/matrix-org/synapse/issues/10261)) +- Mark events received over federation which fail a spam check as "soft-failed". ([\#10263](https://github.com/matrix-org/synapse/issues/10263)) +- Add metrics for new inbound federation staging area. ([\#10284](https://github.com/matrix-org/synapse/issues/10284)) +- Add script to print information about recently registered users. ([\#10290](https://github.com/matrix-org/synapse/issues/10290)) + + +Bugfixes +-------- + +- Fix a long-standing bug which meant that invite rejections and knocks were not sent out over federation in a timely manner. ([\#10223](https://github.com/matrix-org/synapse/issues/10223)) +- Fix a bug introduced in v1.26.0 where only users who have set profile information could be deactivated with erasure enabled. ([\#10252](https://github.com/matrix-org/synapse/issues/10252)) +- Fix a long-standing bug where Synapse would return errors after 231 events were handled by the server. ([\#10264](https://github.com/matrix-org/synapse/issues/10264), [\#10267](https://github.com/matrix-org/synapse/issues/10267), [\#10282](https://github.com/matrix-org/synapse/issues/10282), [\#10286](https://github.com/matrix-org/synapse/issues/10286), [\#10291](https://github.com/matrix-org/synapse/issues/10291), [\#10314](https://github.com/matrix-org/synapse/issues/10314), [\#10326](https://github.com/matrix-org/synapse/issues/10326)) +- Fix the prometheus `synapse_federation_server_pdu_process_time` metric. Broke in v1.37.1. ([\#10279](https://github.com/matrix-org/synapse/issues/10279)) +- Ensure that inbound events from federation that were being processed when Synapse was restarted get promptly processed on start up. ([\#10303](https://github.com/matrix-org/synapse/issues/10303)) + + +Improved Documentation +---------------------- + +- Move the upgrade notes to [docs/upgrade.md](https://github.com/matrix-org/synapse/blob/develop/docs/upgrade.md) and convert them to markdown. ([\#10166](https://github.com/matrix-org/synapse/issues/10166)) +- Choose Welcome & Overview as the default page for synapse documentation website. ([\#10242](https://github.com/matrix-org/synapse/issues/10242)) +- Adjust the URL in the README.rst file to point to irc.libera.chat. ([\#10258](https://github.com/matrix-org/synapse/issues/10258)) +- Fix homeserver config option name in presence router documentation. ([\#10288](https://github.com/matrix-org/synapse/issues/10288)) +- Fix link pointing at the wrong section in the modules documentation page. ([\#10302](https://github.com/matrix-org/synapse/issues/10302)) + + +Internal Changes +---------------- + +- Drop `Origin` and `Accept` from the value of the `Access-Control-Allow-Headers` response header. ([\#10114](https://github.com/matrix-org/synapse/issues/10114)) +- Add type hints to the federation servlets. ([\#10213](https://github.com/matrix-org/synapse/issues/10213)) +- Improve the reliability of auto-joining remote rooms. ([\#10237](https://github.com/matrix-org/synapse/issues/10237)) +- Update the release script to use the semver terminology and determine the release branch based on the next version. ([\#10239](https://github.com/matrix-org/synapse/issues/10239)) +- Fix type hints for computing auth events. ([\#10253](https://github.com/matrix-org/synapse/issues/10253)) +- Improve the performance of the spaces summary endpoint by only recursing into spaces (and not rooms in general). ([\#10256](https://github.com/matrix-org/synapse/issues/10256)) +- Move event authentication methods from `Auth` to `EventAuthHandler`. ([\#10268](https://github.com/matrix-org/synapse/issues/10268)) +- Re-enable a SyTest after it has been fixed. ([\#10292](https://github.com/matrix-org/synapse/issues/10292)) + + +Synapse 1.37.1 (2021-06-30) +=========================== + +This release resolves issues (such as [#9490](https://github.com/matrix-org/synapse/issues/9490)) where one busy room could cause head-of-line blocking, starving Synapse from processing events in other rooms, and causing all federated traffic to fall behind. Synapse 1.37.1 processes inbound federation traffic asynchronously, ensuring that one busy room won't impact others. Please upgrade to Synapse 1.37.1 as soon as possible, in order to increase resilience to other traffic spikes. + +No significant changes since v1.37.1rc1. + + +Synapse 1.37.1rc1 (2021-06-29) +============================== + +Features +-------- + +- Handle inbound events from federation asynchronously. ([\#10269](https://github.com/matrix-org/synapse/issues/10269), [\#10272](https://github.com/matrix-org/synapse/issues/10272)) + + +Synapse 1.37.0 (2021-06-29) +=========================== + +This release deprecates the current spam checker interface. See the [upgrade notes](https://matrix-org.github.io/synapse/develop/upgrade#deprecation-of-the-current-spam-checker-interface) for more information on how to update to the new generic module interface. + +This release also removes support for fetching and renewing TLS certificates using the ACME v1 protocol, which has been fully decommissioned by Let's Encrypt on June 1st 2021. Admins previously using this feature should use a [reverse proxy](https://matrix-org.github.io/synapse/develop/reverse_proxy.html) to handle TLS termination, or use an external ACME client (such as [certbot](https://certbot.eff.org/)) to retrieve a certificate and key and provide them to Synapse using the `tls_certificate_path` and `tls_private_key_path` configuration settings. + +Synapse 1.37.0rc1 (2021-06-24) +============================== + +Features +-------- + +- Implement "room knocking" as per [MSC2403](https://github.com/matrix-org/matrix-doc/pull/2403). Contributed by @Sorunome and anoa. ([\#6739](https://github.com/matrix-org/synapse/issues/6739), [\#9359](https://github.com/matrix-org/synapse/issues/9359), [\#10167](https://github.com/matrix-org/synapse/issues/10167), [\#10212](https://github.com/matrix-org/synapse/issues/10212), [\#10227](https://github.com/matrix-org/synapse/issues/10227)) +- Add experimental support for backfilling history into rooms ([MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716)). ([\#9247](https://github.com/matrix-org/synapse/issues/9247)) +- Implement a generic interface for third-party plugin modules. ([\#10062](https://github.com/matrix-org/synapse/issues/10062), [\#10206](https://github.com/matrix-org/synapse/issues/10206)) +- Implement config option `sso.update_profile_information` to sync SSO users' profile information with the identity provider each time they login. Currently only displayname is supported. ([\#10108](https://github.com/matrix-org/synapse/issues/10108)) +- Ensure that errors during startup are written to the logs and the console. ([\#10191](https://github.com/matrix-org/synapse/issues/10191)) + + +Bugfixes +-------- + +- Fix a bug introduced in Synapse v1.25.0 that prevented the `ip_range_whitelist` configuration option from working for federation and identity servers. Contributed by @mikure. ([\#10115](https://github.com/matrix-org/synapse/issues/10115)) +- Remove a broken import line in Synapse's `admin_cmd` worker. Broke in Synapse v1.33.0. ([\#10154](https://github.com/matrix-org/synapse/issues/10154)) +- Fix a bug introduced in Synapse v1.21.0 which could cause `/sync` to return immediately with an empty response. ([\#10157](https://github.com/matrix-org/synapse/issues/10157), [\#10158](https://github.com/matrix-org/synapse/issues/10158)) +- Fix a minor bug in the response to `/_matrix/client/r0/user/{user}/openid/request_token` causing `expires_in` to be a float instead of an integer. Contributed by @lukaslihotzki. ([\#10175](https://github.com/matrix-org/synapse/issues/10175)) +- Always require users to re-authenticate for dangerous operations: deactivating an account, modifying an account password, and adding 3PIDs. ([\#10184](https://github.com/matrix-org/synapse/issues/10184)) +- Fix a bug introduced in Synpase v1.7.2 where remote server count metrics collection would be incorrectly delayed on startup. Found by @heftig. ([\#10195](https://github.com/matrix-org/synapse/issues/10195)) +- Fix a bug introduced in Synapse v1.35.1 where an `allow` key of a `m.room.join_rules` event could be applied for incorrect room versions and configurations. ([\#10208](https://github.com/matrix-org/synapse/issues/10208)) +- Fix performance regression in responding to user key requests over federation. Introduced in Synapse v1.34.0rc1. ([\#10221](https://github.com/matrix-org/synapse/issues/10221)) + + +Improved Documentation +---------------------- + +- Add a new guide to decoding request logs. ([\#8436](https://github.com/matrix-org/synapse/issues/8436)) +- Mention in the sample homeserver config that you may need to configure max upload size in your reverse proxy. Contributed by @aaronraimist. ([\#10122](https://github.com/matrix-org/synapse/issues/10122)) +- Fix broken links in documentation. ([\#10180](https://github.com/matrix-org/synapse/issues/10180)) +- Deploy a snapshot of the documentation website upon each new Synapse release. ([\#10198](https://github.com/matrix-org/synapse/issues/10198)) + + +Deprecations and Removals +------------------------- + +- The current spam checker interface is deprecated in favour of a new generic modules system. See the [upgrade notes](https://matrix-org.github.io/synapse/develop/upgrade#deprecation-of-the-current-spam-checker-interface) for more information on how to update to the new system. ([\#10062](https://github.com/matrix-org/synapse/issues/10062), [\#10210](https://github.com/matrix-org/synapse/issues/10210), [\#10238](https://github.com/matrix-org/synapse/issues/10238)) +- Stop supporting the unstable spaces prefixes from MSC1772. ([\#10161](https://github.com/matrix-org/synapse/issues/10161)) +- Remove Synapse's support for automatically fetching and renewing certificates using the ACME v1 protocol. This protocol has been fully turned off by Let's Encrypt for existing installations on June 1st 2021. Admins previously using this feature should use a [reverse proxy](https://matrix-org.github.io/synapse/develop/reverse_proxy.html) to handle TLS termination, or use an external ACME client (such as [certbot](https://certbot.eff.org/)) to retrieve a certificate and key and provide them to Synapse using the `tls_certificate_path` and `tls_private_key_path` configuration settings. ([\#10194](https://github.com/matrix-org/synapse/issues/10194)) + + +Internal Changes +---------------- + +- Update the database schema versioning to support gradual migration away from legacy tables. ([\#9933](https://github.com/matrix-org/synapse/issues/9933)) +- Add type hints to the federation servlets. ([\#10080](https://github.com/matrix-org/synapse/issues/10080)) +- Improve OpenTracing for event persistence. ([\#10134](https://github.com/matrix-org/synapse/issues/10134), [\#10193](https://github.com/matrix-org/synapse/issues/10193)) +- Clean up the interface for injecting OpenTracing over HTTP. ([\#10143](https://github.com/matrix-org/synapse/issues/10143)) +- Limit the number of in-flight `/keys/query` requests from a single device. ([\#10144](https://github.com/matrix-org/synapse/issues/10144)) +- Refactor EventPersistenceQueue. ([\#10145](https://github.com/matrix-org/synapse/issues/10145)) +- Document `SYNAPSE_TEST_LOG_LEVEL` to see the logger output when running tests. ([\#10148](https://github.com/matrix-org/synapse/issues/10148)) +- Update the Complement build tags in GitHub Actions to test currently experimental features. ([\#10155](https://github.com/matrix-org/synapse/issues/10155)) +- Add a `synapse_federation_soft_failed_events_total` metric to track how often events are soft failed. ([\#10156](https://github.com/matrix-org/synapse/issues/10156)) +- Fetch the corresponding complement branch when performing CI. ([\#10160](https://github.com/matrix-org/synapse/issues/10160)) +- Add some developer documentation about boolean columns in database schemas. ([\#10164](https://github.com/matrix-org/synapse/issues/10164)) +- Add extra logging fields to better debug where events are being soft failed. ([\#10168](https://github.com/matrix-org/synapse/issues/10168)) +- Add debug logging for when we enter and exit `Measure` blocks. ([\#10183](https://github.com/matrix-org/synapse/issues/10183)) +- Improve comments in structured logging code. ([\#10188](https://github.com/matrix-org/synapse/issues/10188)) +- Update [MSC3083](https://github.com/matrix-org/matrix-doc/pull/3083) support with modifications from the MSC. ([\#10189](https://github.com/matrix-org/synapse/issues/10189)) +- Remove redundant DNS lookup limiter. ([\#10190](https://github.com/matrix-org/synapse/issues/10190)) +- Upgrade `black` linting tool to 21.6b0. ([\#10197](https://github.com/matrix-org/synapse/issues/10197)) +- Expose OpenTracing trace id in response headers. ([\#10199](https://github.com/matrix-org/synapse/issues/10199)) + + +Synapse 1.36.0 (2021-06-15) +=========================== + +No significant changes. + + +Synapse 1.36.0rc2 (2021-06-11) +============================== + +Bugfixes +-------- + +- Fix a bug which caused presence updates to stop working some time after a restart, when using a presence writer worker. Broke in v1.33.0. ([\#10149](https://github.com/matrix-org/synapse/issues/10149)) +- Fix a bug when using federation sender worker where it would send out more presence updates than necessary, leading to high resource usage. Broke in v1.33.0. ([\#10163](https://github.com/matrix-org/synapse/issues/10163)) +- Fix a bug where Synapse could send the same presence update to a remote twice. ([\#10165](https://github.com/matrix-org/synapse/issues/10165)) + + +Synapse 1.36.0rc1 (2021-06-08) +============================== + +Features +-------- + +- Add new endpoint `/_matrix/client/r0/rooms/{roomId}/aliases` from Client-Server API r0.6.1 (previously [MSC2432](https://github.com/matrix-org/matrix-doc/pull/2432)). ([\#9224](https://github.com/matrix-org/synapse/issues/9224)) +- Improve performance of incoming federation transactions in large rooms. ([\#9953](https://github.com/matrix-org/synapse/issues/9953), [\#9973](https://github.com/matrix-org/synapse/issues/9973)) +- Rewrite logic around verifying JSON object and fetching server keys to be more performant and use less memory. ([\#10035](https://github.com/matrix-org/synapse/issues/10035)) +- Add new admin APIs for unprotecting local media from quarantine. Contributed by @dklimpel. ([\#10040](https://github.com/matrix-org/synapse/issues/10040)) +- Add new admin APIs to remove media by media ID from quarantine. Contributed by @dklimpel. ([\#10044](https://github.com/matrix-org/synapse/issues/10044)) +- Make reason and score parameters optional for reporting content. Implements [MSC2414](https://github.com/matrix-org/matrix-doc/pull/2414). Contributed by Callum Brown. ([\#10077](https://github.com/matrix-org/synapse/issues/10077)) +- Add support for routing more requests to workers. ([\#10084](https://github.com/matrix-org/synapse/issues/10084)) +- Report OpenTracing spans for database activity. ([\#10113](https://github.com/matrix-org/synapse/issues/10113), [\#10136](https://github.com/matrix-org/synapse/issues/10136), [\#10141](https://github.com/matrix-org/synapse/issues/10141)) +- Significantly reduce memory usage of joining large remote rooms. ([\#10117](https://github.com/matrix-org/synapse/issues/10117)) + + +Bugfixes +-------- + +- Fixed a bug causing replication requests to fail when receiving a lot of events via federation. ([\#10082](https://github.com/matrix-org/synapse/issues/10082)) +- Fix a bug in the `force_tracing_for_users` option introduced in Synapse v1.35 which meant that the OpenTracing spans produced were missing most tags. ([\#10092](https://github.com/matrix-org/synapse/issues/10092)) +- Fixed a bug that could cause Synapse to stop notifying application services. Contributed by Willem Mulder. ([\#10107](https://github.com/matrix-org/synapse/issues/10107)) +- Fix bug where the server would attempt to fetch the same history in the room from a remote server multiple times in parallel. ([\#10116](https://github.com/matrix-org/synapse/issues/10116)) +- Fix a bug introduced in Synapse 1.33.0 which caused replication requests to fail when receiving a lot of very large events via federation. ([\#10118](https://github.com/matrix-org/synapse/issues/10118)) +- Fix bug when using workers where pagination requests failed if a remote server returned zero events from `/backfill`. Introduced in 1.35.0. ([\#10133](https://github.com/matrix-org/synapse/issues/10133)) + + +Improved Documentation +---------------------- + +- Clarify security note regarding hosting Synapse on the same domain as other web applications. ([\#9221](https://github.com/matrix-org/synapse/issues/9221)) +- Update CAPTCHA documentation to mention turning off the verify origin feature. Contributed by @aaronraimist. ([\#10046](https://github.com/matrix-org/synapse/issues/10046)) +- Tweak wording of database recommendation in `INSTALL.md`. Contributed by @aaronraimist. ([\#10057](https://github.com/matrix-org/synapse/issues/10057)) +- Add initial infrastructure for rendering Synapse documentation with mdbook. ([\#10086](https://github.com/matrix-org/synapse/issues/10086)) +- Convert the remaining Admin API documentation files to markdown. ([\#10089](https://github.com/matrix-org/synapse/issues/10089)) +- Make a link in docs use HTTPS. Contributed by @RhnSharma. ([\#10130](https://github.com/matrix-org/synapse/issues/10130)) +- Fix broken link in Docker docs. ([\#10132](https://github.com/matrix-org/synapse/issues/10132)) + + +Deprecations and Removals +------------------------- + +- Remove the experimental `spaces_enabled` flag. The spaces features are always available now. ([\#10063](https://github.com/matrix-org/synapse/issues/10063)) + + +Internal Changes +---------------- + +- Tell CircleCI to build Docker images from `main` branch. ([\#9906](https://github.com/matrix-org/synapse/issues/9906)) +- Simplify naming convention for release branches to only include the major and minor version numbers. ([\#10013](https://github.com/matrix-org/synapse/issues/10013)) +- Add `parse_strings_from_args` for parsing an array from query parameters. ([\#10048](https://github.com/matrix-org/synapse/issues/10048), [\#10137](https://github.com/matrix-org/synapse/issues/10137)) +- Remove some dead code regarding TLS certificate handling. ([\#10054](https://github.com/matrix-org/synapse/issues/10054)) +- Remove redundant, unmaintained `convert_server_keys` script. ([\#10055](https://github.com/matrix-org/synapse/issues/10055)) +- Improve the error message printed by synctl when synapse fails to start. ([\#10059](https://github.com/matrix-org/synapse/issues/10059)) +- Fix GitHub Actions lint for newsfragments. ([\#10069](https://github.com/matrix-org/synapse/issues/10069)) +- Update opentracing to inject the right context into the carrier. ([\#10074](https://github.com/matrix-org/synapse/issues/10074)) +- Fix up `BatchingQueue` implementation. ([\#10078](https://github.com/matrix-org/synapse/issues/10078)) +- Log method and path when dropping request due to size limit. ([\#10091](https://github.com/matrix-org/synapse/issues/10091)) +- In Github Actions workflows, summarize the Sytest results in an easy-to-read format. ([\#10094](https://github.com/matrix-org/synapse/issues/10094)) +- Make `/sync` do fewer state resolutions. ([\#10102](https://github.com/matrix-org/synapse/issues/10102)) +- Add missing type hints to the admin API servlets. ([\#10105](https://github.com/matrix-org/synapse/issues/10105)) +- Improve opentracing annotations for `Notifier`. ([\#10111](https://github.com/matrix-org/synapse/issues/10111)) +- Enable Prometheus metrics for the jaeger client library. ([\#10112](https://github.com/matrix-org/synapse/issues/10112)) +- Work to improve the responsiveness of `/sync` requests. ([\#10124](https://github.com/matrix-org/synapse/issues/10124)) +- OpenTracing: use a consistent name for background processes. ([\#10135](https://github.com/matrix-org/synapse/issues/10135)) + + +Synapse 1.35.1 (2021-06-03) +=========================== + +Bugfixes +-------- + +- Fix a bug introduced in v1.35.0 where invite-only rooms would be shown to all users in a space, regardless of if the user had access to it. ([\#10109](https://github.com/matrix-org/synapse/issues/10109)) + + +Synapse 1.35.0 (2021-06-01) +=========================== + +Note that [the tag](https://github.com/matrix-org/synapse/releases/tag/v1.35.0rc3) and [docker images](https://hub.docker.com/layers/matrixdotorg/synapse/v1.35.0rc3/images/sha256-34ccc87bd99a17e2cbc0902e678b5937d16bdc1991ead097eee6096481ecf2c4?context=explore) for `v1.35.0rc3` were incorrectly built. If you are experiencing issues with either, it is recommended to upgrade to the equivalent tag or docker image for the `v1.35.0` release. + +Deprecations and Removals +------------------------- + +- The core Synapse development team plan to drop support for the [unstable API of MSC2858](https://github.com/matrix-org/matrix-doc/blob/master/proposals/2858-Multiple-SSO-Identity-Providers.md#unstable-prefix), including the undocumented `experimental.msc2858_enabled` config option, in August 2021. Client authors should ensure that their clients are updated to use the stable API (which has been supported since Synapse 1.30) well before that time, to give their users time to upgrade. ([\#10101](https://github.com/matrix-org/synapse/issues/10101)) + +Bugfixes +-------- + +- Fixed a bug causing replication requests to fail when receiving a lot of events via federation. Introduced in v1.33.0. ([\#10082](https://github.com/matrix-org/synapse/issues/10082)) +- Fix HTTP response size limit to allow joining very large rooms over federation. Introduced in v1.33.0. ([\#10093](https://github.com/matrix-org/synapse/issues/10093)) + + +Internal Changes +---------------- + +- Log method and path when dropping request due to size limit. ([\#10091](https://github.com/matrix-org/synapse/issues/10091)) + + +Synapse 1.35.0rc2 (2021-05-27) +============================== + +Bugfixes +-------- + +- Fix a bug introduced in v1.35.0rc1 when calling the spaces summary API via a GET request. ([\#10079](https://github.com/matrix-org/synapse/issues/10079)) + + +Synapse 1.35.0rc1 (2021-05-25) +============================== + +Features +-------- + +- Add experimental support to allow a user who could join a restricted room to view it in the spaces summary. ([\#9922](https://github.com/matrix-org/synapse/issues/9922), [\#10007](https://github.com/matrix-org/synapse/issues/10007), [\#10038](https://github.com/matrix-org/synapse/issues/10038)) +- Reduce memory usage when joining very large rooms over federation. ([\#9958](https://github.com/matrix-org/synapse/issues/9958)) +- Add a configuration option which allows enabling opentracing by user id. ([\#9978](https://github.com/matrix-org/synapse/issues/9978)) +- Enable experimental support for [MSC2946](https://github.com/matrix-org/matrix-doc/pull/2946) (spaces summary API) and [MSC3083](https://github.com/matrix-org/matrix-doc/pull/3083) (restricted join rules) by default. ([\#10011](https://github.com/matrix-org/synapse/issues/10011)) + + +Bugfixes +-------- + +- Fix a bug introduced in v1.26.0 which meant that `synapse_port_db` would not correctly initialise some postgres sequences, requiring manual updates afterwards. ([\#9991](https://github.com/matrix-org/synapse/issues/9991)) +- Fix `synctl`'s `--no-daemonize` parameter to work correctly with worker processes. ([\#9995](https://github.com/matrix-org/synapse/issues/9995)) +- Fix a validation bug introduced in v1.34.0 in the ordering of spaces in the space summary API. ([\#10002](https://github.com/matrix-org/synapse/issues/10002)) +- Fixed deletion of new presence stream states from database. ([\#10014](https://github.com/matrix-org/synapse/issues/10014), [\#10033](https://github.com/matrix-org/synapse/issues/10033)) +- Fixed a bug with very high resolution image uploads throwing internal server errors. ([\#10029](https://github.com/matrix-org/synapse/issues/10029)) + + +Updates to the Docker image +--------------------------- + +- Fix bug introduced in Synapse 1.33.0 which caused a `Permission denied: '/homeserver.log'` error when starting Synapse with the generated log configuration. Contributed by Sergio Miguéns Iglesias. ([\#10045](https://github.com/matrix-org/synapse/issues/10045)) + + +Improved Documentation +---------------------- + +- Add hardened systemd files as proposed in [#9760](https://github.com/matrix-org/synapse/issues/9760) and added them to `contrib/`. Change the docs to reflect the presence of these files. ([\#9803](https://github.com/matrix-org/synapse/issues/9803)) +- Clarify documentation around SSO mapping providers generating unique IDs and localparts. ([\#9980](https://github.com/matrix-org/synapse/issues/9980)) +- Updates to the PostgreSQL documentation (`postgres.md`). ([\#9988](https://github.com/matrix-org/synapse/issues/9988), [\#9989](https://github.com/matrix-org/synapse/issues/9989)) +- Fix broken link in user directory documentation. Contributed by @junquera. ([\#10016](https://github.com/matrix-org/synapse/issues/10016)) +- Add missing room state entry to the table of contents of room admin API. ([\#10043](https://github.com/matrix-org/synapse/issues/10043)) + + +Deprecations and Removals +------------------------- + +- Removed support for the deprecated `tls_fingerprints` configuration setting. Contributed by Jerin J Titus. ([\#9280](https://github.com/matrix-org/synapse/issues/9280)) + + +Internal Changes +---------------- + +- Allow sending full presence to users via workers other than the one that called `ModuleApi.send_local_online_presence_to`. ([\#9823](https://github.com/matrix-org/synapse/issues/9823)) +- Update comments in the space summary handler. ([\#9974](https://github.com/matrix-org/synapse/issues/9974)) +- Minor enhancements to the `@cachedList` descriptor. ([\#9975](https://github.com/matrix-org/synapse/issues/9975)) +- Split multipart email sending into a dedicated handler. ([\#9977](https://github.com/matrix-org/synapse/issues/9977)) +- Run `black` on files in the `scripts` directory. ([\#9981](https://github.com/matrix-org/synapse/issues/9981)) +- Add missing type hints to `synapse.util` module. ([\#9982](https://github.com/matrix-org/synapse/issues/9982)) +- Simplify a few helper functions. ([\#9984](https://github.com/matrix-org/synapse/issues/9984), [\#9985](https://github.com/matrix-org/synapse/issues/9985), [\#9986](https://github.com/matrix-org/synapse/issues/9986)) +- Remove unnecessary property from SQLBaseStore. ([\#9987](https://github.com/matrix-org/synapse/issues/9987)) +- Remove `keylen` param on `LruCache`. ([\#9993](https://github.com/matrix-org/synapse/issues/9993)) +- Update the Grafana dashboard in `contrib/`. ([\#10001](https://github.com/matrix-org/synapse/issues/10001)) +- Add a batching queue implementation. ([\#10017](https://github.com/matrix-org/synapse/issues/10017)) +- Reduce memory usage when verifying signatures on large numbers of events at once. ([\#10018](https://github.com/matrix-org/synapse/issues/10018)) +- Properly invalidate caches for destination retry timings every (instead of expiring entries every 5 minutes). ([\#10036](https://github.com/matrix-org/synapse/issues/10036)) +- Fix running complement tests with Synapse workers. ([\#10039](https://github.com/matrix-org/synapse/issues/10039)) +- Fix typo in `get_state_ids_for_event` docstring where the return type was incorrect. ([\#10050](https://github.com/matrix-org/synapse/issues/10050)) + + +Synapse 1.34.0 (2021-05-17) +=========================== + +This release deprecates the `room_invite_state_types` configuration setting. See the [upgrade notes](https://github.com/matrix-org/synapse/blob/release-v1.34.0/UPGRADE.rst#upgrading-to-v1340) for instructions on updating your configuration file to use the new `room_prejoin_state` setting. + +This release also deprecates the `POST /_synapse/admin/v1/rooms//delete` admin API route. Server administrators are encouraged to update their scripts to use the new `DELETE /_synapse/admin/v1/rooms/` route instead. + + +No significant changes since v1.34.0rc1. + + +Synapse 1.34.0rc1 (2021-05-12) +============================== + +Features +-------- + +- Add experimental option to track memory usage of the caches. ([\#9881](https://github.com/matrix-org/synapse/issues/9881)) +- Add support for `DELETE /_synapse/admin/v1/rooms/`. ([\#9889](https://github.com/matrix-org/synapse/issues/9889)) +- Add limits to how often Synapse will GC, ensuring that large servers do not end up GC thrashing if `gc_thresholds` has not been correctly set. ([\#9902](https://github.com/matrix-org/synapse/issues/9902)) +- Improve performance of sending events for worker-based deployments using Redis. ([\#9905](https://github.com/matrix-org/synapse/issues/9905), [\#9950](https://github.com/matrix-org/synapse/issues/9950), [\#9951](https://github.com/matrix-org/synapse/issues/9951)) +- Improve performance after joining a large room when presence is enabled. ([\#9910](https://github.com/matrix-org/synapse/issues/9910), [\#9916](https://github.com/matrix-org/synapse/issues/9916)) +- Support stable identifiers for [MSC1772](https://github.com/matrix-org/matrix-doc/pull/1772) Spaces. `m.space.child` events will now be taken into account when populating the experimental spaces summary response. Please see [the upgrade notes](https://github.com/matrix-org/synapse/blob/release-v1.34.0/UPGRADE.rst#upgrading-to-v1340) if you have customised `room_invite_state_types` in your configuration. ([\#9915](https://github.com/matrix-org/synapse/issues/9915), [\#9966](https://github.com/matrix-org/synapse/issues/9966)) +- Improve performance of backfilling in large rooms. ([\#9935](https://github.com/matrix-org/synapse/issues/9935)) +- Add a config option to allow you to prevent device display names from being shared over federation. Contributed by @aaronraimist. ([\#9945](https://github.com/matrix-org/synapse/issues/9945)) +- Update support for [MSC2946](https://github.com/matrix-org/matrix-doc/pull/2946): Spaces Summary. ([\#9947](https://github.com/matrix-org/synapse/issues/9947), [\#9954](https://github.com/matrix-org/synapse/issues/9954)) + + +Bugfixes +-------- + +- Fix a bug introduced in v1.32.0 where the associated connection was improperly logged for SQL logging statements. ([\#9895](https://github.com/matrix-org/synapse/issues/9895)) +- Correct the type hint for the `user_may_create_room_alias` method of spam checkers. It is provided a `RoomAlias`, not a `str`. ([\#9896](https://github.com/matrix-org/synapse/issues/9896)) +- Fix bug where user directory could get out of sync if room visibility and membership changed in quick succession. ([\#9910](https://github.com/matrix-org/synapse/issues/9910)) +- Include the `origin_server_ts` property in the experimental [MSC2946](https://github.com/matrix-org/matrix-doc/pull/2946) support to allow clients to properly sort rooms. ([\#9928](https://github.com/matrix-org/synapse/issues/9928)) +- Fix bugs introduced in v1.23.0 which made the PostgreSQL port script fail when run with a newly-created SQLite database. ([\#9930](https://github.com/matrix-org/synapse/issues/9930)) +- Fix a bug introduced in Synapse 1.29.0 which caused `m.room_key_request` to-device messages sent from one user to another to be dropped. ([\#9961](https://github.com/matrix-org/synapse/issues/9961), [\#9965](https://github.com/matrix-org/synapse/issues/9965)) +- Fix a bug introduced in v1.27.0 preventing users and appservices exempt from ratelimiting from creating rooms with many invitees. ([\#9968](https://github.com/matrix-org/synapse/issues/9968)) + + +Updates to the Docker image +--------------------------- + +- Add `startup_delay` to docker healthcheck to reduce waiting time for coming online and update the documentation with extra options. Contributed by @Maquis196. ([\#9913](https://github.com/matrix-org/synapse/issues/9913)) + + +Improved Documentation +---------------------- + +- Add `port` argument to the Postgres database sample config section. ([\#9911](https://github.com/matrix-org/synapse/issues/9911)) + + +Deprecations and Removals +------------------------- + +- Mark as deprecated `POST /_synapse/admin/v1/rooms//delete`. ([\#9889](https://github.com/matrix-org/synapse/issues/9889)) + + +Internal Changes +---------------- + +- Reduce the length of Synapse's access tokens. ([\#5588](https://github.com/matrix-org/synapse/issues/5588)) +- Export jemalloc stats to Prometheus if it is being used. ([\#9882](https://github.com/matrix-org/synapse/issues/9882)) +- Add type hints to presence handler. ([\#9885](https://github.com/matrix-org/synapse/issues/9885)) +- Reduce memory usage of the LRU caches. ([\#9886](https://github.com/matrix-org/synapse/issues/9886)) +- Add type hints to the `synapse.handlers` module. ([\#9896](https://github.com/matrix-org/synapse/issues/9896)) +- Time response time for external cache requests. ([\#9904](https://github.com/matrix-org/synapse/issues/9904)) +- Minor fixes to the `make_full_schema.sh` script. ([\#9931](https://github.com/matrix-org/synapse/issues/9931)) +- Move database schema files into a common directory. ([\#9932](https://github.com/matrix-org/synapse/issues/9932)) +- Add debug logging for lost/delayed to-device messages. ([\#9959](https://github.com/matrix-org/synapse/issues/9959)) + + +Synapse 1.33.2 (2021-05-11) +=========================== + +Due to the security issue highlighted below, server administrators are encouraged to update Synapse. We are not aware of these vulnerabilities being exploited in the wild. + +Security advisory +----------------- + +This release fixes a denial of service attack ([CVE-2021-29471](https://github.com/matrix-org/synapse/security/advisories/GHSA-x345-32rc-8h85)) against Synapse's push rules implementation. Server admins are encouraged to upgrade. + +Internal Changes +---------------- + +- Unpin attrs dependency. ([\#9946](https://github.com/matrix-org/synapse/issues/9946)) + + +Synapse 1.33.1 (2021-05-06) +=========================== + +Bugfixes +-------- + +- Fix bug where `/sync` would break if using the latest version of `attrs` dependency, by pinning to a previous version. ([\#9937](https://github.com/matrix-org/synapse/issues/9937)) + + +Synapse 1.33.0 (2021-05-05) +=========================== + +Features +-------- + +- Build Debian packages for Ubuntu 21.04 (Hirsute Hippo). ([\#9909](https://github.com/matrix-org/synapse/issues/9909)) + + +Synapse 1.33.0rc2 (2021-04-29) +============================== + +Bugfixes +-------- + +- Fix tight loop when handling presence replication when using workers. Introduced in v1.33.0rc1. ([\#9900](https://github.com/matrix-org/synapse/issues/9900)) + + +Synapse 1.33.0rc1 (2021-04-28) +============================== + +Features +-------- + +- Update experimental support for [MSC3083](https://github.com/matrix-org/matrix-doc/pull/3083): restricting room access via group membership. ([\#9800](https://github.com/matrix-org/synapse/issues/9800), [\#9814](https://github.com/matrix-org/synapse/issues/9814)) +- Add experimental support for handling presence on a worker. ([\#9819](https://github.com/matrix-org/synapse/issues/9819), [\#9820](https://github.com/matrix-org/synapse/issues/9820), [\#9828](https://github.com/matrix-org/synapse/issues/9828), [\#9850](https://github.com/matrix-org/synapse/issues/9850)) +- Return a new template when an user attempts to renew their account multiple times with the same token, stating that their account is set to expire. This replaces the invalid token template that would previously be shown in this case. This change concerns the optional account validity feature. ([\#9832](https://github.com/matrix-org/synapse/issues/9832)) + + +Bugfixes +-------- + +- Fixes the OIDC SSO flow when using a `public_baseurl` value including a non-root URL path. ([\#9726](https://github.com/matrix-org/synapse/issues/9726)) +- Fix thumbnail generation for some sites with non-standard content types. Contributed by @rkfg. ([\#9788](https://github.com/matrix-org/synapse/issues/9788)) +- Add some sanity checks to identity server passed to 3PID bind/unbind endpoints. ([\#9802](https://github.com/matrix-org/synapse/issues/9802)) +- Limit the size of HTTP responses read over federation. ([\#9833](https://github.com/matrix-org/synapse/issues/9833)) +- Fix a bug which could cause Synapse to get stuck in a loop of resyncing device lists. ([\#9867](https://github.com/matrix-org/synapse/issues/9867)) +- Fix a long-standing bug where errors from federation did not propagate to the client. ([\#9868](https://github.com/matrix-org/synapse/issues/9868)) + + +Improved Documentation +---------------------- + +- Add a note to the docker docs mentioning that we mirror upstream's supported Docker platforms. ([\#9801](https://github.com/matrix-org/synapse/issues/9801)) + + +Internal Changes +---------------- + +- Add a dockerfile for running Synapse in worker-mode under Complement. ([\#9162](https://github.com/matrix-org/synapse/issues/9162)) +- Apply `pyupgrade` across the codebase. ([\#9786](https://github.com/matrix-org/synapse/issues/9786)) +- Move some replication processing out of `generic_worker`. ([\#9796](https://github.com/matrix-org/synapse/issues/9796)) +- Replace `HomeServer.get_config()` with inline references. ([\#9815](https://github.com/matrix-org/synapse/issues/9815)) +- Rename some handlers and config modules to not duplicate the top-level module. ([\#9816](https://github.com/matrix-org/synapse/issues/9816)) +- Fix a long-standing bug which caused `max_upload_size` to not be correctly enforced. ([\#9817](https://github.com/matrix-org/synapse/issues/9817)) +- Reduce CPU usage of the user directory by reusing existing calculated room membership. ([\#9821](https://github.com/matrix-org/synapse/issues/9821)) +- Small speed up for joining large remote rooms. ([\#9825](https://github.com/matrix-org/synapse/issues/9825)) +- Introduce flake8-bugbear to the test suite and fix some of its lint violations. ([\#9838](https://github.com/matrix-org/synapse/issues/9838)) +- Only store the raw data in the in-memory caches, rather than objects that include references to e.g. the data stores. ([\#9845](https://github.com/matrix-org/synapse/issues/9845)) +- Limit length of accepted email addresses. ([\#9855](https://github.com/matrix-org/synapse/issues/9855)) +- Remove redundant `synapse.types.Collection` type definition. ([\#9856](https://github.com/matrix-org/synapse/issues/9856)) +- Handle recently added rate limits correctly when using `--no-rate-limit` with the demo scripts. ([\#9858](https://github.com/matrix-org/synapse/issues/9858)) +- Disable invite rate-limiting by default when running the unit tests. ([\#9871](https://github.com/matrix-org/synapse/issues/9871)) +- Pass a reactor into `SynapseSite` to make testing easier. ([\#9874](https://github.com/matrix-org/synapse/issues/9874)) +- Make `DomainSpecificString` an `attrs` class. ([\#9875](https://github.com/matrix-org/synapse/issues/9875)) +- Add type hints to `synapse.api.auth` and `synapse.api.auth_blocking` modules. ([\#9876](https://github.com/matrix-org/synapse/issues/9876)) +- Remove redundant `_PushHTTPChannel` test class. ([\#9878](https://github.com/matrix-org/synapse/issues/9878)) +- Remove backwards-compatibility code for Python versions < 3.6. ([\#9879](https://github.com/matrix-org/synapse/issues/9879)) +- Small performance improvement around handling new local presence updates. ([\#9887](https://github.com/matrix-org/synapse/issues/9887)) + + +Synapse 1.32.2 (2021-04-22) +=========================== + +This release includes a fix for a regression introduced in 1.32.0. + +Bugfixes +-------- + +- Fix a regression in Synapse 1.32.0 and 1.32.1 which caused `LoggingContext` errors in plugins. ([\#9857](https://github.com/matrix-org/synapse/issues/9857)) + + +Synapse 1.32.1 (2021-04-21) +=========================== + +This release fixes [a regression](https://github.com/matrix-org/synapse/issues/9853) +in Synapse 1.32.0 that caused connected Prometheus instances to become unstable. + +However, as this release is still subject to the `LoggingContext` change in 1.32.0, +it is recommended to remain on or downgrade to 1.31.0. + +Bugfixes +-------- + +- Fix a regression in Synapse 1.32.0 which caused Synapse to report large numbers of Prometheus time series, potentially overwhelming Prometheus instances. ([\#9854](https://github.com/matrix-org/synapse/issues/9854)) + + +Synapse 1.32.0 (2021-04-20) +=========================== + +**Note:** This release introduces [a regression](https://github.com/matrix-org/synapse/issues/9853) +that can overwhelm connected Prometheus instances. This issue was not present in +1.32.0rc1. If affected, it is recommended to downgrade to 1.31.0 in the meantime, and +follow [these instructions](https://github.com/matrix-org/synapse/pull/9854#issuecomment-823472183) +to clean up any excess writeahead logs. + +**Note:** This release also mistakenly included a change that may affected Synapse +modules that import `synapse.logging.context.LoggingContext`, such as +[synapse-s3-storage-provider](https://github.com/matrix-org/synapse-s3-storage-provider). +This will be fixed in a later Synapse version. + +**Note:** This release requires Python 3.6+ and Postgres 9.6+ or SQLite 3.22+. + +This release removes the deprecated `GET /_synapse/admin/v1/users/` admin API. Please use the [v2 API](https://github.com/matrix-org/synapse/blob/develop/docs/admin_api/user_admin_api.rst#query-user-account) instead, which has improved capabilities. + +This release requires Application Services to use type `m.login.application_service` when registering users via the `/_matrix/client/r0/register` endpoint to comply with the spec. Please ensure your Application Services are up to date. + +If you are using the `packages.matrix.org` Debian repository for Synapse packages, +note that we have recently updated the expiry date on the gpg signing key. If you see an +error similar to `The following signatures were invalid: EXPKEYSIG F473DD4473365DE1`, you +will need to get a fresh copy of the keys. You can do so with: + +```sh +sudo wget -O /usr/share/keyrings/matrix-org-archive-keyring.gpg https://packages.matrix.org/debian/matrix-org-archive-keyring.gpg +``` + +Bugfixes +-------- + +- Fix the log lines of nested logging contexts. Broke in 1.32.0rc1. ([\#9829](https://github.com/matrix-org/synapse/issues/9829)) + + +Synapse 1.32.0rc1 (2021-04-13) +============================== + +Features +-------- + +- Add a Synapse module for routing presence updates between users. ([\#9491](https://github.com/matrix-org/synapse/issues/9491)) +- Add an admin API to manage ratelimit for a specific user. ([\#9648](https://github.com/matrix-org/synapse/issues/9648)) +- Include request information in structured logging output. ([\#9654](https://github.com/matrix-org/synapse/issues/9654)) +- Add `order_by` to the admin API `GET /_synapse/admin/v2/users`. Contributed by @dklimpel. ([\#9691](https://github.com/matrix-org/synapse/issues/9691)) +- Replace the `room_invite_state_types` configuration setting with `room_prejoin_state`. ([\#9700](https://github.com/matrix-org/synapse/issues/9700)) +- Add experimental support for [MSC3083](https://github.com/matrix-org/matrix-doc/pull/3083): restricting room access via group membership. ([\#9717](https://github.com/matrix-org/synapse/issues/9717), [\#9735](https://github.com/matrix-org/synapse/issues/9735)) +- Update experimental support for Spaces: include `m.room.create` in the room state sent with room-invites. ([\#9710](https://github.com/matrix-org/synapse/issues/9710)) +- Synapse now requires Python 3.6 or later. It also requires Postgres 9.6 or later or SQLite 3.22 or later. ([\#9766](https://github.com/matrix-org/synapse/issues/9766)) + + +Bugfixes +-------- + +- Prevent `synapse_forward_extremities` and `synapse_excess_extremity_events` Prometheus metrics from initially reporting zero-values after startup. ([\#8926](https://github.com/matrix-org/synapse/issues/8926)) +- Fix recently added ratelimits to correctly honour the application service `rate_limited` flag. ([\#9711](https://github.com/matrix-org/synapse/issues/9711)) +- Fix longstanding bug which caused `duplicate key value violates unique constraint "remote_media_cache_thumbnails_media_origin_media_id_thumbna_key"` errors. ([\#9725](https://github.com/matrix-org/synapse/issues/9725)) +- Fix bug where sharded federation senders could get stuck repeatedly querying the DB in a loop, using lots of CPU. ([\#9770](https://github.com/matrix-org/synapse/issues/9770)) +- Fix duplicate logging of exceptions thrown during federation transaction processing. ([\#9780](https://github.com/matrix-org/synapse/issues/9780)) + + +Updates to the Docker image +--------------------------- + +- Move opencontainers labels to the final Docker image such that users can inspect them. ([\#9765](https://github.com/matrix-org/synapse/issues/9765)) + + +Improved Documentation +---------------------- + +- Make the `allowed_local_3pids` regex example in the sample config stricter. ([\#9719](https://github.com/matrix-org/synapse/issues/9719)) + + +Deprecations and Removals +------------------------- + +- Remove old admin API `GET /_synapse/admin/v1/users/`. ([\#9401](https://github.com/matrix-org/synapse/issues/9401)) +- Make `/_matrix/client/r0/register` expect a type of `m.login.application_service` when an Application Service registers a user, to align with [the relevant spec](https://spec.matrix.org/unstable/application-service-api/#server-admin-style-permissions). ([\#9548](https://github.com/matrix-org/synapse/issues/9548)) + + +Internal Changes +---------------- + +- Replace deprecated `imp` module with successor `importlib`. Contributed by Cristina Muñoz. ([\#9718](https://github.com/matrix-org/synapse/issues/9718)) +- Experiment with GitHub Actions for CI. ([\#9661](https://github.com/matrix-org/synapse/issues/9661)) +- Introduce flake8-bugbear to the test suite and fix some of its lint violations. ([\#9682](https://github.com/matrix-org/synapse/issues/9682)) +- Update `scripts-dev/complement.sh` to use a local checkout of Complement, allow running a subset of tests and have it use Synapse's Complement test blacklist. ([\#9685](https://github.com/matrix-org/synapse/issues/9685)) +- Improve Jaeger tracing for `to_device` messages. ([\#9686](https://github.com/matrix-org/synapse/issues/9686)) +- Add release helper script for automating part of the Synapse release process. ([\#9713](https://github.com/matrix-org/synapse/issues/9713)) +- Add type hints to expiring cache. ([\#9730](https://github.com/matrix-org/synapse/issues/9730)) +- Convert various testcases to `HomeserverTestCase`. ([\#9736](https://github.com/matrix-org/synapse/issues/9736)) +- Start linting mypy with `no_implicit_optional`. ([\#9742](https://github.com/matrix-org/synapse/issues/9742)) +- Add missing type hints to federation handler and server. ([\#9743](https://github.com/matrix-org/synapse/issues/9743)) +- Check that a `ConfigError` is raised, rather than simply `Exception`, when appropriate in homeserver config file generation tests. ([\#9753](https://github.com/matrix-org/synapse/issues/9753)) +- Fix incompatibility with `tox` 2.5. ([\#9769](https://github.com/matrix-org/synapse/issues/9769)) +- Enable Complement tests for [MSC2946](https://github.com/matrix-org/matrix-doc/pull/2946): Spaces Summary API. ([\#9771](https://github.com/matrix-org/synapse/issues/9771)) +- Use mock from the standard library instead of a separate package. ([\#9772](https://github.com/matrix-org/synapse/issues/9772)) +- Update Black configuration to target Python 3.6. ([\#9781](https://github.com/matrix-org/synapse/issues/9781)) +- Add option to skip unit tests when building Debian packages. ([\#9793](https://github.com/matrix-org/synapse/issues/9793)) + + +Synapse 1.31.0 (2021-04-06) +=========================== + +**Note:** As announced in v1.25.0, and in line with the deprecation policy for platform dependencies, this is the last release to support Python 3.5 and PostgreSQL 9.5. Future versions of Synapse will require Python 3.6+ and PostgreSQL 9.6+, as per our [deprecation policy](docs/deprecation_policy.md). + +This is also the last release that the Synapse team will be publishing packages for Debian Stretch and Ubuntu Xenial. + + +Improved Documentation +---------------------- + +- Add a document describing the deprecation policy for platform dependencies. ([\#9723](https://github.com/matrix-org/synapse/issues/9723)) + + +Internal Changes +---------------- + +- Revert using `dmypy run` in lint script. ([\#9720](https://github.com/matrix-org/synapse/issues/9720)) +- Pin flake8-bugbear's version. ([\#9734](https://github.com/matrix-org/synapse/issues/9734)) + + +Synapse 1.31.0rc1 (2021-03-30) +============================== + +Features +-------- + +- Add support to OpenID Connect login for requiring attributes on the `userinfo` response. Contributed by Hubbe King. ([\#9609](https://github.com/matrix-org/synapse/issues/9609)) +- Add initial experimental support for a "space summary" API. ([\#9643](https://github.com/matrix-org/synapse/issues/9643), [\#9652](https://github.com/matrix-org/synapse/issues/9652), [\#9653](https://github.com/matrix-org/synapse/issues/9653)) +- Add support for the busy presence state as described in [MSC3026](https://github.com/matrix-org/matrix-doc/pull/3026). ([\#9644](https://github.com/matrix-org/synapse/issues/9644)) +- Add support for credentials for proxy authentication in the `HTTPS_PROXY` environment variable. ([\#9657](https://github.com/matrix-org/synapse/issues/9657)) + + +Bugfixes +-------- + +- Fix a longstanding bug that could cause issues when editing a reply to a message. ([\#9585](https://github.com/matrix-org/synapse/issues/9585)) +- Fix the `/capabilities` endpoint to return `m.change_password` as disabled if the local password database is not used for authentication. Contributed by @dklimpel. ([\#9588](https://github.com/matrix-org/synapse/issues/9588)) +- Check if local passwords are enabled before setting them for the user. ([\#9636](https://github.com/matrix-org/synapse/issues/9636)) +- Fix a bug where federation sending can stall due to `concurrent access` database exceptions when it falls behind. ([\#9639](https://github.com/matrix-org/synapse/issues/9639)) +- Fix a bug introduced in Synapse 1.30.1 which meant the suggested `pip` incantation to install an updated `cryptography` was incorrect. ([\#9699](https://github.com/matrix-org/synapse/issues/9699)) + + +Updates to the Docker image +--------------------------- + +- Speed up Docker builds and make it nicer to test against Complement while developing (install all dependencies before copying the project). ([\#9610](https://github.com/matrix-org/synapse/issues/9610)) +- Include [opencontainers labels](https://github.com/opencontainers/image-spec/blob/master/annotations.md#pre-defined-annotation-keys) in the Docker image. ([\#9612](https://github.com/matrix-org/synapse/issues/9612)) + + +Improved Documentation +---------------------- + +- Clarify that `register_new_matrix_user` is present also when installed via non-pip package. ([\#9074](https://github.com/matrix-org/synapse/issues/9074)) +- Update source install documentation to mention platform prerequisites before the source install steps. ([\#9667](https://github.com/matrix-org/synapse/issues/9667)) +- Improve worker documentation for fallback/web auth endpoints. ([\#9679](https://github.com/matrix-org/synapse/issues/9679)) +- Update the sample configuration for OIDC authentication. ([\#9695](https://github.com/matrix-org/synapse/issues/9695)) + + +Internal Changes +---------------- + +- Preparatory steps for removing redundant `outlier` data from `event_json.internal_metadata` column. ([\#9411](https://github.com/matrix-org/synapse/issues/9411)) +- Add type hints to the caching module. ([\#9442](https://github.com/matrix-org/synapse/issues/9442)) +- Introduce flake8-bugbear to the test suite and fix some of its lint violations. ([\#9499](https://github.com/matrix-org/synapse/issues/9499), [\#9659](https://github.com/matrix-org/synapse/issues/9659)) +- Add additional type hints to the Homeserver object. ([\#9631](https://github.com/matrix-org/synapse/issues/9631), [\#9638](https://github.com/matrix-org/synapse/issues/9638), [\#9675](https://github.com/matrix-org/synapse/issues/9675), [\#9681](https://github.com/matrix-org/synapse/issues/9681)) +- Only save remote cross-signing and device keys if they're different from the current ones. ([\#9634](https://github.com/matrix-org/synapse/issues/9634)) +- Rename storage function to fix spelling and not conflict with another function's name. ([\#9637](https://github.com/matrix-org/synapse/issues/9637)) +- Improve performance of federation catch up by sending the latest events in the room to the remote, rather than just the last event sent by the local server. ([\#9640](https://github.com/matrix-org/synapse/issues/9640), [\#9664](https://github.com/matrix-org/synapse/issues/9664)) +- In the `federation_client` commandline client, stop automatically adding the URL prefix, so that servlets on other prefixes can be tested. ([\#9645](https://github.com/matrix-org/synapse/issues/9645)) +- In the `federation_client` commandline client, handle inline `signing_key`s in `homeserver.yaml`. ([\#9647](https://github.com/matrix-org/synapse/issues/9647)) +- Fixed some antipattern issues to improve code quality. ([\#9649](https://github.com/matrix-org/synapse/issues/9649)) +- Add a storage method for pulling all current user presence state from the database. ([\#9650](https://github.com/matrix-org/synapse/issues/9650)) +- Import `HomeServer` from the proper module. ([\#9665](https://github.com/matrix-org/synapse/issues/9665)) +- Increase default join ratelimiting burst rate. ([\#9674](https://github.com/matrix-org/synapse/issues/9674)) +- Add type hints to third party event rules and visibility modules. ([\#9676](https://github.com/matrix-org/synapse/issues/9676)) +- Bump mypy-zope to 0.2.13 to fix "Cannot determine consistent method resolution order (MRO)" errors when running mypy a second time. ([\#9678](https://github.com/matrix-org/synapse/issues/9678)) +- Use interpreter from `$PATH` via `/usr/bin/env` instead of absolute paths in various scripts. ([\#9689](https://github.com/matrix-org/synapse/issues/9689)) +- Make it possible to use `dmypy`. ([\#9692](https://github.com/matrix-org/synapse/issues/9692)) +- Suppress "CryptographyDeprecationWarning: int_from_bytes is deprecated". ([\#9698](https://github.com/matrix-org/synapse/issues/9698)) +- Use `dmypy run` in lint script for improved performance in type-checking while developing. ([\#9701](https://github.com/matrix-org/synapse/issues/9701)) +- Fix undetected mypy error when using Python 3.6. ([\#9703](https://github.com/matrix-org/synapse/issues/9703)) +- Fix type-checking CI on develop. ([\#9709](https://github.com/matrix-org/synapse/issues/9709)) + + +Synapse 1.30.1 (2021-03-26) +=========================== + +This release is identical to Synapse 1.30.0, with the exception of explicitly +setting a minimum version of Python's Cryptography library to ensure that users +of Synapse are protected from the recent [OpenSSL security advisories](https://mta.openssl.org/pipermail/openssl-announce/2021-March/000198.html), +especially CVE-2021-3449. + +Note that Cryptography defaults to bundling its own statically linked copy of +OpenSSL, which means that you may not be protected by your operating system's +security updates. + +It's also worth noting that Cryptography no longer supports Python 3.5, so +admins deploying to older environments may not be protected against this or +future vulnerabilities. Synapse will be dropping support for Python 3.5 at the +end of March. + + +Updates to the Docker image +--------------------------- + +- Ensure that the docker container has up to date versions of openssl. ([\#9697](https://github.com/matrix-org/synapse/issues/9697)) + + +Internal Changes +---------------- + +- Enforce that `cryptography` dependency is up to date to ensure it has the most recent openssl patches. ([\#9697](https://github.com/matrix-org/synapse/issues/9697)) + + +Synapse 1.30.0 (2021-03-22) +=========================== + +Note that this release deprecates the ability for appservices to +call `POST /_matrix/client/r0/register` without the body parameter `type`. Appservice +developers should use a `type` value of `m.login.application_service` as +per [the spec](https://matrix.org/docs/spec/application_service/r0.1.2#server-admin-style-permissions). +In future releases, calling this endpoint with an access token - but without a `m.login.application_service` +type - will fail. + + +No significant changes. + + +Synapse 1.30.0rc1 (2021-03-16) +============================== + +Features +-------- + +- Add prometheus metrics for number of users successfully registering and logging in. ([\#9510](https://github.com/matrix-org/synapse/issues/9510), [\#9511](https://github.com/matrix-org/synapse/issues/9511), [\#9573](https://github.com/matrix-org/synapse/issues/9573)) +- Add `synapse_federation_last_sent_pdu_time` and `synapse_federation_last_received_pdu_time` prometheus metrics, which monitor federation delays by reporting the timestamps of messages sent and received to a set of remote servers. ([\#9540](https://github.com/matrix-org/synapse/issues/9540)) +- Add support for generating JSON Web Tokens dynamically for use as OIDC client secrets. ([\#9549](https://github.com/matrix-org/synapse/issues/9549)) +- Optimise handling of incomplete room history for incoming federation. ([\#9601](https://github.com/matrix-org/synapse/issues/9601)) +- Finalise support for allowing clients to pick an SSO Identity Provider ([MSC2858](https://github.com/matrix-org/matrix-doc/pull/2858)). ([\#9617](https://github.com/matrix-org/synapse/issues/9617)) +- Tell spam checker modules about the SSO IdP a user registered through if one was used. ([\#9626](https://github.com/matrix-org/synapse/issues/9626)) + + +Bugfixes +-------- + +- Fix long-standing bug when generating thumbnails for some images with transparency: `TypeError: cannot unpack non-iterable int object`. ([\#9473](https://github.com/matrix-org/synapse/issues/9473)) +- Purge chain cover indexes for events that were purged prior to Synapse v1.29.0. ([\#9542](https://github.com/matrix-org/synapse/issues/9542), [\#9583](https://github.com/matrix-org/synapse/issues/9583)) +- Fix bug where federation requests were not correctly retried on 5xx responses. ([\#9567](https://github.com/matrix-org/synapse/issues/9567)) +- Fix re-activating an account via the admin API when local passwords are disabled. ([\#9587](https://github.com/matrix-org/synapse/issues/9587)) +- Fix a bug introduced in Synapse 1.20 which caused incoming federation transactions to stack up, causing slow recovery from outages. ([\#9597](https://github.com/matrix-org/synapse/issues/9597)) +- Fix a bug introduced in v1.28.0 where the OpenID Connect callback endpoint could error with a `MacaroonInitException`. ([\#9620](https://github.com/matrix-org/synapse/issues/9620)) +- Fix Internal Server Error on `GET /_synapse/client/saml2/authn_response` request. ([\#9623](https://github.com/matrix-org/synapse/issues/9623)) + + +Updates to the Docker image +--------------------------- + +- Make use of an improved malloc implementation (`jemalloc`) in the docker image. ([\#8553](https://github.com/matrix-org/synapse/issues/8553)) + + +Improved Documentation +---------------------- + +- Add relayd entry to reverse proxy example configurations. ([\#9508](https://github.com/matrix-org/synapse/issues/9508)) +- Improve the SAML2 upgrade notes for 1.27.0. ([\#9550](https://github.com/matrix-org/synapse/issues/9550)) +- Link to the "List user's media" admin API from the media admin API docs. ([\#9571](https://github.com/matrix-org/synapse/issues/9571)) +- Clarify the spam checker modules documentation example to mention that `parse_config` is a required method. ([\#9580](https://github.com/matrix-org/synapse/issues/9580)) +- Clarify the sample configuration for `stats` settings. ([\#9604](https://github.com/matrix-org/synapse/issues/9604)) + + +Deprecations and Removals +------------------------- + +- The `synapse_federation_last_sent_pdu_age` and `synapse_federation_last_received_pdu_age` prometheus metrics have been removed. They are replaced by `synapse_federation_last_sent_pdu_time` and `synapse_federation_last_received_pdu_time`. ([\#9540](https://github.com/matrix-org/synapse/issues/9540)) +- Registering an Application Service user without using the `m.login.application_service` login type will be unsupported in an upcoming Synapse release. ([\#9559](https://github.com/matrix-org/synapse/issues/9559)) + + +Internal Changes +---------------- + +- Add tests to ResponseCache. ([\#9458](https://github.com/matrix-org/synapse/issues/9458)) +- Add type hints to purge room and server notice admin API. ([\#9520](https://github.com/matrix-org/synapse/issues/9520)) +- Add extra logging to ObservableDeferred when callbacks throw exceptions. ([\#9523](https://github.com/matrix-org/synapse/issues/9523)) +- Fix incorrect type hints. ([\#9528](https://github.com/matrix-org/synapse/issues/9528), [\#9543](https://github.com/matrix-org/synapse/issues/9543), [\#9591](https://github.com/matrix-org/synapse/issues/9591), [\#9608](https://github.com/matrix-org/synapse/issues/9608), [\#9618](https://github.com/matrix-org/synapse/issues/9618)) +- Add an additional test for purging a room. ([\#9541](https://github.com/matrix-org/synapse/issues/9541)) +- Add a `.git-blame-ignore-revs` file with the hashes of auto-formatting. ([\#9560](https://github.com/matrix-org/synapse/issues/9560)) +- Increase the threshold before which outbound federation to a server goes into "catch up" mode, which is expensive for the remote server to handle. ([\#9561](https://github.com/matrix-org/synapse/issues/9561)) +- Fix spurious errors reported by the `config-lint.sh` script. ([\#9562](https://github.com/matrix-org/synapse/issues/9562)) +- Fix type hints and tests for BlacklistingAgentWrapper and BlacklistingReactorWrapper. ([\#9563](https://github.com/matrix-org/synapse/issues/9563)) +- Do not have mypy ignore type hints from unpaddedbase64. ([\#9568](https://github.com/matrix-org/synapse/issues/9568)) +- Improve efficiency of calculating the auth chain in large rooms. ([\#9576](https://github.com/matrix-org/synapse/issues/9576)) +- Convert `synapse.types.Requester` to an `attrs` class. ([\#9586](https://github.com/matrix-org/synapse/issues/9586)) +- Add logging for redis connection setup. ([\#9590](https://github.com/matrix-org/synapse/issues/9590)) +- Improve logging when processing incoming transactions. ([\#9596](https://github.com/matrix-org/synapse/issues/9596)) +- Remove unused `stats.retention` setting, and emit a warning if stats are disabled. ([\#9604](https://github.com/matrix-org/synapse/issues/9604)) +- Prevent attempting to bundle aggregations for state events in /context APIs. ([\#9619](https://github.com/matrix-org/synapse/issues/9619)) + + +Synapse 1.29.0 (2021-03-08) +=========================== + +Note that synapse now expects an `X-Forwarded-Proto` header when used with a reverse proxy. Please see the [upgrade notes](docs/upgrade.md#upgrading-to-v1290) for more details on this change. + + +No significant changes. + + +Synapse 1.29.0rc1 (2021-03-04) +============================== + +Features +-------- + +- Add rate limiters to cross-user key sharing requests. ([\#8957](https://github.com/matrix-org/synapse/issues/8957)) +- Add `order_by` to the admin API `GET /_synapse/admin/v1/users//media`. Contributed by @dklimpel. ([\#8978](https://github.com/matrix-org/synapse/issues/8978)) +- Add some configuration settings to make users' profile data more private. ([\#9203](https://github.com/matrix-org/synapse/issues/9203)) +- The `no_proxy` and `NO_PROXY` environment variables are now respected in proxied HTTP clients with the lowercase form taking precedence if both are present. Additionally, the lowercase `https_proxy` environment variable is now respected in proxied HTTP clients on top of existing support for the uppercase `HTTPS_PROXY` form and takes precedence if both are present. Contributed by Timothy Leung. ([\#9372](https://github.com/matrix-org/synapse/issues/9372)) +- Add a configuration option, `user_directory.prefer_local_users`, which when enabled will make it more likely for users on the same server as you to appear above other users. ([\#9383](https://github.com/matrix-org/synapse/issues/9383), [\#9385](https://github.com/matrix-org/synapse/issues/9385)) +- Add support for regenerating thumbnails if they have been deleted but the original image is still stored. ([\#9438](https://github.com/matrix-org/synapse/issues/9438)) +- Add support for `X-Forwarded-Proto` header when using a reverse proxy. ([\#9472](https://github.com/matrix-org/synapse/issues/9472), [\#9501](https://github.com/matrix-org/synapse/issues/9501), [\#9512](https://github.com/matrix-org/synapse/issues/9512), [\#9539](https://github.com/matrix-org/synapse/issues/9539)) + + +Bugfixes +-------- + +- Fix a bug where users' pushers were not all deleted when they deactivated their account. ([\#9285](https://github.com/matrix-org/synapse/issues/9285), [\#9516](https://github.com/matrix-org/synapse/issues/9516)) +- Fix a bug where a lot of unnecessary presence updates were sent when joining a room. ([\#9402](https://github.com/matrix-org/synapse/issues/9402)) +- Fix a bug that caused multiple calls to the experimental `shared_rooms` endpoint to return stale results. ([\#9416](https://github.com/matrix-org/synapse/issues/9416)) +- Fix a bug in single sign-on which could cause a "No session cookie found" error. ([\#9436](https://github.com/matrix-org/synapse/issues/9436)) +- Fix bug introduced in v1.27.0 where allowing a user to choose their own username when logging in via single sign-on did not work unless an `idp_icon` was defined. ([\#9440](https://github.com/matrix-org/synapse/issues/9440)) +- Fix a bug introduced in v1.26.0 where some sequences were not properly configured when running `synapse_port_db`. ([\#9449](https://github.com/matrix-org/synapse/issues/9449)) +- Fix deleting pushers when using sharded pushers. ([\#9465](https://github.com/matrix-org/synapse/issues/9465), [\#9466](https://github.com/matrix-org/synapse/issues/9466), [\#9479](https://github.com/matrix-org/synapse/issues/9479), [\#9536](https://github.com/matrix-org/synapse/issues/9536)) +- Fix missing startup checks for the consistency of certain PostgreSQL sequences. ([\#9470](https://github.com/matrix-org/synapse/issues/9470)) +- Fix a long-standing bug where the media repository could leak file descriptors while previewing media. ([\#9497](https://github.com/matrix-org/synapse/issues/9497)) +- Properly purge the event chain cover index when purging history. ([\#9498](https://github.com/matrix-org/synapse/issues/9498)) +- Fix missing chain cover index due to a schema delta not being applied correctly. Only affected servers that ran development versions. ([\#9503](https://github.com/matrix-org/synapse/issues/9503)) +- Fix a bug introduced in v1.25.0 where `/_synapse/admin/join/` would fail when given a room alias. ([\#9506](https://github.com/matrix-org/synapse/issues/9506)) +- Prevent presence background jobs from running when presence is disabled. ([\#9530](https://github.com/matrix-org/synapse/issues/9530)) +- Fix rare edge case that caused a background update to fail if the server had rejected an event that had duplicate auth events. ([\#9537](https://github.com/matrix-org/synapse/issues/9537)) + + +Improved Documentation +---------------------- + +- Update the example systemd config to propagate reloads to individual units. ([\#9463](https://github.com/matrix-org/synapse/issues/9463)) + + +Internal Changes +---------------- + +- Add documentation and type hints to `parse_duration`. ([\#9432](https://github.com/matrix-org/synapse/issues/9432)) +- Remove vestiges of `uploads_path` configuration setting. ([\#9462](https://github.com/matrix-org/synapse/issues/9462)) +- Add a comment about systemd-python. ([\#9464](https://github.com/matrix-org/synapse/issues/9464)) +- Test that we require validated email for email pushers. ([\#9496](https://github.com/matrix-org/synapse/issues/9496)) +- Allow python to generate bytecode for synapse. ([\#9502](https://github.com/matrix-org/synapse/issues/9502)) +- Fix incorrect type hints. ([\#9515](https://github.com/matrix-org/synapse/issues/9515), [\#9518](https://github.com/matrix-org/synapse/issues/9518)) +- Add type hints to device and event report admin API. ([\#9519](https://github.com/matrix-org/synapse/issues/9519)) +- Add type hints to user admin API. ([\#9521](https://github.com/matrix-org/synapse/issues/9521)) +- Bump the versions of mypy and mypy-zope used for static type checking. ([\#9529](https://github.com/matrix-org/synapse/issues/9529)) + + +Synapse 1.28.0 (2021-02-25) +=========================== + +Note that this release drops support for ARMv7 in the official Docker images, due to repeated problems building for ARMv7 (and the associated maintenance burden this entails). + +This release also fixes the documentation included in v1.27.0 around the callback URI for SAML2 identity providers. If your server is configured to use single sign-on via a SAML2 IdP, you may need to make configuration changes. Please review the [upgrade notes](docs/upgrade.md) for more details on these changes. + + +Internal Changes +---------------- + +- Revert change in v1.28.0rc1 to remove the deprecated SAML endpoint. ([\#9474](https://github.com/matrix-org/synapse/issues/9474)) + + +Synapse 1.28.0rc1 (2021-02-19) +============================== + +Removal warning +--------------- + +The v1 list accounts API is deprecated and will be removed in a future release. +This API was undocumented and misleading. It can be replaced by the +[v2 list accounts API](https://github.com/matrix-org/synapse/blob/release-v1.28.0/docs/admin_api/user_admin_api.rst#list-accounts), +which has been available since Synapse 1.7.0 (2019-12-13). + +Please check if you're using any scripts which use the admin API and replace +`GET /_synapse/admin/v1/users/` with `GET /_synapse/admin/v2/users`. + + +Features +-------- + +- New admin API to get the context of an event: `/_synapse/admin/rooms/{roomId}/context/{eventId}`. ([\#9150](https://github.com/matrix-org/synapse/issues/9150)) +- Further improvements to the user experience of registration via single sign-on. ([\#9300](https://github.com/matrix-org/synapse/issues/9300), [\#9301](https://github.com/matrix-org/synapse/issues/9301)) +- Add hook to spam checker modules that allow checking file uploads and remote downloads. ([\#9311](https://github.com/matrix-org/synapse/issues/9311)) +- Add support for receiving OpenID Connect authentication responses via form `POST`s rather than `GET`s. ([\#9376](https://github.com/matrix-org/synapse/issues/9376)) +- Add the shadow-banning status to the admin API for user info. ([\#9400](https://github.com/matrix-org/synapse/issues/9400)) + + +Bugfixes +-------- + +- Fix long-standing bug where sending email notifications would fail for rooms that the server had since left. ([\#9257](https://github.com/matrix-org/synapse/issues/9257)) +- Fix bug introduced in Synapse 1.27.0rc1 which meant the "session expired" error page during SSO registration was badly formatted. ([\#9296](https://github.com/matrix-org/synapse/issues/9296)) +- Assert a maximum length for some parameters for spec compliance. ([\#9321](https://github.com/matrix-org/synapse/issues/9321), [\#9393](https://github.com/matrix-org/synapse/issues/9393)) +- Fix additional errors when previewing URLs: "AttributeError 'NoneType' object has no attribute 'xpath'" and "ValueError: Unicode strings with encoding declaration are not supported. Please use bytes input or XML fragments without declaration.". ([\#9333](https://github.com/matrix-org/synapse/issues/9333)) +- Fix a bug causing Synapse to impose the wrong type constraints on fields when processing responses from appservices to `/_matrix/app/v1/thirdparty/user/{protocol}`. ([\#9361](https://github.com/matrix-org/synapse/issues/9361)) +- Fix bug where Synapse would occasionally stop reconnecting to Redis after the connection was lost. ([\#9391](https://github.com/matrix-org/synapse/issues/9391)) +- Fix a long-standing bug when upgrading a room: "TypeError: '>' not supported between instances of 'NoneType' and 'int'". ([\#9395](https://github.com/matrix-org/synapse/issues/9395)) +- Reduce the amount of memory used when generating the URL preview of a file that is larger than the `max_spider_size`. ([\#9421](https://github.com/matrix-org/synapse/issues/9421)) +- Fix a long-standing bug in the deduplication of old presence, resulting in no deduplication. ([\#9425](https://github.com/matrix-org/synapse/issues/9425)) +- The `ui_auth.session_timeout` config option can now be specified in terms of number of seconds/minutes/etc/. Contributed by Rishabh Arya. ([\#9426](https://github.com/matrix-org/synapse/issues/9426)) +- Fix a bug introduced in v1.27.0: "TypeError: int() argument must be a string, a bytes-like object or a number, not 'NoneType." related to the user directory. ([\#9428](https://github.com/matrix-org/synapse/issues/9428)) + + +Updates to the Docker image +--------------------------- + +- Drop support for ARMv7 in Docker images. ([\#9433](https://github.com/matrix-org/synapse/issues/9433)) + + +Improved Documentation +---------------------- + +- Reorganize CHANGELOG.md. ([\#9281](https://github.com/matrix-org/synapse/issues/9281)) +- Add note to `auto_join_rooms` config option explaining existing rooms must be publicly joinable. ([\#9291](https://github.com/matrix-org/synapse/issues/9291)) +- Correct name of Synapse's service file in TURN howto. ([\#9308](https://github.com/matrix-org/synapse/issues/9308)) +- Fix the braces in the `oidc_providers` section of the sample config. ([\#9317](https://github.com/matrix-org/synapse/issues/9317)) +- Update installation instructions on Fedora. ([\#9322](https://github.com/matrix-org/synapse/issues/9322)) +- Add HTTP/2 support to the nginx example configuration. Contributed by David Vo. ([\#9390](https://github.com/matrix-org/synapse/issues/9390)) +- Update docs for using Gitea as OpenID provider. ([\#9404](https://github.com/matrix-org/synapse/issues/9404)) +- Document that pusher instances are shardable. ([\#9407](https://github.com/matrix-org/synapse/issues/9407)) +- Fix erroneous documentation from v1.27.0 about updating the SAML2 callback URL. ([\#9434](https://github.com/matrix-org/synapse/issues/9434)) + + +Deprecations and Removals +------------------------- + +- Deprecate old admin API `GET /_synapse/admin/v1/users/`. ([\#9429](https://github.com/matrix-org/synapse/issues/9429)) + + +Internal Changes +---------------- + +- Fix 'object name reserved for internal use' errors with recent versions of SQLite. ([\#9003](https://github.com/matrix-org/synapse/issues/9003)) +- Add experimental support for running Synapse with PyPy. ([\#9123](https://github.com/matrix-org/synapse/issues/9123)) +- Deny access to additional IP addresses by default. ([\#9240](https://github.com/matrix-org/synapse/issues/9240)) +- Update the `Cursor` type hints to better match PEP 249. ([\#9299](https://github.com/matrix-org/synapse/issues/9299)) +- Add debug logging for SRV lookups. Contributed by @Bubu. ([\#9305](https://github.com/matrix-org/synapse/issues/9305)) +- Improve logging for OIDC login flow. ([\#9307](https://github.com/matrix-org/synapse/issues/9307)) +- Share the code for handling required attributes between the CAS and SAML handlers. ([\#9326](https://github.com/matrix-org/synapse/issues/9326)) +- Clean up the code to load the metadata for OpenID Connect identity providers. ([\#9362](https://github.com/matrix-org/synapse/issues/9362)) +- Convert tests to use `HomeserverTestCase`. ([\#9377](https://github.com/matrix-org/synapse/issues/9377), [\#9396](https://github.com/matrix-org/synapse/issues/9396)) +- Update the version of black used to 20.8b1. ([\#9381](https://github.com/matrix-org/synapse/issues/9381)) +- Allow OIDC config to override discovered values. ([\#9384](https://github.com/matrix-org/synapse/issues/9384)) +- Remove some dead code from the acceptance of room invites path. ([\#9394](https://github.com/matrix-org/synapse/issues/9394)) +- Clean up an unused method in the presence handler code. ([\#9408](https://github.com/matrix-org/synapse/issues/9408)) + + +Synapse 1.27.0 (2021-02-16) +=========================== + +Note that this release includes a change in Synapse to use Redis as a cache ─ as well as a pub/sub mechanism ─ if Redis support is enabled for workers. No action is needed by server administrators, and we do not expect resource usage of the Redis instance to change dramatically. + +This release also changes the callback URI for OpenID Connect (OIDC) and SAML2 identity providers. If your server is configured to use single sign-on via an OIDC/OAuth2 or SAML2 IdP, you may need to make configuration changes. Please review the [upgrade notes](docs/upgrade.md) for more details on these changes. + +This release also changes escaping of variables in the HTML templates for SSO or email notifications. If you have customised these templates, please review the [upgrade notes](docs/upgrade.md) for more details on these changes. + + +Bugfixes +-------- + +- Fix building Docker images for armv7. ([\#9405](https://github.com/matrix-org/synapse/issues/9405)) + + +Synapse 1.27.0rc2 (2021-02-11) +============================== + +Features +-------- + +- Further improvements to the user experience of registration via single sign-on. ([\#9297](https://github.com/matrix-org/synapse/issues/9297)) + + +Bugfixes +-------- + +- Fix ratelimiting introduced in v1.27.0rc1 for invites to respect the `ratelimit` flag on application services. ([\#9302](https://github.com/matrix-org/synapse/issues/9302)) +- Do not automatically calculate `public_baseurl` since it can be wrong in some situations. Reverts behaviour introduced in v1.26.0. ([\#9313](https://github.com/matrix-org/synapse/issues/9313)) + + +Improved Documentation +---------------------- + +- Clarify the sample configuration for changes made to the template loading code. ([\#9310](https://github.com/matrix-org/synapse/issues/9310)) + + +Synapse 1.27.0rc1 (2021-02-02) +============================== + +Features +-------- + +- Add an admin API for getting and deleting forward extremities for a room. ([\#9062](https://github.com/matrix-org/synapse/issues/9062)) +- Add an admin API for retrieving the current room state of a room. ([\#9168](https://github.com/matrix-org/synapse/issues/9168)) +- Add experimental support for allowing clients to pick an SSO Identity Provider ([MSC2858](https://github.com/matrix-org/matrix-doc/pull/2858)). ([\#9183](https://github.com/matrix-org/synapse/issues/9183), [\#9242](https://github.com/matrix-org/synapse/issues/9242)) +- Add an admin API endpoint for shadow-banning users. ([\#9209](https://github.com/matrix-org/synapse/issues/9209)) +- Add ratelimits to the 3PID `/requestToken` APIs. ([\#9238](https://github.com/matrix-org/synapse/issues/9238)) +- Add support to the OpenID Connect integration for adding the user's email address. ([\#9245](https://github.com/matrix-org/synapse/issues/9245)) +- Add ratelimits to invites in rooms and to specific users. ([\#9258](https://github.com/matrix-org/synapse/issues/9258)) +- Improve the user experience of setting up an account via single-sign on. ([\#9262](https://github.com/matrix-org/synapse/issues/9262), [\#9272](https://github.com/matrix-org/synapse/issues/9272), [\#9275](https://github.com/matrix-org/synapse/issues/9275), [\#9276](https://github.com/matrix-org/synapse/issues/9276), [\#9277](https://github.com/matrix-org/synapse/issues/9277), [\#9286](https://github.com/matrix-org/synapse/issues/9286), [\#9287](https://github.com/matrix-org/synapse/issues/9287)) +- Add phone home stats for encrypted messages. ([\#9283](https://github.com/matrix-org/synapse/issues/9283)) +- Update the redirect URI for OIDC authentication. ([\#9288](https://github.com/matrix-org/synapse/issues/9288)) + + +Bugfixes +-------- + +- Fix spurious errors in logs when deleting a non-existant pusher. ([\#9121](https://github.com/matrix-org/synapse/issues/9121)) +- Fix a long-standing bug where Synapse would return a 500 error when a thumbnail did not exist (and auto-generation of thumbnails was not enabled). ([\#9163](https://github.com/matrix-org/synapse/issues/9163)) +- Fix a long-standing bug where an internal server error was raised when attempting to preview an HTML document in an unknown character encoding. ([\#9164](https://github.com/matrix-org/synapse/issues/9164)) +- Fix a long-standing bug where invalid data could cause errors when calculating the presentable room name for push. ([\#9165](https://github.com/matrix-org/synapse/issues/9165)) +- Fix bug where we sometimes didn't detect that Redis connections had died, causing workers to not see new data. ([\#9218](https://github.com/matrix-org/synapse/issues/9218)) +- Fix a bug where `None` was passed to Synapse modules instead of an empty dictionary if an empty module `config` block was provided in the homeserver config. ([\#9229](https://github.com/matrix-org/synapse/issues/9229)) +- Fix a bug in the `make_room_admin` admin API where it failed if the admin with the greatest power level was not in the room. Contributed by Pankaj Yadav. ([\#9235](https://github.com/matrix-org/synapse/issues/9235)) +- Prevent password hashes from getting dropped if a client failed threepid validation during a User Interactive Auth stage. Removes a workaround for an ancient bug in Riot Web /joined_rooms` to work for both local and remote users. ([\#8948](https://github.com/matrix-org/synapse/issues/8948)) +- Add experimental support for handling to-device messages on worker processes. ([\#9042](https://github.com/matrix-org/synapse/issues/9042), [\#9043](https://github.com/matrix-org/synapse/issues/9043), [\#9044](https://github.com/matrix-org/synapse/issues/9044), [\#9130](https://github.com/matrix-org/synapse/issues/9130)) +- Add experimental support for handling `/keys/claim` and `/room_keys` APIs on worker processes. ([\#9068](https://github.com/matrix-org/synapse/issues/9068)) +- Add experimental support for handling `/devices` API on worker processes. ([\#9092](https://github.com/matrix-org/synapse/issues/9092)) +- Add experimental support for moving off receipts and account data persistence off master. ([\#9104](https://github.com/matrix-org/synapse/issues/9104), [\#9166](https://github.com/matrix-org/synapse/issues/9166)) + + +Bugfixes +-------- + +- Fix a long-standing issue where an internal server error would occur when requesting a profile over federation that did not include a display name / avatar URL. ([\#9023](https://github.com/matrix-org/synapse/issues/9023)) +- Fix a long-standing bug where some caches could grow larger than configured. ([\#9028](https://github.com/matrix-org/synapse/issues/9028)) +- Fix error handling during insertion of client IPs into the database. ([\#9051](https://github.com/matrix-org/synapse/issues/9051)) +- Fix bug where we didn't correctly record CPU time spent in `on_new_event` block. ([\#9053](https://github.com/matrix-org/synapse/issues/9053)) +- Fix a minor bug which could cause confusing error messages from invalid configurations. ([\#9054](https://github.com/matrix-org/synapse/issues/9054)) +- Fix incorrect exit code when there is an error at startup. ([\#9059](https://github.com/matrix-org/synapse/issues/9059)) +- Fix `JSONDecodeError` spamming the logs when sending transactions to remote servers. ([\#9070](https://github.com/matrix-org/synapse/issues/9070)) +- Fix "Failed to send request" errors when a client provides an invalid room alias. ([\#9071](https://github.com/matrix-org/synapse/issues/9071)) +- Fix bugs in federation catchup logic that caused outbound federation to be delayed for large servers after start up. Introduced in v1.8.0 and v1.21.0. ([\#9114](https://github.com/matrix-org/synapse/issues/9114), [\#9116](https://github.com/matrix-org/synapse/issues/9116)) +- Fix corruption of `pushers` data when a postgres bouncer is used. ([\#9117](https://github.com/matrix-org/synapse/issues/9117)) +- Fix minor bugs in handling the `clientRedirectUrl` parameter for SSO login. ([\#9128](https://github.com/matrix-org/synapse/issues/9128)) +- Fix "Unhandled error in Deferred: BodyExceededMaxSize" errors when .well-known files that are too large. ([\#9108](https://github.com/matrix-org/synapse/issues/9108)) +- Fix "UnboundLocalError: local variable 'length' referenced before assignment" errors when the response body exceeds the expected size. This bug was introduced in v1.25.0. ([\#9145](https://github.com/matrix-org/synapse/issues/9145)) +- Fix a long-standing bug "ValueError: invalid literal for int() with base 10" when `/publicRooms` is requested with an invalid `server` parameter. ([\#9161](https://github.com/matrix-org/synapse/issues/9161)) + + +Improved Documentation +---------------------- + +- Add some extra docs for getting Synapse running on macOS. ([\#8997](https://github.com/matrix-org/synapse/issues/8997)) +- Correct a typo in the `systemd-with-workers` documentation. ([\#9035](https://github.com/matrix-org/synapse/issues/9035)) +- Correct a typo in `INSTALL.md`. ([\#9040](https://github.com/matrix-org/synapse/issues/9040)) +- Add missing `user_mapping_provider` configuration to the Keycloak OIDC example. Contributed by @chris-ruecker. ([\#9057](https://github.com/matrix-org/synapse/issues/9057)) +- Quote `pip install` packages when extras are used to avoid shells interpreting bracket characters. ([\#9151](https://github.com/matrix-org/synapse/issues/9151)) + + +Deprecations and Removals +------------------------- + +- Remove broken and unmaintained `demo/webserver.py` script. ([\#9039](https://github.com/matrix-org/synapse/issues/9039)) + + +Internal Changes +---------------- + +- Improve efficiency of large state resolutions. ([\#8868](https://github.com/matrix-org/synapse/issues/8868), [\#9029](https://github.com/matrix-org/synapse/issues/9029), [\#9115](https://github.com/matrix-org/synapse/issues/9115), [\#9118](https://github.com/matrix-org/synapse/issues/9118), [\#9124](https://github.com/matrix-org/synapse/issues/9124)) +- Various clean-ups to the structured logging and logging context code. ([\#8939](https://github.com/matrix-org/synapse/issues/8939)) +- Ensure rejected events get added to some metadata tables. ([\#9016](https://github.com/matrix-org/synapse/issues/9016)) +- Ignore date-rotated homeserver logs saved to disk. ([\#9018](https://github.com/matrix-org/synapse/issues/9018)) +- Remove an unused column from `access_tokens` table. ([\#9025](https://github.com/matrix-org/synapse/issues/9025)) +- Add a `-noextras` factor to `tox.ini`, to support running the tests with no optional dependencies. ([\#9030](https://github.com/matrix-org/synapse/issues/9030)) +- Fix running unit tests when optional dependencies are not installed. ([\#9031](https://github.com/matrix-org/synapse/issues/9031)) +- Allow bumping schema version when using split out state database. ([\#9033](https://github.com/matrix-org/synapse/issues/9033)) +- Configure the linters to run on a consistent set of files. ([\#9038](https://github.com/matrix-org/synapse/issues/9038)) +- Various cleanups to device inbox store. ([\#9041](https://github.com/matrix-org/synapse/issues/9041)) +- Drop unused database tables. ([\#9055](https://github.com/matrix-org/synapse/issues/9055)) +- Remove unused `SynapseService` class. ([\#9058](https://github.com/matrix-org/synapse/issues/9058)) +- Remove unnecessary declarations in the tests for the admin API. ([\#9063](https://github.com/matrix-org/synapse/issues/9063)) +- Remove `SynapseRequest.get_user_agent`. ([\#9069](https://github.com/matrix-org/synapse/issues/9069)) +- Remove redundant `Homeserver.get_ip_from_request` method. ([\#9080](https://github.com/matrix-org/synapse/issues/9080)) +- Add type hints to media repository. ([\#9093](https://github.com/matrix-org/synapse/issues/9093)) +- Fix the wrong arguments being passed to `BlacklistingAgentWrapper` from `MatrixFederationAgent`. Contributed by Timothy Leung. ([\#9098](https://github.com/matrix-org/synapse/issues/9098)) +- Reduce the scope of caught exceptions in `BlacklistingAgentWrapper`. ([\#9106](https://github.com/matrix-org/synapse/issues/9106)) +- Improve `UsernamePickerTestCase`. ([\#9112](https://github.com/matrix-org/synapse/issues/9112)) +- Remove dependency on `distutils`. ([\#9125](https://github.com/matrix-org/synapse/issues/9125)) +- Enforce that replication HTTP clients are called with keyword arguments only. ([\#9144](https://github.com/matrix-org/synapse/issues/9144)) +- Fix the Python 3.5 / old dependencies build in CI. ([\#9146](https://github.com/matrix-org/synapse/issues/9146)) +- Replace the old `perspectives` option in the Synapse docker config file template with `trusted_key_servers`. ([\#9157](https://github.com/matrix-org/synapse/issues/9157)) + + +Synapse 1.25.0 (2021-01-13) +=========================== + +Ending Support for Python 3.5 and Postgres 9.5 +---------------------------------------------- + +With this release, the Synapse team is announcing a formal deprecation policy for our platform dependencies, like Python and PostgreSQL: + +All future releases of Synapse will follow the upstream end-of-life schedules. + +Which means: + +* This is the last release which guarantees support for Python 3.5. +* We will end support for PostgreSQL 9.5 early next month. +* We will end support for Python 3.6 and PostgreSQL 9.6 near the end of the year. + +Crucially, this means __we will not produce .deb packages for Debian 9 (Stretch) or Ubuntu 16.04 (Xenial)__ beyond the transition period described below. + +The website https://endoflife.date/ has convenient summaries of the support schedules for projects like [Python](https://endoflife.date/python) and [PostgreSQL](https://endoflife.date/postgresql). + +If you are unable to upgrade your environment to a supported version of Python or +Postgres, we encourage you to consider using the +[Synapse Docker images](https://matrix-org.github.io/synapse/latest/setup/installation.html#docker-images-and-ansible-playbooks) +instead. + +### Transition Period + +We will make a good faith attempt to avoid breaking compatibility in all releases through the end of March 2021. However, critical security vulnerabilities in dependencies or other unanticipated circumstances may arise which necessitate breaking compatibility earlier. + +We intend to continue producing .deb packages for Debian 9 (Stretch) and Ubuntu 16.04 (Xenial) through the transition period. + +Removal warning +--------------- + +The old [Purge Room API](https://github.com/matrix-org/synapse/tree/master/docs/admin_api/purge_room.md) +and [Shutdown Room API](https://github.com/matrix-org/synapse/tree/master/docs/admin_api/shutdown_room.md) +are deprecated and will be removed in a future release. They will be replaced by the +[Delete Room API](https://github.com/matrix-org/synapse/tree/master/docs/admin_api/rooms.md#delete-room-api). + +`POST /_synapse/admin/v1/rooms//delete` replaces `POST /_synapse/admin/v1/purge_room` and +`POST /_synapse/admin/v1/shutdown_room/`. + +Bugfixes +-------- + +- Fix HTTP proxy support when using a proxy that is on a blacklisted IP. Introduced in v1.25.0rc1. Contributed by @Bubu. ([\#9084](https://github.com/matrix-org/synapse/issues/9084)) + + +Synapse 1.25.0rc1 (2021-01-06) +============================== + +Features +-------- + +- Add an admin API that lets server admins get power in rooms in which local users have power. ([\#8756](https://github.com/matrix-org/synapse/issues/8756)) +- Add optional HTTP authentication to replication endpoints. ([\#8853](https://github.com/matrix-org/synapse/issues/8853)) +- Improve the error messages printed as a result of configuration problems for extension modules. ([\#8874](https://github.com/matrix-org/synapse/issues/8874)) +- Add the number of local devices to Room Details Admin API. Contributed by @dklimpel. ([\#8886](https://github.com/matrix-org/synapse/issues/8886)) +- Add `X-Robots-Tag` header to stop web crawlers from indexing media. Contributed by Aaron Raimist. ([\#8887](https://github.com/matrix-org/synapse/issues/8887)) +- Spam-checkers may now define their methods as `async`. ([\#8890](https://github.com/matrix-org/synapse/issues/8890)) +- Add support for allowing users to pick their own user ID during a single-sign-on login. ([\#8897](https://github.com/matrix-org/synapse/issues/8897), [\#8900](https://github.com/matrix-org/synapse/issues/8900), [\#8911](https://github.com/matrix-org/synapse/issues/8911), [\#8938](https://github.com/matrix-org/synapse/issues/8938), [\#8941](https://github.com/matrix-org/synapse/issues/8941), [\#8942](https://github.com/matrix-org/synapse/issues/8942), [\#8951](https://github.com/matrix-org/synapse/issues/8951)) +- Add an `email.invite_client_location` configuration option to send a web client location to the invite endpoint on the identity server which allows customisation of the email template. ([\#8930](https://github.com/matrix-org/synapse/issues/8930)) +- The search term in the list room and list user Admin APIs is now treated as case-insensitive. ([\#8931](https://github.com/matrix-org/synapse/issues/8931)) +- Apply an IP range blacklist to push and key revocation requests. ([\#8821](https://github.com/matrix-org/synapse/issues/8821), [\#8870](https://github.com/matrix-org/synapse/issues/8870), [\#8954](https://github.com/matrix-org/synapse/issues/8954)) +- Add an option to allow re-use of user-interactive authentication sessions for a period of time. ([\#8970](https://github.com/matrix-org/synapse/issues/8970)) +- Allow running the redact endpoint on workers. ([\#8994](https://github.com/matrix-org/synapse/issues/8994)) + + +Bugfixes +-------- + +- Fix bug where we might not correctly calculate the current state for rooms with multiple extremities. ([\#8827](https://github.com/matrix-org/synapse/issues/8827)) +- Fix a long-standing bug in the register admin endpoint (`/_synapse/admin/v1/register`) when the `mac` field was not provided. The endpoint now properly returns a 400 error. Contributed by @edwargix. ([\#8837](https://github.com/matrix-org/synapse/issues/8837)) +- Fix a long-standing bug on Synapse instances supporting Single-Sign-On, where users would be prompted to enter their password to confirm certain actions, even though they have not set a password. ([\#8858](https://github.com/matrix-org/synapse/issues/8858)) +- Fix a longstanding bug where a 500 error would be returned if the `Content-Length` header was not provided to the upload media resource. ([\#8862](https://github.com/matrix-org/synapse/issues/8862)) +- Add additional validation to pusher URLs to be compliant with the specification. ([\#8865](https://github.com/matrix-org/synapse/issues/8865)) +- Fix the error code that is returned when a user tries to register on a homeserver on which new-user registration has been disabled. ([\#8867](https://github.com/matrix-org/synapse/issues/8867)) +- Fix a bug where `PUT /_synapse/admin/v2/users/` failed to create a new user when `avatar_url` is specified. Bug introduced in Synapse v1.9.0. ([\#8872](https://github.com/matrix-org/synapse/issues/8872)) +- Fix a 500 error when attempting to preview an empty HTML file. ([\#8883](https://github.com/matrix-org/synapse/issues/8883)) +- Fix occasional deadlock when handling SIGHUP. ([\#8918](https://github.com/matrix-org/synapse/issues/8918)) +- Fix login API to not ratelimit application services that have ratelimiting disabled. ([\#8920](https://github.com/matrix-org/synapse/issues/8920)) +- Fix bug where we ratelimited auto joining of rooms on registration (using `auto_join_rooms` config). ([\#8921](https://github.com/matrix-org/synapse/issues/8921)) +- Fix a bug where deactivated users appeared in the user directory when their profile information was updated. ([\#8933](https://github.com/matrix-org/synapse/issues/8933), [\#8964](https://github.com/matrix-org/synapse/issues/8964)) +- Fix bug introduced in Synapse v1.24.0 which would cause an exception on startup if both `enabled` and `localdb_enabled` were set to `False` in the `password_config` setting of the configuration file. ([\#8937](https://github.com/matrix-org/synapse/issues/8937)) +- Fix a bug where 500 errors would be returned if the `m.room_history_visibility` event had invalid content. ([\#8945](https://github.com/matrix-org/synapse/issues/8945)) +- Fix a bug causing common English words to not be considered for a user directory search. ([\#8959](https://github.com/matrix-org/synapse/issues/8959)) +- Fix bug where application services couldn't register new ghost users if the server had reached its MAU limit. ([\#8962](https://github.com/matrix-org/synapse/issues/8962)) +- Fix a long-standing bug where a `m.image` event without a `url` would cause errors on push. ([\#8965](https://github.com/matrix-org/synapse/issues/8965)) +- Fix a small bug in v2 state resolution algorithm, which could also cause performance issues for rooms with large numbers of power levels. ([\#8971](https://github.com/matrix-org/synapse/issues/8971)) +- Add validation to the `sendToDevice` API to raise a missing parameters error instead of a 500 error. ([\#8975](https://github.com/matrix-org/synapse/issues/8975)) +- Add validation of group IDs to raise a 400 error instead of a 500 eror. ([\#8977](https://github.com/matrix-org/synapse/issues/8977)) + + +Improved Documentation +---------------------- + +- Fix the "Event persist rate" section of the included grafana dashboard by adding missing prometheus rules. ([\#8802](https://github.com/matrix-org/synapse/issues/8802)) +- Combine related media admin API docs. ([\#8839](https://github.com/matrix-org/synapse/issues/8839)) +- Fix an error in the documentation for the SAML username mapping provider. ([\#8873](https://github.com/matrix-org/synapse/issues/8873)) +- Clarify comments around template directories in `sample_config.yaml`. ([\#8891](https://github.com/matrix-org/synapse/issues/8891)) +- Move instructions for database setup, adjusted heading levels and improved syntax highlighting in [INSTALL.md](../INSTALL.md). Contributed by @fossterer. ([\#8987](https://github.com/matrix-org/synapse/issues/8987)) +- Update the example value of `group_creation_prefix` in the sample configuration. ([\#8992](https://github.com/matrix-org/synapse/issues/8992)) +- Link the Synapse developer room to the development section in the docs. ([\#9002](https://github.com/matrix-org/synapse/issues/9002)) + + +Deprecations and Removals +------------------------- + +- Deprecate Shutdown Room and Purge Room Admin APIs. ([\#8829](https://github.com/matrix-org/synapse/issues/8829)) + + +Internal Changes +---------------- + +- Properly store the mapping of external ID to Matrix ID for CAS users. ([\#8856](https://github.com/matrix-org/synapse/issues/8856), [\#8958](https://github.com/matrix-org/synapse/issues/8958)) +- Remove some unnecessary stubbing from unit tests. ([\#8861](https://github.com/matrix-org/synapse/issues/8861)) +- Remove unused `FakeResponse` class from unit tests. ([\#8864](https://github.com/matrix-org/synapse/issues/8864)) +- Pass `room_id` to `get_auth_chain_difference`. ([\#8879](https://github.com/matrix-org/synapse/issues/8879)) +- Add type hints to push module. ([\#8880](https://github.com/matrix-org/synapse/issues/8880), [\#8882](https://github.com/matrix-org/synapse/issues/8882), [\#8901](https://github.com/matrix-org/synapse/issues/8901), [\#8940](https://github.com/matrix-org/synapse/issues/8940), [\#8943](https://github.com/matrix-org/synapse/issues/8943), [\#9020](https://github.com/matrix-org/synapse/issues/9020)) +- Simplify logic for handling user-interactive-auth via single-sign-on servers. ([\#8881](https://github.com/matrix-org/synapse/issues/8881)) +- Skip the SAML tests if the requirements (`pysaml2` and `xmlsec1`) aren't available. ([\#8905](https://github.com/matrix-org/synapse/issues/8905)) +- Fix multiarch docker image builds. ([\#8906](https://github.com/matrix-org/synapse/issues/8906)) +- Don't publish `latest` docker image until all archs are built. ([\#8909](https://github.com/matrix-org/synapse/issues/8909)) +- Various clean-ups to the structured logging and logging context code. ([\#8916](https://github.com/matrix-org/synapse/issues/8916), [\#8935](https://github.com/matrix-org/synapse/issues/8935)) +- Automatically drop stale forward-extremities under some specific conditions. ([\#8929](https://github.com/matrix-org/synapse/issues/8929)) +- Refactor test utilities for injecting HTTP requests. ([\#8946](https://github.com/matrix-org/synapse/issues/8946)) +- Add a maximum size of 50 kilobytes to .well-known lookups. ([\#8950](https://github.com/matrix-org/synapse/issues/8950)) +- Fix bug in `generate_log_config` script which made it write empty files. ([\#8952](https://github.com/matrix-org/synapse/issues/8952)) +- Clean up tox.ini file; disable coverage checking for non-test runs. ([\#8963](https://github.com/matrix-org/synapse/issues/8963)) +- Add type hints to the admin and room list handlers. ([\#8973](https://github.com/matrix-org/synapse/issues/8973)) +- Add type hints to the receipts and user directory handlers. ([\#8976](https://github.com/matrix-org/synapse/issues/8976)) +- Drop the unused `local_invites` table. ([\#8979](https://github.com/matrix-org/synapse/issues/8979)) +- Add type hints to the base storage code. ([\#8980](https://github.com/matrix-org/synapse/issues/8980)) +- Support using PyJWT v2.0.0 in the test suite. ([\#8986](https://github.com/matrix-org/synapse/issues/8986)) +- Fix `tests.federation.transport.RoomDirectoryFederationTests` and ensure it runs in CI. ([\#8998](https://github.com/matrix-org/synapse/issues/8998)) +- Add type hints to the crypto module. ([\#8999](https://github.com/matrix-org/synapse/issues/8999)) + + +**Changelogs for older versions can be found [here](CHANGES-2020.md).** diff --git a/CHANGES-pre-1.0.md b/docs/changelogs/CHANGES-pre-1.0.md similarity index 100% rename from CHANGES-pre-1.0.md rename to docs/changelogs/CHANGES-pre-1.0.md diff --git a/docs/changelogs/README.md b/docs/changelogs/README.md new file mode 100644 index 0000000000..65b08d9c12 --- /dev/null +++ b/docs/changelogs/README.md @@ -0,0 +1 @@ +This directory contains changelogs for previous years. From 320186319ac4f1d16f8f964d92db8921a4b1073e Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 12 Apr 2022 14:23:43 +0100 Subject: [PATCH 05/60] Resync state after partial-state join (#12394) We work through all the events with partial state, updating the state at each of them. Once it's done, we recalculate the state for the whole room, and then mark the room as having complete state. --- changelog.d/12394.misc | 1 + synapse/handlers/federation.py | 75 +++++++++++++++++++ synapse/handlers/federation_event.py | 39 ++++++++++ synapse/storage/databases/main/events.py | 15 ++++ .../storage/databases/main/events_worker.py | 24 ++++++ synapse/storage/databases/main/room.py | 31 ++++++++ synapse/storage/databases/main/state.py | 48 ++++++++++++ synapse/storage/persist_events.py | 56 ++++++++++++++ 8 files changed, 289 insertions(+) create mode 100644 changelog.d/12394.misc diff --git a/changelog.d/12394.misc b/changelog.d/12394.misc new file mode 100644 index 0000000000..69109fcc37 --- /dev/null +++ b/changelog.d/12394.misc @@ -0,0 +1 @@ +Preparation for faster-room-join work: start a background process to resynchronise the room state after a room join. diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 78d149905f..1434e99056 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -466,6 +466,8 @@ class FederationHandler: ) if ret.partial_state: + # TODO(faster_joins): roll this back if we don't manage to start the + # background resync (eg process_remote_join fails) await self.store.store_partial_state_room(room_id, ret.servers_in_room) max_stream_id = await self._federation_event_handler.process_remote_join( @@ -478,6 +480,18 @@ class FederationHandler: partial_state=ret.partial_state, ) + if ret.partial_state: + # Kick off the process of asynchronously fetching the state for this + # room. + # + # TODO(faster_joins): pick this up again on restart + run_as_background_process( + desc="sync_partial_state_room", + func=self._sync_partial_state_room, + destination=origin, + room_id=room_id, + ) + # We wait here until this instance has seen the events come down # replication (if we're using replication) as the below uses caches. await self._replication.wait_for_stream_position( @@ -1370,3 +1384,64 @@ class FederationHandler: # We fell off the bottom, couldn't get the complexity from anyone. Oh # well. return None + + async def _sync_partial_state_room( + self, + destination: str, + room_id: str, + ) -> None: + """Background process to resync the state of a partial-state room + + Args: + destination: homeserver to pull the state from + room_id: room to be resynced + """ + + # TODO(faster_joins): do we need to lock to avoid races? What happens if other + # worker processes kick off a resync in parallel? Perhaps we should just elect + # a single worker to do the resync. + # + # TODO(faster_joins): what happens if we leave the room during a resync? if we + # really leave, that might mean we have difficulty getting the room state over + # federation. + # + # TODO(faster_joins): try other destinations if the one we have fails + + logger.info("Syncing state for room %s via %s", room_id, destination) + + # we work through the queue in order of increasing stream ordering. + while True: + batch = await self.store.get_partial_state_events_batch(room_id) + if not batch: + # all the events are updated, so we can update current state and + # clear the lazy-loading flag. + logger.info("Updating current state for %s", room_id) + assert ( + self.storage.persistence is not None + ), "TODO(faster_joins): support for workers" + await self.storage.persistence.update_current_state(room_id) + + logger.info("Clearing partial-state flag for %s", room_id) + success = await self.store.clear_partial_state_room(room_id) + if success: + logger.info("State resync complete for %s", room_id) + + # TODO(faster_joins) update room stats and user directory? + return + + # we raced against more events arriving with partial state. Go round + # the loop again. We've already logged a warning, so no need for more. + # TODO(faster_joins): there is still a race here, whereby incoming events which raced + # with us will fail to be persisted after the call to `clear_partial_state_room` due to + # having partial state. + continue + + events = await self.store.get_events_as_list( + batch, + redact_behaviour=EventRedactBehaviour.AS_IS, + allow_rejected=True, + ) + for event in events: + await self._federation_event_handler.update_state_for_partial_state_event( + destination, event + ) diff --git a/synapse/handlers/federation_event.py b/synapse/handlers/federation_event.py index 03c1197c99..32bf02818c 100644 --- a/synapse/handlers/federation_event.py +++ b/synapse/handlers/federation_event.py @@ -477,6 +477,45 @@ class FederationEventHandler: return await self.persist_events_and_notify(room_id, [(event, context)]) + async def update_state_for_partial_state_event( + self, destination: str, event: EventBase + ) -> None: + """Recalculate the state at an event as part of a de-partial-stating process + + Args: + destination: server to request full state from + event: partial-state event to be de-partial-stated + """ + logger.info("Updating state for %s", event.event_id) + with nested_logging_context(suffix=event.event_id): + # if we have all the event's prev_events, then we can work out the + # state based on their states. Otherwise, we request it from the destination + # server. + # + # This is the same operation as we do when we receive a regular event + # over federation. + state = await self._resolve_state_at_missing_prevs(destination, event) + + # build a new state group for it if need be + context = await self._state_handler.compute_event_context( + event, + old_state=state, + ) + if context.partial_state: + # this can happen if some or all of the event's prev_events still have + # partial state - ie, an event has an earlier stream_ordering than one + # or more of its prev_events, so we de-partial-state it before its + # prev_events. + # + # TODO(faster_joins): we probably need to be more intelligent, and + # exclude partial-state prev_events from consideration + logger.warning( + "%s still has partial state: can't de-partial-state it yet", + event.event_id, + ) + return + await self._store.update_state_for_partial_state_event(event, context) + async def backfill( self, dest: str, room_id: str, limit: int, extremities: Collection[str] ) -> None: diff --git a/synapse/storage/databases/main/events.py b/synapse/storage/databases/main/events.py index e3be537cee..2a1e567ce0 100644 --- a/synapse/storage/databases/main/events.py +++ b/synapse/storage/databases/main/events.py @@ -963,6 +963,21 @@ class PersistEventsStore: values=to_insert, ) + async def update_current_state( + self, + room_id: str, + state_delta: DeltaState, + stream_id: int, + ) -> None: + """Update the current state stored in the datatabase for the given room""" + + await self.db_pool.runInteraction( + "update_current_state", + self._update_current_state_txn, + state_delta_by_room={room_id: state_delta}, + stream_id=stream_id, + ) + def _update_current_state_txn( self, txn: LoggingTransaction, diff --git a/synapse/storage/databases/main/events_worker.py b/synapse/storage/databases/main/events_worker.py index a60e3f4fdd..5288cdba03 100644 --- a/synapse/storage/databases/main/events_worker.py +++ b/synapse/storage/databases/main/events_worker.py @@ -1979,3 +1979,27 @@ class EventsWorkerStore(SQLBaseStore): desc="is_partial_state_event", ) return result is not None + + async def get_partial_state_events_batch(self, room_id: str) -> List[str]: + """Get a list of events in the given room that have partial state""" + return await self.db_pool.runInteraction( + "get_partial_state_events_batch", + self._get_partial_state_events_batch_txn, + room_id, + ) + + @staticmethod + def _get_partial_state_events_batch_txn( + txn: LoggingTransaction, room_id: str + ) -> List[str]: + txn.execute( + """ + SELECT event_id FROM partial_state_events AS pse + JOIN events USING (event_id) + WHERE pse.room_id = ? + ORDER BY events.stream_ordering + LIMIT 100 + """, + (room_id,), + ) + return [row[0] for row in txn] diff --git a/synapse/storage/databases/main/room.py b/synapse/storage/databases/main/room.py index 18b1acd9e1..87e9482c60 100644 --- a/synapse/storage/databases/main/room.py +++ b/synapse/storage/databases/main/room.py @@ -1077,6 +1077,37 @@ class RoomWorkerStore(CacheInvalidationWorkerStore): get_rooms_for_retention_period_in_range_txn, ) + async def clear_partial_state_room(self, room_id: str) -> bool: + # this can race with incoming events, so we watch out for FK errors. + # TODO(faster_joins): this still doesn't completely fix the race, since the persist process + # is not atomic. I fear we need an application-level lock. + try: + await self.db_pool.runInteraction( + "clear_partial_state_room", self._clear_partial_state_room_txn, room_id + ) + return True + except self.db_pool.engine.module.DatabaseError as e: + # TODO(faster_joins): how do we distinguish between FK errors and other errors? + logger.warning( + "Exception while clearing lazy partial-state-room %s, retrying: %s", + room_id, + e, + ) + return False + + @staticmethod + def _clear_partial_state_room_txn(txn: LoggingTransaction, room_id: str) -> None: + DatabasePool.simple_delete_txn( + txn, + table="partial_state_rooms_servers", + keyvalues={"room_id": room_id}, + ) + DatabasePool.simple_delete_one_txn( + txn, + table="partial_state_rooms", + keyvalues={"room_id": room_id}, + ) + class _BackgroundUpdates: REMOVE_TOMESTONED_ROOMS_BG_UPDATE = "remove_tombstoned_rooms_from_directory" diff --git a/synapse/storage/databases/main/state.py b/synapse/storage/databases/main/state.py index ecdc1fdc4c..eba35f3700 100644 --- a/synapse/storage/databases/main/state.py +++ b/synapse/storage/databases/main/state.py @@ -21,6 +21,7 @@ from synapse.api.constants import EventTypes, Membership from synapse.api.errors import NotFoundError, UnsupportedRoomVersionError from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, RoomVersion from synapse.events import EventBase +from synapse.events.snapshot import EventContext from synapse.storage._base import SQLBaseStore from synapse.storage.database import ( DatabasePool, @@ -354,6 +355,53 @@ class StateGroupWorkerStore(EventsWorkerStore, SQLBaseStore): return {row["state_group"] for row in rows} + async def update_state_for_partial_state_event( + self, + event: EventBase, + context: EventContext, + ) -> None: + """Update the state group for a partial state event""" + await self.db_pool.runInteraction( + "update_state_for_partial_state_event", + self._update_state_for_partial_state_event_txn, + event, + context, + ) + + def _update_state_for_partial_state_event_txn( + self, + txn, + event: EventBase, + context: EventContext, + ): + # we shouldn't have any outliers here + assert not event.internal_metadata.is_outlier() + + # anything that was rejected should have the same state as its + # predecessor. + if context.rejected: + assert context.state_group == context.state_group_before_event + + self.db_pool.simple_update_txn( + txn, + table="event_to_state_groups", + keyvalues={"event_id": event.event_id}, + updatevalues={"state_group": context.state_group}, + ) + + self.db_pool.simple_delete_one_txn( + txn, + table="partial_state_events", + keyvalues={"event_id": event.event_id}, + ) + + # TODO(faster_joins): need to do something about workers here + txn.call_after( + self._get_state_group_for_event.prefill, + (event.event_id,), + context.state_group, + ) + class MainStateBackgroundUpdateStore(RoomMemberWorkerStore): diff --git a/synapse/storage/persist_events.py b/synapse/storage/persist_events.py index b402922817..e496ba7bed 100644 --- a/synapse/storage/persist_events.py +++ b/synapse/storage/persist_events.py @@ -376,6 +376,62 @@ class EventsPersistenceStorage: pos = PersistedEventPosition(self._instance_name, event_stream_id) return event, pos, self.main_store.get_room_max_token() + async def update_current_state(self, room_id: str) -> None: + """Recalculate the current state for a room, and persist it""" + state = await self._calculate_current_state(room_id) + delta = await self._calculate_state_delta(room_id, state) + + # TODO(faster_joins): get a real stream ordering, to make this work correctly + # across workers. + # + # TODO(faster_joins): this can race against event persistence, in which case we + # will end up with incorrect state. Perhaps we should make this a job we + # farm out to the event persister, somehow. + stream_id = self.main_store.get_room_max_stream_ordering() + await self.persist_events_store.update_current_state(room_id, delta, stream_id) + + async def _calculate_current_state(self, room_id: str) -> StateMap[str]: + """Calculate the current state of a room, based on the forward extremities + + Args: + room_id: room for which to calculate current state + + Returns: + map from (type, state_key) to event id for the current state in the room + """ + latest_event_ids = await self.main_store.get_latest_event_ids_in_room(room_id) + state_groups = set( + ( + await self.main_store._get_state_group_for_events(latest_event_ids) + ).values() + ) + + state_maps_by_state_group = await self.state_store._get_state_for_groups( + state_groups + ) + + if len(state_groups) == 1: + # If there is only one state group, then we know what the current + # state is. + return state_maps_by_state_group[state_groups.pop()] + + # Ok, we need to defer to the state handler to resolve our state sets. + logger.debug("calling resolve_state_groups from preserve_events") + + # Avoid a circular import. + from synapse.state import StateResolutionStore + + room_version = await self.main_store.get_room_version_id(room_id) + res = await self._state_resolution_handler.resolve_state_groups( + room_id, + room_version, + state_maps_by_state_group, + event_map=None, + state_res_store=StateResolutionStore(self.main_store), + ) + + return res.state + async def _persist_event_batch( self, events_and_contexts: List[Tuple[EventBase, EventContext]], From 643c0c50c19ca3c2c2f624a4f3058c559071339f Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 12 Apr 2022 14:56:05 +0100 Subject: [PATCH 06/60] Update UPGRADE notes to include more details about v1.57 upgrade failure mode (#12448) --- docs/upgrade.md | 55 ++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 47 insertions(+), 8 deletions(-) diff --git a/docs/upgrade.md b/docs/upgrade.md index f0c0a3998d..55a6ff7cae 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -98,6 +98,45 @@ without any risk of reusing transaction IDs. Deployments which do not use separate worker processes can be upgraded as normal. Similarly, deployments where no applciation services are in use can be upgraded as normal. +
+Recovering from an incorrect upgrade + +If the database schema is upgraded *without* stopping the worker responsible +for AS traffic, then the following error may be given when attempting to start +a Synapse worker or master process: + +``` +********************************************************************************** + Error during initialisation: + + Postgres sequence 'application_services_txn_id_seq' is inconsistent with associated + table 'application_services_txns'. This can happen if Synapse has been downgraded and + then upgraded again, or due to a bad migration. + + To fix this error, shut down Synapse (including any and all workers) + and run the following SQL: + + SELECT setval('application_services_txn_id_seq', ( + SELECT GREATEST(MAX(txn_id), 0) FROM application_services_txns + )); + + See docs/postgres.md for more information. + + There may be more information in the logs. +********************************************************************************** +``` + +This error may also be seen if Synapse is *downgraded* to an earlier version, +and then upgraded again to v1.57.0 or later. + +In either case: + + 1. Ensure that the worker responsible for AS traffic is stopped. + 2. Run the SQL command given in the error message via `psql`. + +Synapse should then start correctly. +
+ # Upgrading to v1.56.0 ## Open registration without verification is now disabled by default @@ -132,15 +171,15 @@ The `synctl` script [has been made](https://github.com/matrix-org/synapse/pull/12140) an [entry point](https://packaging.python.org/en/latest/specifications/entry-points/) and no longer exists at the root of Synapse's source tree. If you wish to use -`synctl` to manage your homeserver, you should invoke `synctl` directly, e.g. -`synctl start` instead of `./synctl start` or `/path/to/synctl start`. +`synctl` to manage your homeserver, you should invoke `synctl` directly, e.g. +`synctl start` instead of `./synctl start` or `/path/to/synctl start`. You will need to ensure `synctl` is on your `PATH`. - This is automatically the case when using [Debian packages](https://packages.matrix.org/debian/) or [docker images](https://hub.docker.com/r/matrixdotorg/synapse) provided by Matrix.org. - - When installing from a wheel, sdist, or PyPI, a `synctl` executable is added + - When installing from a wheel, sdist, or PyPI, a `synctl` executable is added to your Python installation's `bin`. This should be on your `PATH` automatically, though you might need to activate a virtual environment depending on how you installed Synapse. @@ -176,8 +215,8 @@ the `/_matrix/client/` path. ## Stablisation of MSC3231 -The unstable validity-check endpoint for the -[Registration Tokens](https://spec.matrix.org/v1.2/client-server-api/#get_matrixclientv1registermloginregistration_tokenvalidity) +The unstable validity-check endpoint for the +[Registration Tokens](https://spec.matrix.org/v1.2/client-server-api/#get_matrixclientv1registermloginregistration_tokenvalidity) feature has been stabilised and moved from: `/_matrix/client/unstable/org.matrix.msc3231/register/org.matrix.msc3231.login.registration_token/validity` @@ -191,9 +230,9 @@ Please update any relevant reverse proxy or firewall configurations appropriatel ## Time-based cache expiry is now enabled by default Formerly, entries in the cache were not evicted regardless of whether they were accessed after storing. -This behavior has now changed. By default entries in the cache are now evicted after 30m of not being accessed. -To change the default behavior, go to the `caches` section of the config and change the `expire_caches` and -`cache_entry_ttl` flags as necessary. Please note that these flags replace the `expiry_time` flag in the config. +This behavior has now changed. By default entries in the cache are now evicted after 30m of not being accessed. +To change the default behavior, go to the `caches` section of the config and change the `expire_caches` and +`cache_entry_ttl` flags as necessary. Please note that these flags replace the `expiry_time` flag in the config. The `expiry_time` flag will still continue to work, but it has been deprecated and will be removed in the future. ## Deprecation of `capability` `org.matrix.msc3283.*` From 9b1f3600918f373fc5b9c77c7cf721c71beb917a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 12 Apr 2022 15:04:42 +0100 Subject: [PATCH 07/60] Update changelog --- CHANGES.md | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 0db95bad84..748f6833ca 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -27,7 +27,7 @@ Features Bugfixes -------- -- Fix a long-standing bug that updating the server notices user profile (display name/avatar URL) in the configuration would not be applied to pre-existing rooms. Contributed by Jorge Florian. ([\#12115](https://github.com/matrix-org/synapse/issues/12115)) +- Fix a long-standing bug where updates to the server notices user profile (display name/avatar URL) in the configuration would not be applied to pre-existing rooms. Contributed by Jorge Florian. ([\#12115](https://github.com/matrix-org/synapse/issues/12115)) - Fix a long-standing bug where events from ignored users were still considered for bundled aggregations. ([\#12235](https://github.com/matrix-org/synapse/issues/12235), [\#12338](https://github.com/matrix-org/synapse/issues/12338)) - Fix non-member state events not resolving for historical events when used in [MSC2716](https://github.com/matrix-org/matrix-spec-proposals/pull/2716) `/batch_send` `state_events_at_start`. ([\#12329](https://github.com/matrix-org/synapse/issues/12329)) - Fix a long-standing bug affecting URL previews that would generate a 500 response instead of a 403 if the previewed URL includes a port that isn't allowed by the relevant blacklist. ([\#12333](https://github.com/matrix-org/synapse/issues/12333)) @@ -39,7 +39,8 @@ Bugfixes Updates to the Docker image --------------------------- -- Bundle locked versions of dependencies into the Docker image. ([\#12385](https://github.com/matrix-org/synapse/issues/12385)) +- Bundle locked versions of dependencies into the Docker image. ([\#12385](https://github.com/matrix-org/synapse/issues/12385), [\#12439](https://github.com/matrix-org/synapse/issues/12439)) +- Fix up healthcheck generation for workers docker image. ([\#12405](https://github.com/matrix-org/synapse/issues/12405)) Improved Documentation @@ -50,7 +51,7 @@ Improved Documentation - Update dead links in `check-newsfragment.sh` to point to the correct documentation URL. ([\#12331](https://github.com/matrix-org/synapse/issues/12331)) - Upgrade the version of `mdbook` in CI to 0.4.17. ([\#12339](https://github.com/matrix-org/synapse/issues/12339)) - Updates to the Room DAG concepts development document to clarify that we mark events as outliers because we don't have any state for them. ([\#12345](https://github.com/matrix-org/synapse/issues/12345)) -- Update the link to Redis pub/sub documentation in the workers documentation.. ([\#12369](https://github.com/matrix-org/synapse/issues/12369)) +- Update the link to Redis pub/sub documentation in the workers documentation. ([\#12369](https://github.com/matrix-org/synapse/issues/12369)) - Remove documentation for converting a legacy structured logging configuration to the new format. ([\#12392](https://github.com/matrix-org/synapse/issues/12392)) @@ -66,7 +67,7 @@ Internal Changes - Remove lingering unstable references to MSC2403 (knocking). ([\#12165](https://github.com/matrix-org/synapse/issues/12165)) - Avoid trying to calculate the state at outlier events. ([\#12191](https://github.com/matrix-org/synapse/issues/12191), [\#12316](https://github.com/matrix-org/synapse/issues/12316), [\#12330](https://github.com/matrix-org/synapse/issues/12330), [\#12332](https://github.com/matrix-org/synapse/issues/12332), [\#12409](https://github.com/matrix-org/synapse/issues/12409)) - Omit sending "offline" presence updates to application services after they are initially configured. ([\#12193](https://github.com/matrix-org/synapse/issues/12193)) -- Switch to using a sequence to generate AS transaction IDs. Contributed by Nick Beeper. If running synapse with a dedicated appservice worker, this MUST be stopped before upgrading the main process and database. ([\#12209](https://github.com/matrix-org/synapse/issues/12209)) +- Switch to using a sequence to generate AS transaction IDs. Contributed by Nick @ Beeper. If running synapse with a dedicated appservice worker, this MUST be stopped before upgrading the main process and database. ([\#12209](https://github.com/matrix-org/synapse/issues/12209)) - Add missing type hints for storage. ([\#12267](https://github.com/matrix-org/synapse/issues/12267)) - Add missing type definitions for scripts in docker folder. Contributed by Jorge Florian. ([\#12280](https://github.com/matrix-org/synapse/issues/12280)) - Move [MSC2654](https://github.com/matrix-org/matrix-doc/pull/2654) support behind an experimental configuration flag. ([\#12295](https://github.com/matrix-org/synapse/issues/12295)) @@ -89,23 +90,18 @@ Internal Changes - Allow specifying the Postgres database's port when running unit tests with Postgres. ([\#12376](https://github.com/matrix-org/synapse/issues/12376)) - Remove temporary pin of signedjson<=1.1.1 that was added in Synapse 1.56.0. ([\#12379](https://github.com/matrix-org/synapse/issues/12379)) - Add opentracing spans to calls to external cache. ([\#12380](https://github.com/matrix-org/synapse/issues/12380)) -- Lay groundwork for using `poetry` to manage Synapse's dependencies. ([\#12381](https://github.com/matrix-org/synapse/issues/12381)) +- Lay groundwork for using `poetry` to manage Synapse's dependencies. ([\#12381](https://github.com/matrix-org/synapse/issues/12381), [\#12407](https://github.com/matrix-org/synapse/issues/12407), [\#12412](https://github.com/matrix-org/synapse/issues/12412), [\#12418](https://github.com/matrix-org/synapse/issues/12418)) - Make missing `importlib_metadata` dependency explicit. ([\#12384](https://github.com/matrix-org/synapse/issues/12384), [\#12400](https://github.com/matrix-org/synapse/issues/12400)) - Update type annotations for compatiblity with prometheus_client 0.14. ([\#12389](https://github.com/matrix-org/synapse/issues/12389)) - Remove support for the unstable identifiers specified in [MSC3288](https://github.com/matrix-org/matrix-doc/pull/3288). ([\#12398](https://github.com/matrix-org/synapse/issues/12398)) - Add missing type hints to configuration classes. ([\#12402](https://github.com/matrix-org/synapse/issues/12402)) - Add files used to build the Docker image used for complement testing into the Synapse repository. ([\#12404](https://github.com/matrix-org/synapse/issues/12404)) -- Fix up healthcheck generation for workers docker image. ([\#12405](https://github.com/matrix-org/synapse/issues/12405)) -- Run the olddeps CI job using Poetry. ([\#12407](https://github.com/matrix-org/synapse/issues/12407)) - Do not include groups in the sync response when disabled. ([\#12408](https://github.com/matrix-org/synapse/issues/12408)) -- Run the CI portdb script in the locked poetry environment. ([\#12412](https://github.com/matrix-org/synapse/issues/12412)) - Improve type hints related to HTTP query parameters. ([\#12415](https://github.com/matrix-org/synapse/issues/12415)) -- Run the CI export-data script in the locked poetry environment. ([\#12418](https://github.com/matrix-org/synapse/issues/12418)) - Stop maintaining a list of lint targets. ([\#12420](https://github.com/matrix-org/synapse/issues/12420)) - Make `synapse._scripts` pass type checks. ([\#12421](https://github.com/matrix-org/synapse/issues/12421), [\#12422](https://github.com/matrix-org/synapse/issues/12422)) - Add some type hints to datastore. ([\#12423](https://github.com/matrix-org/synapse/issues/12423)) - Enable certificate checking during complement tests. ([\#12435](https://github.com/matrix-org/synapse/issues/12435)) -- Bundle locked versions of dependencies into the Docker image. ([\#12439](https://github.com/matrix-org/synapse/issues/12439)) - Explicitly specify the `tls` extra for Twisted dependency. ([\#12444](https://github.com/matrix-org/synapse/issues/12444)) From 9535fd0f9c4227be14452271e163c2ddbc3a0a19 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Tue, 12 Apr 2022 10:20:46 -0400 Subject: [PATCH 08/60] Disable groups/communities by default. (#12344) This disables the endpoints (and sync response fields) for groups/communities by default. --- changelog.d/12344.removal | 1 + docs/upgrade.md | 7 +++++++ synapse/config/experimental.py | 2 +- tests/rest/admin/test_admin.py | 1 + 4 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 changelog.d/12344.removal diff --git a/changelog.d/12344.removal b/changelog.d/12344.removal new file mode 100644 index 0000000000..ecefa76d8e --- /dev/null +++ b/changelog.d/12344.removal @@ -0,0 +1 @@ +The groups/communities feature in Synapse has been disabled by default. diff --git a/docs/upgrade.md b/docs/upgrade.md index f0c0a3998d..3d518c81c3 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -85,6 +85,13 @@ process, for example: dpkg -i matrix-synapse-py3_1.3.0+stretch1_amd64.deb ``` +# Upgrading to v1.58.0 + +## Groups/communities feature has been disabled by default + +The non-standard groups/communities feature in Synapse has been disabled by default +and will be removed in Synapse v1.61.0. + # Upgrading to v1.57.0 ## Changes to database schema for application services diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py index 0772dce411..979059e723 100644 --- a/synapse/config/experimental.py +++ b/synapse/config/experimental.py @@ -74,7 +74,7 @@ class ExperimentalConfig(Config): self.msc3720_enabled: bool = experimental.get("msc3720_enabled", False) # The deprecated groups feature. - self.groups_enabled: bool = experimental.get("groups_enabled", True) + self.groups_enabled: bool = experimental.get("groups_enabled", False) # MSC2654: Unread counts self.msc2654_enabled: bool = experimental.get("msc2654_enabled", False) diff --git a/tests/rest/admin/test_admin.py b/tests/rest/admin/test_admin.py index 849d00ab4d..40571b753a 100644 --- a/tests/rest/admin/test_admin.py +++ b/tests/rest/admin/test_admin.py @@ -63,6 +63,7 @@ class DeleteGroupTestCase(unittest.HomeserverTestCase): self.other_user = self.register_user("user", "pass") self.other_user_token = self.login("user", "pass") + @unittest.override_config({"experimental_features": {"groups_enabled": True}}) def test_delete_group(self) -> None: # Create a new group channel = self.make_request( From ba1588461beadff7d5525d05308731ba286c425f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 12 Apr 2022 16:12:57 +0100 Subject: [PATCH 09/60] Fix typos in release script docs (#12450) --- changelog.d/12450.misc | 1 + scripts-dev/release.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 changelog.d/12450.misc diff --git a/changelog.d/12450.misc b/changelog.d/12450.misc new file mode 100644 index 0000000000..4b1c8cba87 --- /dev/null +++ b/changelog.d/12450.misc @@ -0,0 +1 @@ +Fix typo in the release script help string. diff --git a/scripts-dev/release.py b/scripts-dev/release.py index 685fa32b03..518eaf417c 100755 --- a/scripts-dev/release.py +++ b/scripts-dev/release.py @@ -69,11 +69,12 @@ def cli(): # ... wait for assets to build ... ./scripts-dev/release.py publish + ./scripts-dev/release.py upload # Optional: generate some nice links for the announcement - ./scripts-dev/release.py upload + ./scripts-dev/release.py announce If the env var GH_TOKEN (or GITHUB_TOKEN) is set, or passed into the `tag`/`publish` command, then a new draft release will be created/published. From 4bdbebccb98bcc0e8862b1e8b532d9a491a90dac Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Tue, 12 Apr 2022 11:27:45 -0400 Subject: [PATCH 10/60] Remove the unstable event field for `/send_join` per MSC3083. (#12395) This was missed when initially stabilising room version 8 and was left in as a compatibility shim. Most homeservers have upgraded to a version which expects the proper field name, and the failure mode is reasonable (a user on an older server may have to attempt joining the room twice with an obscure error message the first time). --- changelog.d/12395.misc | 1 + synapse/federation/federation_server.py | 2 -- synapse/federation/transport/client.py | 10 ---------- 3 files changed, 1 insertion(+), 12 deletions(-) create mode 100644 changelog.d/12395.misc diff --git a/changelog.d/12395.misc b/changelog.d/12395.misc new file mode 100644 index 0000000000..0a2123b294 --- /dev/null +++ b/changelog.d/12395.misc @@ -0,0 +1 @@ +Remove an unstable identifier from [MSC3083](https://github.com/matrix-org/matrix-doc/pull/3083). diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 69d833585f..e67af6463f 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -687,8 +687,6 @@ class FederationServer(FederationBase): time_now = self._clock.time_msec() event_json = event.get_pdu_json(time_now) resp = { - # TODO Remove the unstable prefix when servers have updated. - "org.matrix.msc3083.v2.event": event_json, "event": event_json, "state": [p.get_pdu_json(time_now) for p in state_events], "auth_chain": [p.get_pdu_json(time_now) for p in auth_chain_events], diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py index 01dc5ca94f..1421050b9a 100644 --- a/synapse/federation/transport/client.py +++ b/synapse/federation/transport/client.py @@ -1380,16 +1380,6 @@ class SendJoinParser(ByteParser[SendJoinResponse]): prefix + "auth_chain.item", use_float=True, ), - # TODO Remove the unstable prefix when servers have updated. - # - # By re-using the same event dictionary this will cause the parsing of - # org.matrix.msc3083.v2.event and event to stomp over each other. - # Generally this should be fine. - ijson.kvitems_coro( - _event_parser(self._response.event_dict), - prefix + "org.matrix.msc3083.v2.event", - use_float=True, - ), ijson.kvitems_coro( _event_parser(self._response.event_dict), prefix + "event", From aa2811026402394b4013033f075d8f509cdc1257 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 12 Apr 2022 16:50:40 +0100 Subject: [PATCH 11/60] Process device list updates asynchronously (#12365) --- changelog.d/12365.feature | 1 + synapse/config/server.py | 8 --- synapse/handlers/device.py | 28 ---------- synapse/storage/databases/main/devices.py | 61 ++++------------------ synapse/storage/schema/__init__.py | 6 +-- tests/federation/test_federation_sender.py | 8 --- tests/storage/test_devices.py | 47 ++++++++++------- 7 files changed, 40 insertions(+), 119 deletions(-) create mode 100644 changelog.d/12365.feature diff --git a/changelog.d/12365.feature b/changelog.d/12365.feature new file mode 100644 index 0000000000..642dea966c --- /dev/null +++ b/changelog.d/12365.feature @@ -0,0 +1 @@ +Enable processing of device list updates asynchronously. diff --git a/synapse/config/server.py b/synapse/config/server.py index 415279d269..d771045b52 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -680,14 +680,6 @@ class ServerConfig(Config): config.get("use_account_validity_in_account_status") or False ) - # This is a temporary option that enables fully using the new - # `device_lists_changes_in_room` without the backwards compat code. This - # is primarily for testing. If enabled the server should *not* be - # downgraded, as it may lead to missing device list updates. - self.use_new_device_lists_changes_in_room = ( - config.get("use_new_device_lists_changes_in_room") or False - ) - self.rooms_to_exclude_from_sync: List[str] = ( config.get("exclude_rooms_from_sync") or [] ) diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py index ffa28b2a30..958599e7b8 100644 --- a/synapse/handlers/device.py +++ b/synapse/handlers/device.py @@ -291,12 +291,6 @@ class DeviceHandler(DeviceWorkerHandler): # On start up check if there are any updates pending. hs.get_reactor().callWhenRunning(self._handle_new_device_update_async) - # Used to decide if we calculate outbound pokes up front or not. By - # default we do to allow safely downgrading Synapse. - self.use_new_device_lists_changes_in_room = ( - hs.config.server.use_new_device_lists_changes_in_room - ) - def _check_device_name_length(self, name: Optional[str]) -> None: """ Checks whether a device name is longer than the maximum allowed length. @@ -490,23 +484,9 @@ class DeviceHandler(DeviceWorkerHandler): room_ids = await self.store.get_rooms_for_user(user_id) - hosts: Optional[Set[str]] = None - if not self.use_new_device_lists_changes_in_room: - hosts = set() - - if self.hs.is_mine_id(user_id): - for room_id in room_ids: - joined_users = await self.store.get_users_in_room(room_id) - hosts.update(get_domain_from_id(u) for u in joined_users) - - set_tag("target_hosts", hosts) - - hosts.discard(self.server_name) - position = await self.store.add_device_change_to_streams( user_id, device_ids, - hosts=hosts, room_ids=room_ids, ) @@ -528,14 +508,6 @@ class DeviceHandler(DeviceWorkerHandler): # We may need to do some processing asynchronously. self._handle_new_device_update_async() - if hosts: - logger.info( - "Sending device list update notif for %r to: %r", user_id, hosts - ) - for host in hosts: - self.federation_sender.send_device_messages(host, immediate=False) - log_kv({"message": "sent device update to host", "host": host}) - async def notify_user_signature_update( self, from_user_id: str, user_ids: List[str] ) -> None: diff --git a/synapse/storage/databases/main/devices.py b/synapse/storage/databases/main/devices.py index dc8009b23d..74e4e2122a 100644 --- a/synapse/storage/databases/main/devices.py +++ b/synapse/storage/databases/main/devices.py @@ -1582,7 +1582,6 @@ class DeviceStore(DeviceWorkerStore, DeviceBackgroundUpdateStore): self, user_id: str, device_ids: Collection[str], - hosts: Optional[Collection[str]], room_ids: Collection[str], ) -> Optional[int]: """Persist that a user's devices have been updated, and which hosts @@ -1592,9 +1591,6 @@ class DeviceStore(DeviceWorkerStore, DeviceBackgroundUpdateStore): user_id: The ID of the user whose device changed. device_ids: The IDs of any changed devices. If empty, this function will return None. - hosts: The remote destinations that should be notified of the change. If - None then the set of hosts have *not* been calculated, and will be - calculated later by a background task. room_ids: The rooms that the user is in Returns: @@ -1606,14 +1602,12 @@ class DeviceStore(DeviceWorkerStore, DeviceBackgroundUpdateStore): context = get_active_span_text_map() - def add_device_changes_txn( - txn, stream_ids_for_device_change, stream_ids_for_outbound_pokes - ): + def add_device_changes_txn(txn, stream_ids): self._add_device_change_to_stream_txn( txn, user_id, device_ids, - stream_ids_for_device_change, + stream_ids, ) self._add_device_outbound_room_poke_txn( @@ -1621,43 +1615,17 @@ class DeviceStore(DeviceWorkerStore, DeviceBackgroundUpdateStore): user_id, device_ids, room_ids, - stream_ids_for_device_change, - context, - hosts_have_been_calculated=hosts is not None, - ) - - # If the set of hosts to send to has not been calculated yet (and so - # `hosts` is None) or there are no `hosts` to send to, then skip - # trying to persist them to the DB. - if not hosts: - return - - self._add_device_outbound_poke_to_stream_txn( - txn, - user_id, - device_ids, - hosts, - stream_ids_for_outbound_pokes, + stream_ids, context, ) - # `device_lists_stream` wants a stream ID per device update. - num_stream_ids = len(device_ids) - - if hosts: - # `device_lists_outbound_pokes` wants a different stream ID for - # each row, which is a row per host per device update. - num_stream_ids += len(hosts) * len(device_ids) - - async with self._device_list_id_gen.get_next_mult(num_stream_ids) as stream_ids: - stream_ids_for_device_change = stream_ids[: len(device_ids)] - stream_ids_for_outbound_pokes = stream_ids[len(device_ids) :] - + async with self._device_list_id_gen.get_next_mult( + len(device_ids) + ) as stream_ids: await self.db_pool.runInteraction( "add_device_change_to_stream", add_device_changes_txn, - stream_ids_for_device_change, - stream_ids_for_outbound_pokes, + stream_ids, ) return stream_ids[-1] @@ -1752,19 +1720,8 @@ class DeviceStore(DeviceWorkerStore, DeviceBackgroundUpdateStore): room_ids: Collection[str], stream_ids: List[str], context: Dict[str, str], - hosts_have_been_calculated: bool, ) -> None: - """Record the user in the room has updated their device. - - Args: - hosts_have_been_calculated: True if `device_lists_outbound_pokes` - has been updated already with the updates. - """ - - # We only need to convert to outbound pokes if they are our user. - converted_to_destinations = ( - hosts_have_been_calculated or not self.hs.is_mine_id(user_id) - ) + """Record the user in the room has updated their device.""" encoded_context = json_encoder.encode(context) @@ -1789,7 +1746,7 @@ class DeviceStore(DeviceWorkerStore, DeviceBackgroundUpdateStore): device_id, room_id, stream_id, - converted_to_destinations, + False, encoded_context, ) for room_id in room_ids diff --git a/synapse/storage/schema/__init__.py b/synapse/storage/schema/__init__.py index 151f2aa9bb..871d4ace12 100644 --- a/synapse/storage/schema/__init__.py +++ b/synapse/storage/schema/__init__.py @@ -66,9 +66,9 @@ Changes in SCHEMA_VERSION = 69: SCHEMA_COMPAT_VERSION = ( - # we now have `state_key` columns in both `events` and `state_events`, so - # now incompatible with synapses wth SCHEMA_VERSION < 66. - 66 + # We now assume that `device_lists_changes_in_room` has been filled out for + # recent device_list_updates. + 69 ) """Limit on how far the synapse codebase can be rolled back without breaking db compat diff --git a/tests/federation/test_federation_sender.py b/tests/federation/test_federation_sender.py index a6e91956af..63ea4f9ee4 100644 --- a/tests/federation/test_federation_sender.py +++ b/tests/federation/test_federation_sender.py @@ -14,7 +14,6 @@ from typing import Optional from unittest.mock import Mock -from parameterized import parameterized_class from signedjson import key, sign from signedjson.types import BaseKey, SigningKey @@ -155,12 +154,6 @@ class FederationSenderReceiptsTestCases(HomeserverTestCase): ) -@parameterized_class( - [ - {"enable_room_poke_code_path": False}, - {"enable_room_poke_code_path": True}, - ] -) class FederationSenderDevicesTestCases(HomeserverTestCase): servlets = [ admin.register_servlets, @@ -175,7 +168,6 @@ class FederationSenderDevicesTestCases(HomeserverTestCase): def default_config(self): c = super().default_config() c["send_federation"] = True - c["use_new_device_lists_changes_in_room"] = self.enable_room_poke_code_path return c def prepare(self, reactor, clock, hs): diff --git a/tests/storage/test_devices.py b/tests/storage/test_devices.py index d1227dd4ac..5491fbf6da 100644 --- a/tests/storage/test_devices.py +++ b/tests/storage/test_devices.py @@ -21,6 +21,29 @@ class DeviceStoreTestCase(HomeserverTestCase): def prepare(self, reactor, clock, hs): self.store = hs.get_datastores().main + def add_device_change(self, user_id, device_ids, host): + """Add a device list change for the given device to + `device_lists_outbound_pokes` table. + """ + + for device_id in device_ids: + stream_id = self.get_success( + self.store.add_device_change_to_streams( + "user_id", [device_id], ["!some:room"] + ) + ) + + self.get_success( + self.store.add_device_list_outbound_pokes( + user_id=user_id, + device_id=device_id, + room_id="!some:room", + stream_id=stream_id, + hosts=[host], + context={}, + ) + ) + def test_store_new_device(self): self.get_success( self.store.store_device("user_id", "device_id", "display_name") @@ -95,11 +118,7 @@ class DeviceStoreTestCase(HomeserverTestCase): device_ids = ["device_id1", "device_id2"] # Add two device updates with sequential `stream_id`s - self.get_success( - self.store.add_device_change_to_streams( - "user_id", device_ids, ["somehost"], ["!some:room"] - ) - ) + self.add_device_change("user_id", device_ids, "somehost") # Get all device updates ever meant for this remote now_stream_id, device_updates = self.get_success( @@ -123,11 +142,7 @@ class DeviceStoreTestCase(HomeserverTestCase): "device_id4", "device_id5", ] - self.get_success( - self.store.add_device_change_to_streams( - "user_id", device_ids, ["somehost"], ["!some:room"] - ) - ) + self.add_device_change("user_id", device_ids, "somehost") # Get device updates meant for this remote next_stream_id, device_updates = self.get_success( @@ -147,11 +162,7 @@ class DeviceStoreTestCase(HomeserverTestCase): # Add some more device updates to ensure it still resumes properly device_ids = ["device_id6", "device_id7"] - self.get_success( - self.store.add_device_change_to_streams( - "user_id", device_ids, ["somehost"], ["!some:room"] - ) - ) + self.add_device_change("user_id", device_ids, "somehost") # Get the next batch of device updates next_stream_id, device_updates = self.get_success( @@ -224,11 +235,7 @@ class DeviceStoreTestCase(HomeserverTestCase): "fakeSelfSigning", ] - self.get_success( - self.store.add_device_change_to_streams( - "@user_id:test", device_ids, ["somehost"], ["!some:room"] - ) - ) + self.add_device_change("@user_id:test", device_ids, "somehost") # Get device updates meant for this remote next_stream_id, device_updates = self.get_success( From 58c657322a18cadf8036819ef87fce9889d65fb0 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 12 Apr 2022 17:35:48 +0100 Subject: [PATCH 12/60] Run lints under poetry in CI; remove lint tox jobs (#12434) Co-authored-by: Dan Callahan --- .github/workflows/tests.yml | 20 +++++++------------- changelog.d/12434.misc | 1 + poetry.lock | 6 +++--- tox.ini | 31 +------------------------------ 4 files changed, 12 insertions(+), 46 deletions(-) create mode 100644 changelog.d/12434.misc diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5a98f61932..c300d6547b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -20,19 +20,13 @@ jobs: - run: scripts-dev/config-lint.sh lint: - runs-on: ubuntu-latest - strategy: - matrix: - toxenv: - - "check_codestyle" - - "check_isort" - - "mypy" - - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - - run: pip install tox - - run: tox -e ${{ matrix.toxenv }} + # This does a vanilla `poetry install` - no extras. I'm slightly anxious + # that we might skip some typechecks on code that uses extras. However, + # I think the right way to fix this is to mark any extras needed for + # typechecking as development dependencies. To detect this, we ought to + # turn up mypy's strictness: disallow unknown imports and be accept fewer + # uses of `Any`. + uses: "matrix-org/backend-meta/.github/workflows/python-poetry-ci.yml@v1" lint-crlf: runs-on: ubuntu-latest diff --git a/changelog.d/12434.misc b/changelog.d/12434.misc new file mode 100644 index 0000000000..88dab428d2 --- /dev/null +++ b/changelog.d/12434.misc @@ -0,0 +1 @@ +Run lints under poetry in CI, and remove corresponding tox lint jobs. diff --git a/poetry.lock b/poetry.lock index bbe8eba96d..761ee03abc 100644 --- a/poetry.lock +++ b/poetry.lock @@ -717,7 +717,7 @@ test = ["appdirs (==1.4.4)", "pytest (>=6)", "pytest-cov (>=2.7)", "pytest-mock [[package]] name = "prometheus-client" -version = "0.13.1" +version = "0.14.0" description = "Python client for the Prometheus monitoring system." category = "main" optional = false @@ -2225,8 +2225,8 @@ platformdirs = [ {file = "platformdirs-2.5.1.tar.gz", hash = "sha256:7535e70dfa32e84d4b34996ea99c5e432fa29a708d0f4e394bbcb2a8faa4f16d"}, ] prometheus-client = [ - {file = "prometheus_client-0.13.1-py3-none-any.whl", hash = "sha256:357a447fd2359b0a1d2e9b311a0c5778c330cfbe186d880ad5a6b39884652316"}, - {file = "prometheus_client-0.13.1.tar.gz", hash = "sha256:ada41b891b79fca5638bd5cfe149efa86512eaa55987893becd2c6d8d0a5dfc5"}, + {file = "prometheus_client-0.14.0-py3-none-any.whl", hash = "sha256:f4aba3fdd1735852049f537c1f0ab177159b7ab76f271ecc4d2f45aa2a1d01f2"}, + {file = "prometheus_client-0.14.0.tar.gz", hash = "sha256:8f7a922dd5455ad524b6ba212ce8eb2b4b05e073f4ec7218287f88b1cac34750"}, ] psycopg2 = [ {file = "psycopg2-2.9.3-cp310-cp310-win32.whl", hash = "sha256:083707a696e5e1c330af2508d8fab36f9700b26621ccbcb538abe22e15485362"}, diff --git a/tox.ini b/tox.ini index b4ce400edf..4cd9dfb966 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py37, py38, py39, py310, check_codestyle, check_isort +envlist = py37, py38, py39, py310 # we require tox>=2.3.2 for the fix to https://github.com/tox-dev/tox/issues/208 minversion = 2.3.2 @@ -32,20 +32,6 @@ deps = # install the "enum34" dependency of cryptography. pip>=10 -# directories/files we run the linters on. -# TODO: this is now out of date; we will remove as part of poetry migration. -lint_targets = - setup.py - synapse - tests - # annoyingly, black doesn't find these so we have to list them - scripts-dev - stubs - contrib - synmark - .ci - docker - # default settings for all tox environments [testenv] deps = @@ -116,18 +102,3 @@ setenv = commands = python -m synmark {posargs:} -[testenv:check_codestyle] -extras = lint -commands = - python -m black --check --diff {[base]lint_targets} - flake8 {[base]lint_targets} {env:PEP8SUFFIX:} - -[testenv:check_isort] -extras = lint -commands = isort -c --df {[base]lint_targets} - -[testenv:mypy] -deps = - {[base]deps} -extras = all,mypy -commands = mypy From 5a275a237700a606fde5c03e4a0f116a1c9aeda8 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 12 Apr 2022 17:41:21 +0100 Subject: [PATCH 13/60] Run "main" trial tests under poetry (#12438) * Run "main" trial tests under poetry Olddeps and twisted trunk tests are handled in separate PRs. The PyPy config is a best-effort only; it's completely untested. Pulled out from #12337. * Changelog --- .github/workflows/tests.yml | 26 +++++++++++++------------- changelog.d/12438.misc | 1 + 2 files changed, 14 insertions(+), 13 deletions(-) create mode 100644 changelog.d/12438.misc diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c300d6547b..7946b76dba 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -65,23 +65,23 @@ jobs: matrix: python-version: ["3.7", "3.8", "3.9", "3.10"] database: ["sqlite"] - toxenv: ["py"] + extras: ["all"] include: # Newest Python without optional deps - python-version: "3.10" - toxenv: "py-noextras" + extras: "" # Oldest Python with PostgreSQL - python-version: "3.7" database: "postgres" postgres-version: "10" - toxenv: "py" + extras: "all" # Newest Python with newest PostgreSQL - python-version: "3.10" database: "postgres" postgres-version: "14" - toxenv: "py" + extras: "all" steps: - uses: actions/checkout@v2 @@ -93,17 +93,16 @@ jobs: -e POSTGRES_PASSWORD=postgres \ -e POSTGRES_INITDB_ARGS="--lc-collate C --lc-ctype C --encoding UTF8" \ postgres:${{ matrix.postgres-version }} - - uses: actions/setup-python@v2 + - uses: matrix-org/setup-python-poetry@v1 with: python-version: ${{ matrix.python-version }} - - run: pip install tox + extras: ${{ matrix.extras }} - name: Await PostgreSQL if: ${{ matrix.postgres-version }} timeout-minutes: 2 run: until pg_isready -h localhost; do sleep 1; done - - run: tox -e ${{ matrix.toxenv }} + - run: poetry run trial --jobs=2 tests env: - TRIAL_FLAGS: "--jobs=2" SYNAPSE_POSTGRES: ${{ matrix.database == 'postgres' || '' }} SYNAPSE_POSTGRES_HOST: localhost SYNAPSE_POSTGRES_USER: postgres @@ -150,23 +149,24 @@ jobs: trial-pypy: # Very slow; only run if the branch name includes 'pypy' + # Note: sqlite only; no postgres. Completely untested since poetry move. if: ${{ contains(github.ref, 'pypy') && !failure() && !cancelled() }} needs: linting-done runs-on: ubuntu-latest strategy: matrix: python-version: ["pypy-3.7"] + extras: ["all"] steps: - uses: actions/checkout@v2 + # Install libs necessary for PyPy to build binary wheels for dependencies - run: sudo apt-get -qq install xmlsec1 libxml2-dev libxslt-dev - - uses: actions/setup-python@v2 + - uses: matrix-org/setup-python-poetry@v1 with: python-version: ${{ matrix.python-version }} - - run: pip install tox - - run: tox -e py - env: - TRIAL_FLAGS: "--jobs=2" + extras: ${{ matrix.extras }} + - run: poetry run trial --jobs=2 tests - name: Dump logs # Logs are most useful when the command fails, always include them. if: ${{ always() }} diff --git a/changelog.d/12438.misc b/changelog.d/12438.misc new file mode 100644 index 0000000000..f2c07a56da --- /dev/null +++ b/changelog.d/12438.misc @@ -0,0 +1 @@ +Run "main" trial tests under `poetry`. From 32545d2e2611743633d9f98a40d5f62791e19a6f Mon Sep 17 00:00:00 2001 From: Shay Date: Tue, 12 Apr 2022 13:46:55 -0700 Subject: [PATCH 14/60] Bump twisted version to the latest in lockfile (#12441) --- changelog.d/12441.misc | 1 + poetry.lock | 19 ++++++++++--------- 2 files changed, 11 insertions(+), 9 deletions(-) create mode 100644 changelog.d/12441.misc diff --git a/changelog.d/12441.misc b/changelog.d/12441.misc new file mode 100644 index 0000000000..c2619f1654 --- /dev/null +++ b/changelog.d/12441.misc @@ -0,0 +1 @@ +Bump twisted version in `poetry.lock` to work around [pip bug #9644](https://github.com/pypa/pip/issues/9644). diff --git a/poetry.lock b/poetry.lock index 761ee03abc..a9f3e61015 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1285,7 +1285,7 @@ urllib3 = ">=1.26.0" [[package]] name = "twisted" -version = "22.2.0" +version = "22.4.0" description = "An asynchronous networking framework written in Python" category = "main" optional = false @@ -1305,19 +1305,20 @@ typing-extensions = ">=3.6.5" "zope.interface" = ">=4.4.2" [package.extras] -all_non_platform = ["cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<4.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)", "contextvars (>=2.4,<3)"] +all_non_platform = ["cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<5.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)", "contextvars (>=2.4,<3)"] conch = ["pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)"] +conch_nacl = ["pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pynacl"] contextvars = ["contextvars (>=2.4,<3)"] dev = ["towncrier (>=19.2,<20.0)", "sphinx-rtd-theme (>=0.5,<1.0)", "readthedocs-sphinx-ext (>=2.1,<3.0)", "sphinx (>=4.1.2,<6)", "pyflakes (>=2.2,<3.0)", "twistedchecker (>=0.7,<1.0)", "coverage (>=6b1,<7)", "python-subunit (>=1.4,<2.0)", "pydoctor (>=21.9.0,<21.10.0)"] dev_release = ["towncrier (>=19.2,<20.0)", "sphinx-rtd-theme (>=0.5,<1.0)", "readthedocs-sphinx-ext (>=2.1,<3.0)", "sphinx (>=4.1.2,<6)", "pydoctor (>=21.9.0,<21.10.0)"] -http2 = ["h2 (>=3.0,<4.0)", "priority (>=1.1.0,<2.0)"] -macos_platform = ["pyobjc-core", "pyobjc-framework-cfnetwork", "pyobjc-framework-cocoa", "cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<4.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)", "contextvars (>=2.4,<3)"] -mypy = ["mypy (==0.930)", "mypy-zope (==0.3.4)", "types-setuptools", "types-pyopenssl", "towncrier (>=19.2,<20.0)", "sphinx-rtd-theme (>=0.5,<1.0)", "readthedocs-sphinx-ext (>=2.1,<3.0)", "sphinx (>=4.1.2,<6)", "pyflakes (>=2.2,<3.0)", "twistedchecker (>=0.7,<1.0)", "coverage (>=6b1,<7)", "cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<4.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)", "python-subunit (>=1.4,<2.0)", "contextvars (>=2.4,<3)", "pydoctor (>=21.9.0,<21.10.0)"] -osx_platform = ["pyobjc-core", "pyobjc-framework-cfnetwork", "pyobjc-framework-cocoa", "cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<4.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)", "contextvars (>=2.4,<3)"] +http2 = ["h2 (>=3.0,<5.0)", "priority (>=1.1.0,<2.0)"] +macos_platform = ["pyobjc-core", "pyobjc-framework-cfnetwork", "pyobjc-framework-cocoa", "cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<5.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)", "contextvars (>=2.4,<3)"] +mypy = ["mypy (==0.930)", "mypy-zope (==0.3.4)", "types-setuptools", "types-pyopenssl", "towncrier (>=19.2,<20.0)", "sphinx-rtd-theme (>=0.5,<1.0)", "readthedocs-sphinx-ext (>=2.1,<3.0)", "sphinx (>=4.1.2,<6)", "pyflakes (>=2.2,<3.0)", "twistedchecker (>=0.7,<1.0)", "coverage (>=6b1,<7)", "cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<5.0)", "priority (>=1.1.0,<2.0)", "pynacl", "pywin32 (!=226)", "python-subunit (>=1.4,<2.0)", "contextvars (>=2.4,<3)", "pydoctor (>=21.9.0,<21.10.0)"] +osx_platform = ["pyobjc-core", "pyobjc-framework-cfnetwork", "pyobjc-framework-cocoa", "cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<5.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)", "contextvars (>=2.4,<3)"] serial = ["pyserial (>=3.0)", "pywin32 (!=226)"] test = ["cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)"] tls = ["pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)"] -windows_platform = ["pywin32 (!=226)", "cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<4.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)", "contextvars (>=2.4,<3)"] +windows_platform = ["pywin32 (!=226)", "cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<5.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)", "contextvars (>=2.4,<3)"] [[package]] name = "twisted-iocpsupport" @@ -2592,8 +2593,8 @@ twine = [ {file = "twine-3.8.0.tar.gz", hash = "sha256:8efa52658e0ae770686a13b675569328f1fba9837e5de1867bfe5f46a9aefe19"}, ] twisted = [ - {file = "Twisted-22.2.0-py3-none-any.whl", hash = "sha256:5c63c149eb6b8fe1e32a0215b1cef96fabdba04f705d8efb9174b1ccf5b49d49"}, - {file = "Twisted-22.2.0.tar.gz", hash = "sha256:57f32b1f6838facb8c004c89467840367ad38e9e535f8252091345dba500b4f2"}, + {file = "Twisted-22.4.0-py3-none-any.whl", hash = "sha256:f9f7a91f94932477a9fc3b169d57f54f96c6e74a23d78d9ce54039a7f48928a2"}, + {file = "Twisted-22.4.0.tar.gz", hash = "sha256:a047990f57dfae1e0bd2b7df2526d4f16dcdc843774dc108b78c52f2a5f13680"}, ] twisted-iocpsupport = [ {file = "twisted-iocpsupport-1.0.2.tar.gz", hash = "sha256:72068b206ee809c9c596b57b5287259ea41ddb4774d86725b19f35bf56aa32a9"}, From 36d8b83888aaa30dd2738f551ecab4f7eb5c206e Mon Sep 17 00:00:00 2001 From: Jonathan de Jong Date: Wed, 13 Apr 2022 11:32:44 +0200 Subject: [PATCH 15/60] Rename Mutual Rooms `unstable_features` flag to match MSC (#12445) Signed-off-by: Jonathan de Jong --- changelog.d/12445.misc | 1 + synapse/rest/client/versions.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/12445.misc diff --git a/changelog.d/12445.misc b/changelog.d/12445.misc new file mode 100644 index 0000000000..954248115a --- /dev/null +++ b/changelog.d/12445.misc @@ -0,0 +1 @@ +Change Mutual Rooms' `unstable_features` flag to `uk.half-shot.msc2666.mutual_rooms` which matches the current MSC iteration. \ No newline at end of file diff --git a/synapse/rest/client/versions.py b/synapse/rest/client/versions.py index 7ee6b5505b..7b29026381 100644 --- a/synapse/rest/client/versions.py +++ b/synapse/rest/client/versions.py @@ -86,7 +86,7 @@ class VersionsRestServlet(RestServlet): # Implements additional endpoints as described in MSC2432 "org.matrix.msc2432": True, # Implements additional endpoints as described in MSC2666 - "uk.half-shot.msc2666": True, + "uk.half-shot.msc2666.mutual_rooms": True, # Whether new rooms will be set to encrypted or not (based on presets). "io.element.e2ee_forced.public": self.e2ee_forced_public, "io.element.e2ee_forced.private": self.e2ee_forced_private, From d24cd17820e9f3bd654cfd923b27ca981100c691 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Wed, 13 Apr 2022 11:26:53 +0100 Subject: [PATCH 16/60] Use poetry lockfile in twisted trunk CI job (#12425) Fixes #12458 Co-authored-by: Sean Quah Co-authored-by: Dan Callahan --- .ci/patch_for_twisted_trunk.sh | 8 ----- .github/workflows/twisted_trunk.yml | 46 +++++++++++++++++++++-------- changelog.d/12425.misc | 1 + 3 files changed, 35 insertions(+), 20 deletions(-) delete mode 100755 .ci/patch_for_twisted_trunk.sh create mode 100644 changelog.d/12425.misc diff --git a/.ci/patch_for_twisted_trunk.sh b/.ci/patch_for_twisted_trunk.sh deleted file mode 100755 index f524581986..0000000000 --- a/.ci/patch_for_twisted_trunk.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh - -# replaces the dependency on Twisted in `python_dependencies` with trunk. - -set -e -cd "$(dirname "$0")"/.. - -sed -i -e 's#"Twisted.*"#"Twisted @ git+https://github.com/twisted/twisted"#' synapse/python_dependencies.py diff --git a/.github/workflows/twisted_trunk.yml b/.github/workflows/twisted_trunk.yml index fb9d46b7bf..8fc1affb77 100644 --- a/.github/workflows/twisted_trunk.yml +++ b/.github/workflows/twisted_trunk.yml @@ -6,16 +6,25 @@ on: workflow_dispatch: +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: mypy: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - - run: .ci/patch_for_twisted_trunk.sh - - run: pip install tox - - run: tox -e mypy + - uses: matrix-org/setup-python-poetry@v1 + with: + python-version: "3.x" + extras: "all" + - run: | + poetry remove twisted + poetry add --extras tls git+https://github.com/twisted/twisted.git#trunk + poetry install --no-interaction --extras "all test" + - run: poetry run mypy trial: runs-on: ubuntu-latest @@ -23,14 +32,15 @@ jobs: steps: - uses: actions/checkout@v2 - run: sudo apt-get -qq install xmlsec1 - - uses: actions/setup-python@v2 + - uses: matrix-org/setup-python-poetry@v1 with: - python-version: 3.7 - - run: .ci/patch_for_twisted_trunk.sh - - run: pip install tox - - run: tox -e py - env: - TRIAL_FLAGS: "--jobs=2" + python-version: "3.x" + extras: "all test" + - run: | + poetry remove twisted + poetry add --extras tls git+https://github.com/twisted/twisted.git#trunk + poetry install --no-interaction --extras "all test" + - run: poetry run trial --jobs 2 tests - name: Dump logs # Logs are most useful when the command fails, always include them. @@ -55,11 +65,23 @@ jobs: steps: - uses: actions/checkout@v2 - name: Patch dependencies - run: .ci/patch_for_twisted_trunk.sh + # Note: The poetry commands want to create a virtualenv in /src/.venv/, + # but the sytest-synapse container expects it to be in /venv/. + # We symlink it before running poetry so that poetry actually + # ends up installing to `/venv`. + run: | + ln -s -T /venv /src/.venv + poetry remove twisted + poetry add --extras tls git+https://github.com/twisted/twisted.git#trunk + poetry install --no-interaction --extras "all test" working-directory: /src - name: Run SyTest run: /bootstrap.sh synapse working-directory: /src + env: + # Use offline mode to avoid reinstalling the pinned version of + # twisted. + OFFLINE: 1 - name: Summarise results.tap if: ${{ always() }} run: /sytest/scripts/tap_to_gha.pl /logs/results.tap diff --git a/changelog.d/12425.misc b/changelog.d/12425.misc new file mode 100644 index 0000000000..3b076be0bd --- /dev/null +++ b/changelog.d/12425.misc @@ -0,0 +1 @@ +Run twisted trunk CI job in the locked poetry environment. From e3a49f4784d5c915355ac9306e60b09433db60b5 Mon Sep 17 00:00:00 2001 From: Nick Mills-Barrett Date: Wed, 13 Apr 2022 11:38:35 +0100 Subject: [PATCH 17/60] Fix missing sync events during historical batch imports (#12319) Discovered after much in-depth investigation in #12281. Closes: #12281 Closes: #3305 Signed off by: Nick Mills-Barrett nick@beeper.com --- changelog.d/12319.bugfix | 1 + synapse/handlers/message.py | 14 +-- synapse/handlers/sync.py | 15 ++- synapse/storage/databases/main/stream.py | 26 +++++ tests/rest/client/test_room_batch.py | 125 ++++++++++++++++++++++- 5 files changed, 162 insertions(+), 19 deletions(-) create mode 100644 changelog.d/12319.bugfix diff --git a/changelog.d/12319.bugfix b/changelog.d/12319.bugfix new file mode 100644 index 0000000000..a50191feaa --- /dev/null +++ b/changelog.d/12319.bugfix @@ -0,0 +1 @@ +Fix bug with incremental sync missing events when rejoining/backfilling. Contributed by Nick @ Beeper. diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 47a63005a9..1b092e900e 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -175,17 +175,13 @@ class MessageHandler: state_filter = state_filter or StateFilter.all() if at_token: - # FIXME this claims to get the state at a stream position, but - # get_recent_events_for_room operates by topo ordering. This therefore - # does not reliably give you the state at the given stream position. - # (https://github.com/matrix-org/synapse/issues/3305) - last_events, _ = await self.store.get_recent_events_for_room( - room_id, end_token=at_token.room_key, limit=1 + last_event = await self.store.get_last_event_in_room_before_stream_ordering( + room_id, + end_token=at_token.room_key, ) - if not last_events: + if not last_event: raise NotFoundError("Can't find event for token %s" % (at_token,)) - last_event = last_events[0] # check whether the user is in the room at that time to determine # whether they should be treated as peeking. @@ -204,7 +200,7 @@ class MessageHandler: visible_events = await filter_events_for_client( self.storage, user_id, - last_events, + [last_event], filter_send_to_client=False, is_peeking=is_peeking, ) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 6c8b17c420..5125126a80 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -661,16 +661,15 @@ class SyncHandler: stream_position: point at which to get state state_filter: The state filter used to fetch state from the database. """ - # FIXME this claims to get the state at a stream position, but - # get_recent_events_for_room operates by topo ordering. This therefore - # does not reliably give you the state at the given stream position. - # (https://github.com/matrix-org/synapse/issues/3305) - last_events, _ = await self.store.get_recent_events_for_room( - room_id, end_token=stream_position.room_key, limit=1 + # FIXME: This gets the state at the latest event before the stream ordering, + # which might not be the same as the "current state" of the room at the time + # of the stream token if there were multiple forward extremities at the time. + last_event = await self.store.get_last_event_in_room_before_stream_ordering( + room_id, + end_token=stream_position.room_key, ) - if last_events: - last_event = last_events[-1] + if last_event: state = await self.get_state_after_event( last_event, state_filter=state_filter or StateFilter.all() ) diff --git a/synapse/storage/databases/main/stream.py b/synapse/storage/databases/main/stream.py index 6d45a8a9f6..793e906630 100644 --- a/synapse/storage/databases/main/stream.py +++ b/synapse/storage/databases/main/stream.py @@ -758,6 +758,32 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore): "get_room_event_before_stream_ordering", _f ) + async def get_last_event_in_room_before_stream_ordering( + self, + room_id: str, + end_token: RoomStreamToken, + ) -> Optional[EventBase]: + """Returns the last event in a room at or before a stream ordering + + Args: + room_id + end_token: The token used to stream from + + Returns: + The most recent event. + """ + + last_row = await self.get_room_event_before_stream_ordering( + room_id=room_id, + stream_ordering=end_token.stream, + ) + if last_row: + _, _, event_id = last_row + event = await self.get_event(event_id, get_prev_content=True) + return event + + return None + async def get_current_room_stream_token_for_room_id( self, room_id: Optional[str] = None ) -> RoomStreamToken: diff --git a/tests/rest/client/test_room_batch.py b/tests/rest/client/test_room_batch.py index 44f333a0ee..41a1bf6d89 100644 --- a/tests/rest/client/test_room_batch.py +++ b/tests/rest/client/test_room_batch.py @@ -7,9 +7,9 @@ from twisted.test.proto_helpers import MemoryReactor from synapse.api.constants import EventContentFields, EventTypes from synapse.appservice import ApplicationService from synapse.rest import admin -from synapse.rest.client import login, register, room, room_batch +from synapse.rest.client import login, register, room, room_batch, sync from synapse.server import HomeServer -from synapse.types import JsonDict +from synapse.types import JsonDict, RoomStreamToken from synapse.util import Clock from tests import unittest @@ -63,6 +63,7 @@ class RoomBatchTestCase(unittest.HomeserverTestCase): room.register_servlets, register.register_servlets, login.register_servlets, + sync.register_servlets, ] def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: @@ -178,3 +179,123 @@ class RoomBatchTestCase(unittest.HomeserverTestCase): "Expected a single state_group to be returned by saw state_groups=%s" % (state_group_map.keys(),), ) + + @unittest.override_config({"experimental_features": {"msc2716_enabled": True}}) + def test_sync_while_batch_importing(self) -> None: + """ + Make sure that /sync correctly returns full room state when a user joins + during ongoing batch backfilling. + See: https://github.com/matrix-org/synapse/issues/12281 + """ + # Create user who will be invited & join room + user_id = self.register_user("beep", "test") + user_tok = self.login("beep", "test") + + time_before_room = int(self.clock.time_msec()) + + # Create a room with some events + room_id, _, _, _ = self._create_test_room() + # Invite the user + self.helper.invite( + room_id, src=self.appservice.sender, tok=self.appservice.token, targ=user_id + ) + + # Create another room, send a bunch of events to advance the stream token + other_room_id = self.helper.create_room_as( + self.appservice.sender, tok=self.appservice.token + ) + for _ in range(5): + self.helper.send_event( + room_id=other_room_id, + type=EventTypes.Message, + content={"msgtype": "m.text", "body": "C"}, + tok=self.appservice.token, + ) + + # Join the room as the normal user + self.helper.join(room_id, user_id, tok=user_tok) + + # Create an event to hang the historical batch from - In order to see + # the failure case originally reported in #12281, the historical batch + # must be hung from the most recent event in the room so the base + # insertion event ends up with the highest `topogological_ordering` + # (`depth`) in the room but will have a negative `stream_ordering` + # because it's a `historical` event. Previously, when assembling the + # `state` for the `/sync` response, the bugged logic would sort by + # `topological_ordering` descending and pick up the base insertion + # event because it has a negative `stream_ordering` below the given + # pagination token. Now we properly sort by `stream_ordering` + # descending which puts `historical` events with a negative + # `stream_ordering` way at the bottom and aren't selected as expected. + response = self.helper.send_event( + room_id=room_id, + type=EventTypes.Message, + content={ + "msgtype": "m.text", + "body": "C", + }, + tok=self.appservice.token, + ) + event_to_hang_id = response["event_id"] + + channel = self.make_request( + "POST", + "/_matrix/client/unstable/org.matrix.msc2716/rooms/%s/batch_send?prev_event_id=%s" + % (room_id, event_to_hang_id), + content={ + "events": _create_message_events_for_batch_send_request( + self.virtual_user_id, time_before_room, 3 + ), + "state_events_at_start": _create_join_state_events_for_batch_send_request( + [self.virtual_user_id], time_before_room + ), + }, + access_token=self.appservice.token, + ) + self.assertEqual(channel.code, 200, channel.result) + + # Now we need to find the invite + join events stream tokens so we can sync between + main_store = self.hs.get_datastores().main + events, next_key = self.get_success( + main_store.get_recent_events_for_room( + room_id, + 50, + end_token=main_store.get_room_max_token(), + ), + ) + invite_event_position = None + for event in events: + if ( + event.type == "m.room.member" + and event.content["membership"] == "invite" + ): + invite_event_position = self.get_success( + main_store.get_topological_token_for_event(event.event_id) + ) + break + + assert invite_event_position is not None, "No invite event found" + + # Remove the topological order from the token by re-creating w/stream only + invite_event_position = RoomStreamToken(None, invite_event_position.stream) + + # Sync everything after this token + since_token = self.get_success(invite_event_position.to_string(main_store)) + sync_response = self.make_request( + "GET", + f"/sync?since={since_token}", + access_token=user_tok, + ) + + # Assert that, for this room, the user was considered to have joined and thus + # receives the full state history + state_event_types = [ + event["type"] + for event in sync_response.json_body["rooms"]["join"][room_id]["state"][ + "events" + ] + ] + + assert ( + "m.room.create" in state_event_types + ), "Missing room full state in sync response" From 73d8ded0b030a81e828c07bb134c08db67569e5d Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 13 Apr 2022 16:21:07 +0100 Subject: [PATCH 18/60] Prevent a sync request from removing a user's busy presence status (#12213) In trying to use the MSC3026 busy presence status, the user's status would be set back to 'online' next time they synced. This change makes it so that syncing does not affect a user's presence status if it is currently set to 'busy': it must be removed through the presence API. The MSC defers to implementations on the behaviour of busy presence, so this ought to remain compatible with the MSC. --- changelog.d/12213.bugfix | 1 + synapse/handlers/events.py | 6 ++- synapse/handlers/presence.py | 56 ++++++++++++++++++----- synapse/rest/client/sync.py | 9 ++-- tests/handlers/test_presence.py | 79 +++++++++++++++++++++++++++++++++ 5 files changed, 133 insertions(+), 18 deletions(-) create mode 100644 changelog.d/12213.bugfix diff --git a/changelog.d/12213.bugfix b/changelog.d/12213.bugfix new file mode 100644 index 0000000000..9278e3a9c1 --- /dev/null +++ b/changelog.d/12213.bugfix @@ -0,0 +1 @@ +Prevent a sync request from removing a user's busy presence status. diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py index d2ccb5c5d3..e89c4df314 100644 --- a/synapse/handlers/events.py +++ b/synapse/handlers/events.py @@ -16,7 +16,7 @@ import logging import random from typing import TYPE_CHECKING, Iterable, List, Optional -from synapse.api.constants import EduTypes, EventTypes, Membership +from synapse.api.constants import EduTypes, EventTypes, Membership, PresenceState from synapse.api.errors import AuthError, SynapseError from synapse.events import EventBase from synapse.events.utils import SerializeEventConfig @@ -67,7 +67,9 @@ class EventStreamHandler: presence_handler = self.hs.get_presence_handler() context = await presence_handler.user_syncing( - auth_user_id, affect_presence=affect_presence + auth_user_id, + affect_presence=affect_presence, + presence_state=PresenceState.ONLINE, ) with context: if timeout: diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 209a4b0e52..d078162c29 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -151,7 +151,7 @@ class BasePresenceHandler(abc.ABC): @abc.abstractmethod async def user_syncing( - self, user_id: str, affect_presence: bool + self, user_id: str, affect_presence: bool, presence_state: str ) -> ContextManager[None]: """Returns a context manager that should surround any stream requests from the user. @@ -165,6 +165,7 @@ class BasePresenceHandler(abc.ABC): affect_presence: If false this function will be a no-op. Useful for streams that are not associated with an actual client that is being used by a user. + presence_state: The presence state indicated in the sync request """ @abc.abstractmethod @@ -228,6 +229,11 @@ class BasePresenceHandler(abc.ABC): return states + async def current_state_for_user(self, user_id: str) -> UserPresenceState: + """Get the current presence state for a user.""" + res = await self.current_state_for_users([user_id]) + return res[user_id] + @abc.abstractmethod async def set_state( self, @@ -461,7 +467,7 @@ class WorkerPresenceHandler(BasePresenceHandler): self.send_user_sync(user_id, False, last_sync_ms) async def user_syncing( - self, user_id: str, affect_presence: bool + self, user_id: str, affect_presence: bool, presence_state: str ) -> ContextManager[None]: """Record that a user is syncing. @@ -471,6 +477,17 @@ class WorkerPresenceHandler(BasePresenceHandler): if not affect_presence or not self._presence_enabled: return _NullContextManager() + prev_state = await self.current_state_for_user(user_id) + if prev_state != PresenceState.BUSY: + # We set state here but pass ignore_status_msg = True as we don't want to + # cause the status message to be cleared. + # Note that this causes last_active_ts to be incremented which is not + # what the spec wants: see comment in the BasePresenceHandler version + # of this function. + await self.set_state( + UserID.from_string(user_id), {"presence": presence_state}, True + ) + curr_sync = self._user_to_num_current_syncs.get(user_id, 0) self._user_to_num_current_syncs[user_id] = curr_sync + 1 @@ -942,7 +959,10 @@ class PresenceHandler(BasePresenceHandler): await self._update_states([prev_state.copy_and_replace(**new_fields)]) async def user_syncing( - self, user_id: str, affect_presence: bool = True + self, + user_id: str, + affect_presence: bool = True, + presence_state: str = PresenceState.ONLINE, ) -> ContextManager[None]: """Returns a context manager that should surround any stream requests from the user. @@ -956,6 +976,7 @@ class PresenceHandler(BasePresenceHandler): affect_presence: If false this function will be a no-op. Useful for streams that are not associated with an actual client that is being used by a user. + presence_state: The presence state indicated in the sync request """ # Override if it should affect the user's presence, if presence is # disabled. @@ -967,9 +988,25 @@ class PresenceHandler(BasePresenceHandler): self.user_to_num_current_syncs[user_id] = curr_sync + 1 prev_state = await self.current_state_for_user(user_id) + + # If they're busy then they don't stop being busy just by syncing, + # so just update the last sync time. + if prev_state.state != PresenceState.BUSY: + # XXX: We set_state separately here and just update the last_active_ts above + # This keeps the logic as similar as possible between the worker and single + # process modes. Using set_state will actually cause last_active_ts to be + # updated always, which is not what the spec calls for, but synapse has done + # this for... forever, I think. + await self.set_state( + UserID.from_string(user_id), {"presence": presence_state}, True + ) + # Retrieve the new state for the logic below. This should come from the + # in-memory cache. + prev_state = await self.current_state_for_user(user_id) + + # To keep the single process behaviour consistent with worker mode, run the + # same logic as `update_external_syncs_row`, even though it looks weird. if prev_state.state == PresenceState.OFFLINE: - # If they're currently offline then bring them online, otherwise - # just update the last sync times. await self._update_states( [ prev_state.copy_and_replace( @@ -979,6 +1016,10 @@ class PresenceHandler(BasePresenceHandler): ) ] ) + # otherwise, set the new presence state & update the last sync time, + # but don't update last_active_ts as this isn't an indication that + # they've been active (even though it's probably been updated by + # set_state above) else: await self._update_states( [ @@ -1086,11 +1127,6 @@ class PresenceHandler(BasePresenceHandler): ) self.external_process_last_updated_ms.pop(process_id, None) - async def current_state_for_user(self, user_id: str) -> UserPresenceState: - """Get the current presence state for a user.""" - res = await self.current_state_for_users([user_id]) - return res[user_id] - async def _persist_and_notify(self, states: List[UserPresenceState]) -> None: """Persist states in the database, poke the notifier and send to interested remote servers diff --git a/synapse/rest/client/sync.py b/synapse/rest/client/sync.py index 2e25e8638b..e8772f86e7 100644 --- a/synapse/rest/client/sync.py +++ b/synapse/rest/client/sync.py @@ -180,13 +180,10 @@ class SyncRestServlet(RestServlet): affect_presence = set_presence != PresenceState.OFFLINE - if affect_presence: - await self.presence_handler.set_state( - user, {"presence": set_presence}, True - ) - context = await self.presence_handler.user_syncing( - user.to_string(), affect_presence=affect_presence + user.to_string(), + affect_presence=affect_presence, + presence_state=set_presence, ) with context: sync_result = await self.sync_handler.wait_for_sync_for_user( diff --git a/tests/handlers/test_presence.py b/tests/handlers/test_presence.py index b2ed9cbe37..c96dc6caf2 100644 --- a/tests/handlers/test_presence.py +++ b/tests/handlers/test_presence.py @@ -657,6 +657,85 @@ class PresenceHandlerTestCase(unittest.HomeserverTestCase): # Mark user as online and `status_msg = None` self._set_presencestate_with_status_msg(user_id, PresenceState.ONLINE, None) + def test_set_presence_from_syncing_not_set(self): + """Test that presence is not set by syncing if affect_presence is false""" + user_id = "@test:server" + status_msg = "I'm here!" + + self._set_presencestate_with_status_msg( + user_id, PresenceState.UNAVAILABLE, status_msg + ) + + self.get_success( + self.presence_handler.user_syncing(user_id, False, PresenceState.ONLINE) + ) + + state = self.get_success( + self.presence_handler.get_state(UserID.from_string(user_id)) + ) + # we should still be unavailable + self.assertEqual(state.state, PresenceState.UNAVAILABLE) + # and status message should still be the same + self.assertEqual(state.status_msg, status_msg) + + def test_set_presence_from_syncing_is_set(self): + """Test that presence is set by syncing if affect_presence is true""" + user_id = "@test:server" + status_msg = "I'm here!" + + self._set_presencestate_with_status_msg( + user_id, PresenceState.UNAVAILABLE, status_msg + ) + + self.get_success( + self.presence_handler.user_syncing(user_id, True, PresenceState.ONLINE) + ) + + state = self.get_success( + self.presence_handler.get_state(UserID.from_string(user_id)) + ) + # we should now be online + self.assertEqual(state.state, PresenceState.ONLINE) + + def test_set_presence_from_syncing_keeps_status(self): + """Test that presence set by syncing retains status message""" + user_id = "@test:server" + status_msg = "I'm here!" + + self._set_presencestate_with_status_msg( + user_id, PresenceState.UNAVAILABLE, status_msg + ) + + self.get_success( + self.presence_handler.user_syncing(user_id, True, PresenceState.ONLINE) + ) + + state = self.get_success( + self.presence_handler.get_state(UserID.from_string(user_id)) + ) + # our status message should be the same as it was before + self.assertEqual(state.status_msg, status_msg) + + def test_set_presence_from_syncing_keeps_busy(self): + """Test that presence set by syncing doesn't affect busy status""" + # while this isn't the default + self.presence_handler._busy_presence_enabled = True + + user_id = "@test:server" + status_msg = "I'm busy!" + + self._set_presencestate_with_status_msg(user_id, PresenceState.BUSY, status_msg) + + self.get_success( + self.presence_handler.user_syncing(user_id, True, PresenceState.ONLINE) + ) + + state = self.get_success( + self.presence_handler.get_state(UserID.from_string(user_id)) + ) + # we should still be busy + self.assertEqual(state.state, PresenceState.BUSY) + def _set_presencestate_with_status_msg( self, user_id: str, state: str, status_msg: Optional[str] ): From 0922462fc7df951e88c8ec0fb35e53e3cd801b76 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Wed, 13 Apr 2022 08:27:51 -0700 Subject: [PATCH 19/60] docs: Don't render the table of contents on the print page (#12340) --- changelog.d/12340.doc | 1 + docs/website_files/table-of-contents.js | 14 ++++++++++++++ 2 files changed, 15 insertions(+) create mode 100644 changelog.d/12340.doc diff --git a/changelog.d/12340.doc b/changelog.d/12340.doc new file mode 100644 index 0000000000..8354f2259e --- /dev/null +++ b/changelog.d/12340.doc @@ -0,0 +1 @@ +Fix rendering of the documentation site when using the 'print' feature. diff --git a/docs/website_files/table-of-contents.js b/docs/website_files/table-of-contents.js index 0de5960b22..772da97fb9 100644 --- a/docs/website_files/table-of-contents.js +++ b/docs/website_files/table-of-contents.js @@ -75,6 +75,20 @@ function setTocEntry() { * Populate sidebar on load */ window.addEventListener('load', () => { + // Prevent rendering the table of contents of the "print book" page, as it + // will end up being rendered into the output (in a broken-looking way) + + // Get the name of the current page (i.e. 'print.html') + const pageNameExtension = window.location.pathname.split('/').pop(); + + // Split off the extension (as '.../print' is also a valid page name), which + // should result in 'print' + const pageName = pageNameExtension.split('.')[0]; + if (pageName === "print") { + // Don't render the table of contents on this page + return; + } + // Only create table of contents if there is more than one header on the page if (headers.length <= 1) { return; From 8e2759f2d86d68fa621ba51ae73171e25fe9510d Mon Sep 17 00:00:00 2001 From: Shay Date: Wed, 13 Apr 2022 10:04:01 -0700 Subject: [PATCH 20/60] Limit `device_id` size to 512B (#12454) * --- changelog.d/12454.misc | 1 + synapse/rest/client/login.py | 9 +++++++++ tests/rest/client/test_login.py | 27 ++++++++++++++++++++++++++- 3 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 changelog.d/12454.misc diff --git a/changelog.d/12454.misc b/changelog.d/12454.misc new file mode 100644 index 0000000000..cb7ff74b4c --- /dev/null +++ b/changelog.d/12454.misc @@ -0,0 +1 @@ +Limit length of device_id to less than 512 characters. diff --git a/synapse/rest/client/login.py b/synapse/rest/client/login.py index c9d44c5964..4a4dbe75de 100644 --- a/synapse/rest/client/login.py +++ b/synapse/rest/client/login.py @@ -342,6 +342,15 @@ class LoginRestServlet(RestServlet): user_id = canonical_uid device_id = login_submission.get("device_id") + + # If device_id is present, check that device_id is not longer than a reasonable 512 characters + if device_id and len(device_id) > 512: + raise LoginError( + 400, + "device_id cannot be longer than 512 characters.", + errcode=Codes.INVALID_PARAM, + ) + initial_display_name = login_submission.get("initial_device_display_name") ( device_id, diff --git a/tests/rest/client/test_login.py b/tests/rest/client/test_login.py index 090d2d0a29..0a3d017dc9 100644 --- a/tests/rest/client/test_login.py +++ b/tests/rest/client/test_login.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +import json import time import urllib.parse from typing import Any, Dict, List, Optional, Union @@ -384,6 +384,31 @@ class LoginRestServletTestCase(unittest.HomeserverTestCase): channel = self.make_request(b"POST", "/logout/all", access_token=access_token) self.assertEqual(channel.result["code"], b"200", channel.result) + def test_login_with_overly_long_device_id_fails(self) -> None: + self.register_user("mickey", "cheese") + + # create a device_id longer than 512 characters + device_id = "yolo" * 512 + + body = { + "type": "m.login.password", + "user": "mickey", + "password": "cheese", + "device_id": device_id, + } + + # make a login request with the bad device_id + channel = self.make_request( + "POST", + "/_matrix/client/v3/login", + json.dumps(body).encode("utf8"), + custom_headers=None, + ) + + # test that the login fails with the correct error code + self.assertEqual(channel.code, 400) + self.assertEqual(channel.json_body["errcode"], "M_INVALID_PARAM") + @skip_unless(has_saml2 and HAS_OIDC, "Requires SAML2 and OIDC") class MultiSSOTestCase(unittest.HomeserverTestCase): From 8af8a9bce5319ebf99573433f9f89d94767e173c Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Wed, 13 Apr 2022 20:50:08 +0100 Subject: [PATCH 21/60] Dockerfile-workers: give the master its own log config (#12466) When we run a worker-mode synapse under docker, everything gets logged to stdout. Currently, output from the workers is tacked with a worker name, for example: ``` 2022-04-13 15:27:56,810 - worker:frontend_proxy1 - synapse.util.caches.lrucache - 154 - INFO - LruCache._expire_old_entries-0 - Dropped 0 items from caches ``` - note `worker:frontend_proxy1`. No such tag is applied to log lines from the master, which makes somewhat confusing reading. To fix this, we generate a dedicated log config file for the master in the same way that we do for the workers, and use that. --- changelog.d/12466.misc | 1 + docker/configure_workers_and_start.py | 48 +++++++++++++++++---------- 2 files changed, 32 insertions(+), 17 deletions(-) create mode 100644 changelog.d/12466.misc diff --git a/changelog.d/12466.misc b/changelog.d/12466.misc new file mode 100644 index 0000000000..b0c2c950fe --- /dev/null +++ b/changelog.d/12466.misc @@ -0,0 +1 @@ +Dockerfile-workers: give the master its own log config. diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py index 3e91024e8c..8f1e61930e 100755 --- a/docker/configure_workers_and_start.py +++ b/docker/configure_workers_and_start.py @@ -29,7 +29,7 @@ import os import subprocess import sys -from typing import Any, Dict, Set +from typing import Any, Dict, Mapping, Set import jinja2 import yaml @@ -341,7 +341,7 @@ def generate_worker_files(environ, config_path: str, data_dir: str): # base shared worker jinja2 template. # # This config file will be passed to all workers, included Synapse's main process. - shared_config = {"listeners": listeners} + shared_config: Dict[str, Any] = {"listeners": listeners} # The supervisord config. The contents of which will be inserted into the # base supervisord jinja2 template. @@ -446,21 +446,7 @@ def generate_worker_files(environ, config_path: str, data_dir: str): # Write out the worker's logging config file - # Check whether we should write worker logs to disk, in addition to the console - extra_log_template_args = {} - if environ.get("SYNAPSE_WORKERS_WRITE_LOGS_TO_DISK"): - extra_log_template_args["LOG_FILE_PATH"] = "{dir}/logs/{name}.log".format( - dir=data_dir, name=worker_name - ) - - # Render and write the file - log_config_filepath = "/conf/workers/{name}.log.config".format(name=worker_name) - convert( - "/conf/log.config", - log_config_filepath, - worker_name=worker_name, - **extra_log_template_args, - ) + log_config_filepath = generate_worker_log_config(environ, worker_name, data_dir) # Then a worker config file convert( @@ -496,6 +482,10 @@ def generate_worker_files(environ, config_path: str, data_dir: str): # Finally, we'll write out the config files. + # log config for the master process + master_log_config = generate_worker_log_config(environ, "master", data_dir) + shared_config["log_config"] = master_log_config + # Shared homeserver config convert( "/conf/shared.yaml.j2", @@ -532,6 +522,30 @@ def generate_worker_files(environ, config_path: str, data_dir: str): os.mkdir(log_dir) +def generate_worker_log_config( + environ: Mapping[str, str], worker_name: str, data_dir: str +) -> str: + """Generate a log.config file for the given worker. + + Returns: the path to the generated file + """ + # Check whether we should write worker logs to disk, in addition to the console + extra_log_template_args = {} + if environ.get("SYNAPSE_WORKERS_WRITE_LOGS_TO_DISK"): + extra_log_template_args["LOG_FILE_PATH"] = "{dir}/logs/{name}.log".format( + dir=data_dir, name=worker_name + ) + # Render and write the file + log_config_filepath = "/conf/workers/{name}.log.config".format(name=worker_name) + convert( + "/conf/log.config", + log_config_filepath, + worker_name=worker_name, + **extra_log_template_args, + ) + return log_config_filepath + + def start_supervisord(): """Starts up supervisord which then starts and monitors all other necessary processes From 6b3e0ea6bdff15aa24a42f1f5938785460a323a7 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Thu, 14 Apr 2022 11:03:24 +0100 Subject: [PATCH 22/60] Use `poetry` to manage the virtualenv in debian packages (#12449) * Use `poetry` to build venv in debian packages Co-authored-by: Dan Callahan Co-authored-by: Shay * Changelog * Only pull in from requirements.txt Addresses the same problem as #12439. * Include `test` and `all` extras `poetry export` helpfully silently ignores an unknown extra Haven't seen this before because it's the only place we export `all` and `test`. I could have __sworm__ that the syntax `--extra "all test"` worked for `poetry install`... * Clean up requirements file on subsequence builds * Fix shell syntax Co-authored-by: Dan Callahan Co-authored-by: Shay --- changelog.d/12449.misc | 1 + debian/build_virtualenv | 18 +++++++++++++++--- debian/changelog | 7 +++++++ debian/clean | 1 + 4 files changed, 24 insertions(+), 3 deletions(-) create mode 100644 changelog.d/12449.misc create mode 100644 debian/clean diff --git a/changelog.d/12449.misc b/changelog.d/12449.misc new file mode 100644 index 0000000000..03e08aace4 --- /dev/null +++ b/changelog.d/12449.misc @@ -0,0 +1 @@ +Use `poetry` to manage the virtualenv in debian packages. diff --git a/debian/build_virtualenv b/debian/build_virtualenv index e691163619..b068792592 100755 --- a/debian/build_virtualenv +++ b/debian/build_virtualenv @@ -30,9 +30,19 @@ case $(dpkg-architecture -q DEB_HOST_ARCH) in ;; esac -# Use --builtin-venv to use the better `venv` module from CPython 3.4+ rather -# than the 2/3 compatible `virtualenv`. +# Manually install Poetry and export a pip-compatible `requirements.txt` +# We need a Poetry pre-release as the export command is buggy in < 1.2 +TEMP_VENV="$(mktemp -d)" +python3 -m venv "$TEMP_VENV" +source "$TEMP_VENV/bin/activate" +pip install -U pip +pip install poetry==1.2.0b1 +poetry export --extras all --extras test -o exported_requirements.txt +deactivate +rm -rf "$TEMP_VENV" +# Use --no-deps to only install pinned versions in exported_requirements.txt, +# and to avoid https://github.com/pypa/pip/issues/9644 dh_virtualenv \ --install-suffix "matrix-synapse" \ --builtin-venv \ @@ -41,9 +51,11 @@ dh_virtualenv \ --preinstall="lxml" \ --preinstall="mock" \ --preinstall="wheel" \ + --extra-pip-arg="--no-deps" \ --extra-pip-arg="--no-cache-dir" \ --extra-pip-arg="--compile" \ - --extras="all,systemd,test" + --extras="all,systemd,test" \ + --requirements="exported_requirements.txt" PACKAGE_BUILD_DIR="debian/matrix-synapse-py3" VIRTUALENV_DIR="${PACKAGE_BUILD_DIR}${DH_VIRTUALENV_INSTALL_ROOT}/matrix-synapse" diff --git a/debian/changelog b/debian/changelog index 67215aa080..fdbc92df6b 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,10 @@ +matrix-synapse-py3 (1.58.0+nmu1) UNRELEASED; urgency=medium + + * Non-maintainer upload. + * Use poetry to manage the bundled virtualenv included with this package. + + -- Synapse Packaging Team Wed, 30 Mar 2022 12:21:43 +0100 + matrix-synapse-py3 (1.57.0~rc1) stable; urgency=medium * New synapse release 1.57.0~rc1. diff --git a/debian/clean b/debian/clean new file mode 100644 index 0000000000..d488f298d5 --- /dev/null +++ b/debian/clean @@ -0,0 +1 @@ +exported_requirements.txt From 535a689cfcec896dd4ce74ef567bd30c11ade9e3 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Thu, 14 Apr 2022 11:33:06 +0100 Subject: [PATCH 23/60] Reintroduce the lint targets in the linter script (#12455) --- changelog.d/12455.misc | 1 + docs/code_style.md | 54 ++++++++++++------------------------------ scripts-dev/lint.sh | 16 +++++++++++-- 3 files changed, 30 insertions(+), 41 deletions(-) create mode 100644 changelog.d/12455.misc diff --git a/changelog.d/12455.misc b/changelog.d/12455.misc new file mode 100644 index 0000000000..9b19945673 --- /dev/null +++ b/changelog.d/12455.misc @@ -0,0 +1 @@ +Reintroduce the list of targets to the linter script, to avoid linting unwanted local-only directories during development. diff --git a/docs/code_style.md b/docs/code_style.md index ebda6dcc85..db7edcd76b 100644 --- a/docs/code_style.md +++ b/docs/code_style.md @@ -6,60 +6,36 @@ The Synapse codebase uses a number of code formatting tools in order to quickly and automatically check for formatting (and sometimes logical) errors in code. -The necessary tools are detailed below. +The necessary tools are: -First install them with: +- [black](https://black.readthedocs.io/en/stable/), a source code formatter; +- [isort](https://pycqa.github.io/isort/), which organises each file's imports; +- [flake8](https://flake8.pycqa.org/en/latest/), which can spot common errors; and +- [mypy](https://mypy.readthedocs.io/en/stable/), a type checker. + +Install them with: ```sh pip install -e ".[lint,mypy]" ``` -- **black** +The easiest way to run the lints is to invoke the linter script as follows. - The Synapse codebase uses [black](https://pypi.org/project/black/) - as an opinionated code formatter, ensuring all comitted code is - properly formatted. - - Have `black` auto-format your code (it shouldn't change any - functionality) with: - - ```sh - black . - ``` - -- **flake8** - - `flake8` is a code checking tool. We require code to pass `flake8` - before being merged into the codebase. - - Check all application and test code with: - - ```sh - flake8 . - ``` - -- **isort** - - `isort` ensures imports are nicely formatted, and can suggest and - auto-fix issues such as double-importing. - - Auto-fix imports with: - - ```sh - isort . - ``` +```sh +scripts-dev/lint.sh +``` It's worth noting that modern IDEs and text editors can run these tools automatically on save. It may be worth looking into whether this functionality is supported in your editor for a more convenient -development workflow. It is not, however, recommended to run `flake8` on -save as it takes a while and is very resource intensive. +development workflow. It is not, however, recommended to run `flake8` or `mypy` +on save as they take a while and can be very resource intensive. ## General rules - **Naming**: - - Use camel case for class and type names - - Use underscores for functions and variables. + - Use `CamelCase` for class and type names + - Use underscores for `function_names` and `variable_names`. - **Docstrings**: should follow the [google code style](https://google.github.io/styleguide/pyguide.html#38-comments-and-docstrings). See the diff --git a/scripts-dev/lint.sh b/scripts-dev/lint.sh index 4698d2d5be..91a704d982 100755 --- a/scripts-dev/lint.sh +++ b/scripts-dev/lint.sh @@ -79,8 +79,20 @@ else # If we were not asked to lint changed files, and no paths were found as a result, # then lint everything! if [[ -z ${files+x} ]]; then - # Lint all source code files and directories - files=( "." ) + # CI runs each linter on the entire checkout, e.g. `black .`. So don't + # rely on this list to *find* lint targets if that misses a file; instead; + # use it to exclude files from linters when this can't be done by config. + # + # To check which files the linters examine, use: + # black --verbose . 2>&1 | \grep -v ignored + # isort --show-files . + # flake8 --verbose . # This isn't a great option + # mypy has explicit config in mypy.ini; there is also mypy --verbose + files=( + "synapse" "docker" "tests" + "scripts-dev" + "contrib" "setup.py" "synmark" "stubs" ".ci" + ) fi fi From 0b014eb25e8c0766d2552fae2ba366492dc16d4d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 14 Apr 2022 13:05:31 +0100 Subject: [PATCH 24/60] Only send out device list updates for our own users (#12465) Broke in #12365 --- changelog.d/12465.feature | 1 + synapse/handlers/device.py | 10 +++-- synapse/storage/databases/main/devices.py | 4 +- tests/federation/test_federation_sender.py | 43 +++++++++++++++++++++- tests/storage/test_devices.py | 6 +-- 5 files changed, 56 insertions(+), 8 deletions(-) create mode 100644 changelog.d/12465.feature diff --git a/changelog.d/12465.feature b/changelog.d/12465.feature new file mode 100644 index 0000000000..642dea966c --- /dev/null +++ b/changelog.d/12465.feature @@ -0,0 +1 @@ +Enable processing of device list updates asynchronously. diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py index 958599e7b8..3c0fc756d4 100644 --- a/synapse/handlers/device.py +++ b/synapse/handlers/device.py @@ -649,9 +649,13 @@ class DeviceHandler(DeviceWorkerHandler): return for user_id, device_id, room_id, stream_id, opentracing_context in rows: - joined_user_ids = await self.store.get_users_in_room(room_id) - hosts = {get_domain_from_id(u) for u in joined_user_ids} - hosts.discard(self.server_name) + hosts = set() + + # Ignore any users that aren't ours + if self.hs.is_mine_id(user_id): + joined_user_ids = await self.store.get_users_in_room(room_id) + hosts = {get_domain_from_id(u) for u in joined_user_ids} + hosts.discard(self.server_name) # Check if we've already sent this update to some hosts if current_stream_id == stream_id: diff --git a/synapse/storage/databases/main/devices.py b/synapse/storage/databases/main/devices.py index 74e4e2122a..318e4df376 100644 --- a/synapse/storage/databases/main/devices.py +++ b/synapse/storage/databases/main/devices.py @@ -1703,7 +1703,9 @@ class DeviceStore(DeviceWorkerStore, DeviceBackgroundUpdateStore): next(stream_id_iterator), user_id, device_id, - False, + not self.hs.is_mine_id( + user_id + ), # We only need to send out update for *our* users now, encoded_context if whitelisted_homeserver(destination) else "{}", ) diff --git a/tests/federation/test_federation_sender.py b/tests/federation/test_federation_sender.py index 63ea4f9ee4..91f982518e 100644 --- a/tests/federation/test_federation_sender.py +++ b/tests/federation/test_federation_sender.py @@ -162,7 +162,9 @@ class FederationSenderDevicesTestCases(HomeserverTestCase): def make_homeserver(self, reactor, clock): return self.setup_test_homeserver( - federation_transport_client=Mock(spec=["send_transaction"]), + federation_transport_client=Mock( + spec=["send_transaction", "query_user_devices"] + ), ) def default_config(self): @@ -218,6 +220,45 @@ class FederationSenderDevicesTestCases(HomeserverTestCase): self.assertEqual(len(self.edus), 1) self.check_device_update_edu(self.edus.pop(0), u1, "D2", stream_id) + def test_dont_send_device_updates_for_remote_users(self): + """Check that we don't send device updates for remote users""" + + # Send the server a device list EDU for the other user, this will cause + # it to try and resync the device lists. + self.hs.get_federation_transport_client().query_user_devices.return_value = ( + defer.succeed( + { + "stream_id": "1", + "user_id": "@user2:host2", + "devices": [{"device_id": "D1"}], + } + ) + ) + + self.get_success( + self.hs.get_device_handler().device_list_updater.incoming_device_list_update( + "host2", + { + "user_id": "@user2:host2", + "device_id": "D1", + "stream_id": "1", + "prev_ids": [], + }, + ) + ) + + self.reactor.advance(1) + + # We shouldn't see an EDU for that update + self.assertEqual(self.edus, []) + + # Check that we did successfully process the inbound EDU (otherwise this + # test would pass if we failed to process the EDU) + devices = self.get_success( + self.hs.get_datastores().main.get_cached_devices_for_user("@user2:host2") + ) + self.assertIn("D1", devices) + def test_upload_signatures(self): """Uploading signatures on some devices should produce updates for that user""" diff --git a/tests/storage/test_devices.py b/tests/storage/test_devices.py index 5491fbf6da..ccc3893869 100644 --- a/tests/storage/test_devices.py +++ b/tests/storage/test_devices.py @@ -118,7 +118,7 @@ class DeviceStoreTestCase(HomeserverTestCase): device_ids = ["device_id1", "device_id2"] # Add two device updates with sequential `stream_id`s - self.add_device_change("user_id", device_ids, "somehost") + self.add_device_change("@user_id:test", device_ids, "somehost") # Get all device updates ever meant for this remote now_stream_id, device_updates = self.get_success( @@ -142,7 +142,7 @@ class DeviceStoreTestCase(HomeserverTestCase): "device_id4", "device_id5", ] - self.add_device_change("user_id", device_ids, "somehost") + self.add_device_change("@user_id:test", device_ids, "somehost") # Get device updates meant for this remote next_stream_id, device_updates = self.get_success( @@ -162,7 +162,7 @@ class DeviceStoreTestCase(HomeserverTestCase): # Add some more device updates to ensure it still resumes properly device_ids = ["device_id6", "device_id7"] - self.add_device_change("user_id", device_ids, "somehost") + self.add_device_change("@user_id:test", device_ids, "somehost") # Get the next batch of device updates next_stream_id, device_updates = self.get_success( From a743f7d33ebbd383fbe9bf70e024db79525d891c Mon Sep 17 00:00:00 2001 From: Dirk Klimpel <5740567+dklimpel@users.noreply.github.com> Date: Thu, 14 Apr 2022 14:09:07 +0200 Subject: [PATCH 25/60] Replace `federation_reader` with `generic_worker` in docs (#12457) --- changelog.d/12457.doc | 1 + docs/systemd-with-workers/README.md | 10 +++++----- .../workers/federation_reader.yaml | 13 ------------- .../workers/generic_worker.yaml | 13 +++++++++++++ docs/workers.md | 10 ++++------ 5 files changed, 23 insertions(+), 24 deletions(-) create mode 100644 changelog.d/12457.doc delete mode 100644 docs/systemd-with-workers/workers/federation_reader.yaml create mode 100644 docs/systemd-with-workers/workers/generic_worker.yaml diff --git a/changelog.d/12457.doc b/changelog.d/12457.doc new file mode 100644 index 0000000000..a4871622cf --- /dev/null +++ b/changelog.d/12457.doc @@ -0,0 +1 @@ +Update worker documentation and replace old `federation_reader` with `generic_worker`. \ No newline at end of file diff --git a/docs/systemd-with-workers/README.md b/docs/systemd-with-workers/README.md index b160d93528..d516501085 100644 --- a/docs/systemd-with-workers/README.md +++ b/docs/systemd-with-workers/README.md @@ -10,15 +10,15 @@ See the folder [system](https://github.com/matrix-org/synapse/tree/develop/docs/ for the systemd unit files. The folder [workers](https://github.com/matrix-org/synapse/tree/develop/docs/systemd-with-workers/workers/) -contains an example configuration for the `federation_reader` worker. +contains an example configuration for the `generic_worker` worker. ## Synapse configuration files See [the worker documentation](../workers.md) for information on how to set up the configuration files and reverse-proxy correctly. -Below is a sample `federation_reader` worker configuration file. +Below is a sample `generic_worker` worker configuration file. ```yaml -{{#include workers/federation_reader.yaml}} +{{#include workers/generic_worker.yaml}} ``` Systemd manages daemonization itself, so ensure that none of the configuration @@ -61,9 +61,9 @@ systemctl stop matrix-synapse.target # Restart the master alone systemctl start matrix-synapse.service -# Restart a specific worker (eg. federation_reader); the master is +# Restart a specific worker (eg. generic_worker); the master is # unaffected by this. -systemctl restart matrix-synapse-worker@federation_reader.service +systemctl restart matrix-synapse-worker@generic_worker.service # Add a new worker (assuming all configs are set up already) systemctl enable matrix-synapse-worker@federation_writer.service diff --git a/docs/systemd-with-workers/workers/federation_reader.yaml b/docs/systemd-with-workers/workers/federation_reader.yaml deleted file mode 100644 index 13e69e62c9..0000000000 --- a/docs/systemd-with-workers/workers/federation_reader.yaml +++ /dev/null @@ -1,13 +0,0 @@ -worker_app: synapse.app.federation_reader -worker_name: federation_reader1 - -worker_replication_host: 127.0.0.1 -worker_replication_http_port: 9093 - -worker_listeners: - - type: http - port: 8011 - resources: - - names: [federation] - -worker_log_config: /etc/matrix-synapse/federation-reader-log.yaml diff --git a/docs/systemd-with-workers/workers/generic_worker.yaml b/docs/systemd-with-workers/workers/generic_worker.yaml new file mode 100644 index 0000000000..8561e2cda5 --- /dev/null +++ b/docs/systemd-with-workers/workers/generic_worker.yaml @@ -0,0 +1,13 @@ +worker_app: synapse.app.generic_worker +worker_name: generic_worker1 + +worker_replication_host: 127.0.0.1 +worker_replication_http_port: 9093 + +worker_listeners: + - type: http + port: 8011 + resources: + - names: [client, federation] + +worker_log_config: /etc/matrix-synapse/generic-worker-log.yaml diff --git a/docs/workers.md b/docs/workers.md index caef44b614..446b7e5064 100644 --- a/docs/workers.md +++ b/docs/workers.md @@ -146,12 +146,10 @@ worker_replication_host: 127.0.0.1 worker_replication_http_port: 9093 worker_listeners: - - type: http - port: 8083 - resources: - - names: - - client - - federation + - type: http + port: 8083 + resources: + - names: [client, federation] worker_log_config: /home/matrix/synapse/config/worker1_log_config.yaml ``` From 960b4fb40927d890b2db32d4ef83a47a52baaf23 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Thu, 14 Apr 2022 14:56:10 +0100 Subject: [PATCH 26/60] complement-synapse-workers: factor out separate entry point script (#12467) ... with a bit more verbosity. --- changelog.d/12467.misc | 1 + docker/complement/SynapseWorkers.Dockerfile | 40 +++------------ .../start-complement-synapse-workers.sh | 50 +++++++++++++++++++ 3 files changed, 58 insertions(+), 33 deletions(-) create mode 100644 changelog.d/12467.misc create mode 100755 docker/complement/conf-workers/start-complement-synapse-workers.sh diff --git a/changelog.d/12467.misc b/changelog.d/12467.misc new file mode 100644 index 0000000000..fbf415f707 --- /dev/null +++ b/changelog.d/12467.misc @@ -0,0 +1 @@ +complement-synapse-workers: factor out separate entry point script. diff --git a/docker/complement/SynapseWorkers.Dockerfile b/docker/complement/SynapseWorkers.Dockerfile index 982219a91e..65df2d114d 100644 --- a/docker/complement/SynapseWorkers.Dockerfile +++ b/docker/complement/SynapseWorkers.Dockerfile @@ -13,8 +13,8 @@ RUN curl -OL "https://github.com/caddyserver/caddy/releases/download/v2.3.0/cadd tar xzf caddy_2.3.0_linux_amd64.tar.gz && rm caddy_2.3.0_linux_amd64.tar.gz && mv caddy /root # Install postgresql -RUN apt-get update -RUN apt-get install -y postgresql +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y postgresql-13 # Configure a user and create a database for Synapse RUN pg_ctlcluster 13 main start && su postgres -c "echo \ @@ -34,40 +34,14 @@ WORKDIR /data # Copy the caddy config COPY conf-workers/caddy.complement.json /root/caddy.json +# Copy the entrypoint +COPY conf-workers/start-complement-synapse-workers.sh / + # Expose caddy's listener ports EXPOSE 8008 8448 -ENTRYPOINT \ - # Replace the server name in the caddy config - sed -i "s/{{ server_name }}/${SERVER_NAME}/g" /root/caddy.json && \ - # Start postgres - pg_ctlcluster 13 main start 2>&1 && \ - # Start caddy - /root/caddy start --config /root/caddy.json 2>&1 && \ - # Set the server name of the homeserver - SYNAPSE_SERVER_NAME=${SERVER_NAME} \ - # No need to report stats here - SYNAPSE_REPORT_STATS=no \ - # Set postgres authentication details which will be placed in the homeserver config file - POSTGRES_PASSWORD=somesecret POSTGRES_USER=postgres POSTGRES_HOST=localhost \ - # Specify the workers to test with - SYNAPSE_WORKER_TYPES="\ - event_persister, \ - event_persister, \ - background_worker, \ - frontend_proxy, \ - event_creator, \ - user_dir, \ - media_repository, \ - federation_inbound, \ - federation_reader, \ - federation_sender, \ - synchrotron, \ - appservice, \ - pusher" \ - # Run the script that writes the necessary config files and starts supervisord, which in turn - # starts everything else - /configure_workers_and_start.py +ENTRYPOINT /start-complement-synapse-workers.sh +# Update the healthcheck to have a shorter check interval HEALTHCHECK --start-period=5s --interval=1s --timeout=1s \ CMD /bin/sh /healthcheck.sh diff --git a/docker/complement/conf-workers/start-complement-synapse-workers.sh b/docker/complement/conf-workers/start-complement-synapse-workers.sh new file mode 100755 index 0000000000..2c1e05bd62 --- /dev/null +++ b/docker/complement/conf-workers/start-complement-synapse-workers.sh @@ -0,0 +1,50 @@ +#!/bin/bash +# +# Default ENTRYPOINT for the docker image used for testing synapse with workers under complement + +set -e + +function log { + d=$(date +"%Y-%m-%d %H:%M:%S,%3N") + echo "$d $@" +} + +# Replace the server name in the caddy config +sed -i "s/{{ server_name }}/${SERVER_NAME}/g" /root/caddy.json + +log "starting postgres" +pg_ctlcluster 13 main start + +log "starting caddy" +/root/caddy start --config /root/caddy.json + +# Set the server name of the homeserver +export SYNAPSE_SERVER_NAME=${SERVER_NAME} + +# No need to report stats here +export SYNAPSE_REPORT_STATS=no + +# Set postgres authentication details which will be placed in the homeserver config file +export POSTGRES_PASSWORD=somesecret +export POSTGRES_USER=postgres +export POSTGRES_HOST=localhost + +# Specify the workers to test with +export SYNAPSE_WORKER_TYPES="\ + event_persister, \ + event_persister, \ + background_worker, \ + frontend_proxy, \ + event_creator, \ + user_dir, \ + media_repository, \ + federation_inbound, \ + federation_reader, \ + federation_sender, \ + synchrotron, \ + appservice, \ + pusher" + +# Run the script that writes the necessary config files and starts supervisord, which in turn +# starts everything else +exec /configure_workers_and_start.py From 7efddbebefbadce3e5f0938ea123e7e37ad2503c Mon Sep 17 00:00:00 2001 From: reivilibre Date: Thu, 14 Apr 2022 15:25:22 +0100 Subject: [PATCH 27/60] Update documentation to reflect that `run_background_tasks_on` is no longer experimental. (#12451) * Background workers aren't experimental anymore * Newsfile Signed-off-by: Olivier Wilkinson (reivilibre) * Stream writers aren't experimental either --- changelog.d/12451.doc | 1 + docs/workers.md | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) create mode 100644 changelog.d/12451.doc diff --git a/changelog.d/12451.doc b/changelog.d/12451.doc new file mode 100644 index 0000000000..c8b23c1285 --- /dev/null +++ b/changelog.d/12451.doc @@ -0,0 +1 @@ +Update documentation to reflect that both the `run_background_tasks_on` option and the options for moving stream writers off of the main process are no longer experimental. diff --git a/docs/workers.md b/docs/workers.md index 446b7e5064..858411b15e 100644 --- a/docs/workers.md +++ b/docs/workers.md @@ -341,9 +341,9 @@ effects of bursts of events from that bridge on events sent by normal users. #### Stream writers -Additionally, there is *experimental* support for moving writing of specific -streams (such as events) off of the main process to a particular worker. (This -is only supported with Redis-based replication.) +Additionally, the writing of specific streams (such as events) can be moved off +of the main process to a particular worker. +(This is only supported with Redis-based replication.) To enable this, the worker must have a HTTP replication listener configured, have a `worker_name` and be listed in the `instance_map` config. The same worker @@ -420,7 +420,7 @@ the stream writer for the `presence` stream: #### Background tasks -There is also *experimental* support for moving background tasks to a separate +There is also support for moving background tasks to a separate worker. Background tasks are run periodically or started via replication. Exactly which tasks are configured to run depends on your Synapse configuration (e.g. if stats is enabled). From aaaff98202fc1be4e3794291c4b670b8fc62cee5 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Thu, 14 Apr 2022 15:36:49 +0100 Subject: [PATCH 28/60] Dockerfile-workers: reduce the amount we install (#12464) This is an attempt to reduce the rebuild time. In short, we reduce the amount of stuff that the dockerfile installs, so as to give a faster startup. --- changelog.d/12464.misc | 1 + docker/Dockerfile-workers | 17 ++++++++++++++--- docker/conf-workers/supervisord.conf.j2 | 3 +++ docker/configure_workers_and_start.py | 19 +++++++++---------- 4 files changed, 27 insertions(+), 13 deletions(-) create mode 100644 changelog.d/12464.misc diff --git a/changelog.d/12464.misc b/changelog.d/12464.misc new file mode 100644 index 0000000000..7a8cc6ba51 --- /dev/null +++ b/changelog.d/12464.misc @@ -0,0 +1 @@ +Dockerfile-workers: reduce the amount we install in the image. diff --git a/docker/Dockerfile-workers b/docker/Dockerfile-workers index 6fb1cdbfb0..9ccb2b22a7 100644 --- a/docker/Dockerfile-workers +++ b/docker/Dockerfile-workers @@ -2,10 +2,19 @@ FROM matrixdotorg/synapse # Install deps -RUN apt-get update -RUN apt-get install -y supervisor redis nginx +RUN \ + --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + redis-server nginx-light -# Remove the default nginx sites +# Install supervisord with pip instead of apt, to avoid installing a second +# copy of python. +RUN --mount=type=cache,target=/root/.cache/pip \ + pip install supervisor~=4.2 + +# Disable the default nginx sites RUN rm /etc/nginx/sites-enabled/default # Copy Synapse worker, nginx and supervisord configuration template files @@ -19,5 +28,7 @@ EXPOSE 8080/tcp COPY ./docker/configure_workers_and_start.py /configure_workers_and_start.py ENTRYPOINT ["/configure_workers_and_start.py"] +# Replace the healthcheck with one which checks *all* the workers. The script +# is generated by configure_workers_and_start.py. HEALTHCHECK --start-period=5s --interval=15s --timeout=5s \ CMD /bin/sh /healthcheck.sh diff --git a/docker/conf-workers/supervisord.conf.j2 b/docker/conf-workers/supervisord.conf.j2 index 0de2c6143b..408ef72787 100644 --- a/docker/conf-workers/supervisord.conf.j2 +++ b/docker/conf-workers/supervisord.conf.j2 @@ -5,6 +5,9 @@ nodaemon=true user=root +[include] +files = /etc/supervisor/conf.d/*.conf + [program:nginx] command=/usr/sbin/nginx -g "daemon off;" priority=500 diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py index 8f1e61930e..23cac18e8d 100755 --- a/docker/configure_workers_and_start.py +++ b/docker/configure_workers_and_start.py @@ -502,9 +502,10 @@ def generate_worker_files(environ, config_path: str, data_dir: str): ) # Supervisord config + os.makedirs("/etc/supervisor", exist_ok=True) convert( "/conf/supervisord.conf.j2", - "/etc/supervisor/conf.d/supervisord.conf", + "/etc/supervisor/supervisord.conf", main_config_path=config_path, worker_config=supervisord_config, ) @@ -546,14 +547,6 @@ def generate_worker_log_config( return log_config_filepath -def start_supervisord(): - """Starts up supervisord which then starts and monitors all other necessary processes - - Raises: CalledProcessError if calling start.py return a non-zero exit code. - """ - subprocess.run(["/usr/bin/supervisord"], stdin=subprocess.PIPE) - - def main(args, environ): config_dir = environ.get("SYNAPSE_CONFIG_DIR", "/data") config_path = environ.get("SYNAPSE_CONFIG_PATH", config_dir + "/homeserver.yaml") @@ -581,7 +574,13 @@ def main(args, environ): # Start supervisord, which will start Synapse, all of the configured worker # processes, redis, nginx etc. according to the config we created above. - start_supervisord() + log("Starting supervisord") + os.execl( + "/usr/local/bin/supervisord", + "supervisord", + "-c", + "/etc/supervisor/supervisord.conf", + ) if __name__ == "__main__": From 3c758d9808ddc00ee96c5426dda3caea356c8745 Mon Sep 17 00:00:00 2001 From: Shay Date: Mon, 18 Apr 2022 11:32:30 -0700 Subject: [PATCH 29/60] Add a manual documenting config file options (#12368) --- changelog.d/12368.doc | 1 + docs/SUMMARY.md | 1 + .../configuration/config_documentation.md | 3412 +++++++++++++++++ 3 files changed, 3414 insertions(+) create mode 100644 changelog.d/12368.doc create mode 100644 docs/usage/configuration/config_documentation.md diff --git a/changelog.d/12368.doc b/changelog.d/12368.doc new file mode 100644 index 0000000000..62e4cb2c7e --- /dev/null +++ b/changelog.d/12368.doc @@ -0,0 +1 @@ +Add a manual documenting config file options. \ No newline at end of file diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 6aa48e1919..65570cefbe 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -17,6 +17,7 @@ # Usage - [Federation](federate.md) - [Configuration](usage/configuration/README.md) + - [Configuration Manual](usage/configuration/config_documentation.md) - [Homeserver Sample Config File](usage/configuration/homeserver_sample_config.md) - [Logging Sample Config File](usage/configuration/logging_sample_config.md) - [Structured Logging](structured_logging.md) diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md new file mode 100644 index 0000000000..9c864af6ec --- /dev/null +++ b/docs/usage/configuration/config_documentation.md @@ -0,0 +1,3412 @@ +# Configuring Synapse + +This is intended as a guide to the Synapse configuration. The behavior of a Synapse instance can be modified +through the many configuration settings documented here — each config option is explained, +including what the default is, how to change the default and what sort of behaviour the setting governs. +Also included is an example configuration for each setting. If you don't want to spend a lot of time +thinking about options, the config as generated sets sensible defaults for all values. Do note however that the +database defaults to SQLite, which is not recommended for production usage. You can read more on this subject +[here](../../setup/installation.md#using-postgresql). + +## Config Conventions + +Configuration options that take a time period can be set using a number +followed by a letter. Letters have the following meanings: + +* `s` = second +* `m` = minute +* `h` = hour +* `d` = day +* `w` = week +* `y` = year + +For example, setting `redaction_retention_period: 5m` would remove redacted +messages from the database after 5 minutes, rather than 5 months. + +### YAML +The configuration file is a [YAML](https://yaml.org/) file, which means that certain syntax rules +apply if you want your config file to be read properly. A few helpful things to know: +* `#` before any option in the config will comment out that setting and either a default (if available) will + be applied or Synapse will ignore the setting. Thus, in example #1 below, the setting will be read and + applied, but in example #2 the setting will not be read and a default will be applied. + + Example #1: + ```yaml + pid_file: DATADIR/homeserver.pid + ``` + Example #2: + ```yaml + #pid_file: DATADIR/homeserver.pid + ``` +* Indentation matters! The indentation before a setting + will determine whether a given setting is read as part of another + setting, or considered on its own. Thus, in example #1, the `enabled` setting + is read as a sub-option of the `presence` setting, and will be properly applied. + + However, the lack of indentation before the `enabled` setting in example #2 means + that when reading the config, Synapse will consider both `presence` and `enabled` as + different settings. In this case, `presence` has no value, and thus a default applied, and `enabled` + is an option that Synapse doesn't recognize and thus ignores. + + Example #1: + ```yaml + presence: + enabled: false + ``` + Example #2: + ```yaml + presence: + enabled: false + ``` + In this manual, all top-level settings (ones with no indentation) are identified + at the beginning of their section (i.e. "Config option: `example_setting`") and + the sub-options, if any, are identified and listed in the body of the section. + In addition, each setting has an example of its usage, with the proper indentation + shown. + + +## Modules + +Server admins can expand Synapse's functionality with external modules. + +See [here](../../modules/index.md) for more +documentation on how to configure or create custom modules for Synapse. + + +--- +Config option: `modules` + +Use the `module` sub-option to add modules under this option to extend functionality. +The `module` setting then has a sub-option, `config`, which can be used to define some configuration +for the `module`. + +Defaults to none. + +Example configuration: +```yaml +modules: + - module: my_super_module.MySuperClass + config: + do_thing: true + - module: my_other_super_module.SomeClass + config: {} +``` +--- +## Server ## + +Define your homeserver name and other base options. + +--- +Config option: `server_name` + +This sets the public-facing domain of the server. + +The `server_name` name will appear at the end of usernames and room addresses +created on your server. For example if the `server_name` was example.com, +usernames on your server would be in the format `@user:example.com` + +In most cases you should avoid using a matrix specific subdomain such as +matrix.example.com or synapse.example.com as the `server_name` for the same +reasons you wouldn't use user@email.example.com as your email address. +See [here](../../delegate.md) +for information on how to host Synapse on a subdomain while preserving +a clean `server_name`. + +The `server_name` cannot be changed later so it is important to +configure this correctly before you start Synapse. It should be all +lowercase and may contain an explicit port. + +There is no default for this option. + +Example configuration #1: +```yaml +server_name: matrix.org +``` +Example configuration #2: +```yaml +server_name: localhost:8080 +``` +--- +Config option: `pid_file` + +When running Synapse as a daemon, the file to store the pid in. Defaults to none. + +Example configuration: +```yaml +pid_file: DATADIR/homeserver.pid +``` +--- +Config option: `web_client_location` + +The absolute URL to the web client which `/` will redirect to. Defaults to none. + +Example configuration: +```yaml +web_client_location: https://riot.example.com/ +``` +--- +Config option: `public_baseurl` + +The public-facing base URL that clients use to access this Homeserver (not +including _matrix/...). This is the same URL a user might enter into the +'Custom Homeserver URL' field on their client. If you use Synapse with a +reverse proxy, this should be the URL to reach Synapse via the proxy. +Otherwise, it should be the URL to reach Synapse's client HTTP listener (see +'listeners' below). + +Defaults to `https:///`. + +Example configuration: +```yaml +public_baseurl: https://example.com/ +``` +--- +Config option: `serve_server_wellknown` + +By default, other servers will try to reach our server on port 8448, which can +be inconvenient in some environments. + +Provided `https:///` on port 443 is routed to Synapse, this +option configures Synapse to serve a file at `https:///.well-known/matrix/server`. +This will tell other servers to send traffic to port 443 instead. + +This option currently defaults to false. + +See https://matrix-org.github.io/synapse/latest/delegate.html for more +information. + +Example configuration: +```yaml +serve_server_wellknown: true +``` +--- +Config option: `soft_file_limit` + +Set the soft limit on the number of file descriptors synapse can use. +Zero is used to indicate synapse should set the soft limit to the hard limit. +Defaults to 0. + +Example configuration: +```yaml +soft_file_limit: 3 +``` +--- +Config option: `presence` + +Presence tracking allows users to see the state (e.g online/offline) +of other local and remote users. Set the `enabled` sub-option to false to +disable presence tracking on this homeserver. Defaults to true. +This option replaces the previous top-level 'use_presence' option. + +Example configuration: +```yaml +presence: + enabled: false +``` +--- +Config option: `require_auth_for_profile_requests` + +Whether to require authentication to retrieve profile data (avatars, display names) of other +users through the client API. Defaults to false. Note that profile data is also available +via the federation API, unless `allow_profile_lookup_over_federation` is set to false. + +Example configuration: +```yaml +require_auth_for_profile_requests: true +``` +--- +Config option: `limit_profile_requests_to_users_who_share_rooms` + +Use this option to require a user to share a room with another user in order +to retrieve their profile information. Only checked on Client-Server +requests. Profile requests from other servers should be checked by the +requesting server. Defaults to false. + +Example configuration: +```yaml +limit_profile_requests_to_users_who_share_rooms: true +``` +--- +Config option: `include_profile_data_on_invite` + +Use this option to prevent a user's profile data from being retrieved and +displayed in a room until they have joined it. By default, a user's +profile data is included in an invite event, regardless of the values +of the above two settings, and whether or not the users share a server. +Defaults to true. + +Example configuration: +```yaml +include_profile_data_on_invite: false +``` +--- +Config option: `allow_public_rooms_without_auth` + +If set to true, removes the need for authentication to access the server's +public rooms directory through the client API, meaning that anyone can +query the room directory. Defaults to false. + +Example configuration: +```yaml +allow_public_rooms_without_auth: true +``` +--- +Config option: `allow_public_rooms_without_auth` + +If set to true, allows any other homeserver to fetch the server's public +rooms directory via federation. Defaults to false. + +Example configuration: +```yaml +allow_public_rooms_over_federation: true +``` +--- +Config option: `default_room_version` + +The default room version for newly created rooms on this server. + +Known room versions are listed [here](https://spec.matrix.org/latest/rooms/#complete-list-of-room-versions) + +For example, for room version 1, `default_room_version` should be set +to "1". + +Currently defaults to "9". + +Example configuration: +```yaml +default_room_version: "8" +``` +--- +Config option: `gc_thresholds` + +The garbage collection threshold parameters to pass to `gc.set_threshold`, if defined. +Defaults to none. + +Example configuration: +```yaml +gc_thresholds: [700, 10, 10] +``` +--- +Config option: `gc_min_interval` + +The minimum time in seconds between each GC for a generation, regardless of +the GC thresholds. This ensures that we don't do GC too frequently. A value of `[1s, 10s, 30s]` +indicates that a second must pass between consecutive generation 0 GCs, etc. + +Defaults to `[1s, 10s, 30s]`. + +Example configuration: +```yaml +gc_min_interval: [0.5s, 30s, 1m] +``` +--- +Config option: `filter_timeline_limit` + +Set the limit on the returned events in the timeline in the get +and sync operations. Defaults to 100. A value of -1 means no upper limit. + + +Example configuration: +```yaml +filter_timeline_limit: 5000 +``` +--- +Config option: `block_non_admin_invites` + +Whether room invites to users on this server should be blocked +(except those sent by local server admins). Defaults to false. + +Example configuration: +```yaml +block_non_admin_invites: true +``` +--- +Config option: `enable_search` + +If set to false, new messages will not be indexed for searching and users +will receive errors when searching for messages. Defaults to true. + +Example configuration: +```yaml +enable_search: false +``` +--- +Config option: `ip_range_blacklist` + +This option prevents outgoing requests from being sent to the specified blacklisted IP address +CIDR ranges. If this option is not specified then it defaults to private IP +address ranges (see the example below). + +The blacklist applies to the outbound requests for federation, identity servers, +push servers, and for checking key validity for third-party invite events. + +(0.0.0.0 and :: are always blacklisted, whether or not they are explicitly +listed here, since they correspond to unroutable addresses.) + +This option replaces `federation_ip_range_blacklist` in Synapse v1.25.0. + +Note: The value is ignored when an HTTP proxy is in use. + +Example configuration: +```yaml +ip_range_blacklist: + - '127.0.0.0/8' + - '10.0.0.0/8' + - '172.16.0.0/12' + - '192.168.0.0/16' + - '100.64.0.0/10' + - '192.0.0.0/24' + - '169.254.0.0/16' + - '192.88.99.0/24' + - '198.18.0.0/15' + - '192.0.2.0/24' + - '198.51.100.0/24' + - '203.0.113.0/24' + - '224.0.0.0/4' + - '::1/128' + - 'fe80::/10' + - 'fc00::/7' + - '2001:db8::/32' + - 'ff00::/8' + - 'fec0::/10' +``` +--- +Config option: `ip_range_whitelist` + +List of IP address CIDR ranges that should be allowed for federation, +identity servers, push servers, and for checking key validity for +third-party invite events. This is useful for specifying exceptions to +wide-ranging blacklisted target IP ranges - e.g. for communication with +a push server only visible in your network. + +This whitelist overrides `ip_range_blacklist` and defaults to an empty +list. + +Example configuration: +```yaml +ip_range_whitelist: + - '192.168.1.1' +``` +--- +Config option: `listeners` + +List of ports that Synapse should listen on, their purpose and their +configuration. + +Sub-options for each listener include: + +* `port`: the TCP port to bind to. + +* `bind_addresses`: a list of local addresses to listen on. The default is + 'all local interfaces'. + +* `type`: the type of listener. Normally `http`, but other valid options are: + + * `manhole`: (see the docs [here](../../manhole.md)), + + * `metrics`: (see the docs [here](../../metrics-howto.md)), + + * `replication`: (see the docs [here](../../workers.md)). + +* `tls`: set to true to enable TLS for this listener. Will use the TLS key/cert specified in tls_private_key_path / tls_certificate_path. + +* `x_forwarded`: Only valid for an 'http' listener. Set to true to use the X-Forwarded-For header as the client IP. Useful when Synapse is + behind a reverse-proxy. + +* `resources`: Only valid for an 'http' listener. A list of resources to host + on this port. Sub-options for each resource are: + + * `names`: a list of names of HTTP resources. See below for a list of valid resource names. + + * `compress`: set to true to enable HTTP compression for this resource. + +* `additional_resources`: Only valid for an 'http' listener. A map of + additional endpoints which should be loaded via dynamic modules. + +Valid resource names are: + +* `client`: the client-server API (/_matrix/client), and the synapse admin API (/_synapse/admin). Also implies 'media' and 'static'. + +* `consent`: user consent forms (/_matrix/consent). See [here](../../consent_tracking.md) for more. + +* `federation`: the server-server API (/_matrix/federation). Also implies `media`, `keys`, `openid` + +* `keys`: the key discovery API (/_matrix/keys). + +* `media`: the media API (/_matrix/media). + +* `metrics`: the metrics interface. See [here](../../metrics-howto.md). + +* `openid`: OpenID authentication. See [here](../../openid.md). + +* `replication`: the HTTP replication API (/_synapse/replication). See [here](../../workers.md). + +* `static`: static resources under synapse/static (/_matrix/static). (Mostly useful for 'fallback authentication'.) + +Example configuration #1: +```yaml +listeners: + # TLS-enabled listener: for when matrix traffic is sent directly to synapse. + # + # (Note that you will also need to give Synapse a TLS key and certificate: see the TLS section + # below.) + # + - port: 8448 + type: http + tls: true + resources: + - names: [client, federation] +``` +Example configuration #2: +```yaml +listeners: + # Unsecure HTTP listener: for when matrix traffic passes through a reverse proxy + # that unwraps TLS. + # + # If you plan to use a reverse proxy, please see + # https://matrix-org.github.io/synapse/latest/reverse_proxy.html. + # + - port: 8008 + tls: false + type: http + x_forwarded: true + bind_addresses: ['::1', '127.0.0.1'] + + resources: + - names: [client, federation] + compress: false + + # example additional_resources: + additional_resources: + "/_matrix/my/custom/endpoint": + module: my_module.CustomRequestHandler + config: {} + + # Turn on the twisted ssh manhole service on localhost on the given + # port. + - port: 9000 + bind_addresses: ['::1', '127.0.0.1'] + type: manhole +``` +--- +Config option: `manhole_settings` + +Connection settings for the manhole. You can find more information +on the manhole [here](../../manhole.md). Manhole sub-options include: +* `username` : the username for the manhole. This defaults to 'matrix'. +* `password`: The password for the manhole. This defaults to 'rabbithole'. +* `ssh_priv_key_path` and `ssh_pub_key_path`: The private and public SSH key pair used to encrypt the manhole traffic. + If these are left unset, then hardcoded and non-secret keys are used, + which could allow traffic to be intercepted if sent over a public network. + +Example configuration: +```yaml +manhole_settings: + username: manhole + password: mypassword + ssh_priv_key_path: CONFDIR/id_rsa + ssh_pub_key_path: CONFDIR/id_rsa.pub +``` +--- +Config option: `dummy_events_threshold` + +Forward extremities can build up in a room due to networking delays between +homeservers. Once this happens in a large room, calculation of the state of +that room can become quite expensive. To mitigate this, once the number of +forward extremities reaches a given threshold, Synapse will send an +`org.matrix.dummy_event` event, which will reduce the forward extremities +in the room. + +This setting defines the threshold (i.e. number of forward extremities in the room) at which dummy events are sent. +The default value is 10. + +Example configuration: +```yaml +dummy_events_threshold: 5 +``` +--- +## Homeserver blocking ## +Useful options for Synapse admins. + +--- + +Config option: `admin_contact` + +How to reach the server admin, used in `ResourceLimitError`. Defaults to none. + +Example configuration: +```yaml +admin_contact: 'mailto:admin@server.com' +``` +--- +Config option: `hs_disabled` and `hs_disabled_message` + +Blocks users from connecting to the homeserver and provides a human-readable reason +why the connection was blocked. Defaults to false. + +Example configuration: +```yaml +hs_disabled: true +hs_disabled_message: 'Reason for why the HS is blocked' +``` +--- +Config option: `limit_usage_by_mau` + +This option disables/enables monthly active user blocking. Used in cases where the admin or +server owner wants to limit to the number of monthly active users. When enabled and a limit is +reached the server returns a `ResourceLimitError` with error type `Codes.RESOURCE_LIMIT_EXCEEDED`. +Defaults to false. If this is enabled, a value for `max_mau_value` must also be set. + +Example configuration: +```yaml +limit_usage_by_mau: true +``` +--- +Config option: `max_mau_value` + +This option sets the hard limit of monthly active users above which the server will start +blocking user actions if `limit_usage_by_mau` is enabled. Defaults to 0. + +Example configuration: +```yaml +max_mau_value: 50 +``` +--- +Config option: `mau_trial_days` + +The option `mau_trial_days` is a means to add a grace period for active users. It +means that users must be active for the specified number of days before they +can be considered active and guards against the case where lots of users +sign up in a short space of time never to return after their initial +session. Defaults to 0. + +Example configuration: +```yaml +mau_trial_days: 5 +``` +--- +Config option: `mau_limit_alerting` + +The option `mau_limit_alerting` is a means of limiting client-side alerting +should the mau limit be reached. This is useful for small instances +where the admin has 5 mau seats (say) for 5 specific people and no +interest increasing the mau limit further. Defaults to true, which +means that alerting is enabled. + +Example configuration: +```yaml +mau_limit_alerting: false +``` +--- +Config option: `mau_stats_only` + +If enabled, the metrics for the number of monthly active users will +be populated, however no one will be limited based on these numbers. If `limit_usage_by_mau` +is true, this is implied to be true. Defaults to false. + +Example configuration: +```yaml +mau_stats_only: true +``` +--- +Config option: `mau_limit_reserved_threepids` + +Sometimes the server admin will want to ensure certain accounts are +never blocked by mau checking. These accounts are specified by this option. +Defaults to none. Add accounts by specifying the `medium` and `address` of the +reserved threepid (3rd party identifier). + +Example configuration: +```yaml +mau_limit_reserved_threepids: + - medium: 'email' + address: 'reserved_user@example.com' +``` +--- +Config option: `server_context` + +This option is used by phonehome stats to group together related servers. +Defaults to none. + +Example configuration: +```yaml +server_context: context +``` +--- +Config option: `limit_remote_rooms` + +When this option is enabled, the room "complexity" will be checked before a user +joins a new remote room. If it is above the complexity limit, the server will +disallow joining, or will instantly leave. This is useful for homeservers that are +resource-constrained. Options for this setting include: +* `enabled`: whether this check is enabled. Defaults to false. +* `complexity`: the limit above which rooms cannot be joined. The default is 1.0. +* `complexity_error`: override the error which is returned when the room is too complex with a + custom message. +* `admins_can_join`: allow server admins to join complex rooms. Default is false. + +Room complexity is an arbitrary measure based on factors such as the number of +users in the room. + +Example configuration: +```yaml +limit_remote_rooms: + enabled: true + complexity: 0.5 + complexity_error: "I can't let you do that, Dave." + admins_can_join: true +``` +--- +Config option: `require_membership_for_aliases` + +Whether to require a user to be in the room to add an alias to it. +Defaults to true. + +Example configuration: +```yaml +require_membership_for_aliases: false +``` +--- +Config option: `allow_per_room_profiles` + +Whether to allow per-room membership profiles through the sending of membership +events with profile information that differs from the target's global profile. +Defaults to true. + +Example configuration: +```yaml +allow_per_room_profiles: false +``` +--- +Config option: `max_avatar_size` + +The largest permissible file size in bytes for a user avatar. Defaults to no restriction. +Use M for MB and K for KB. + +Note that user avatar changes will not work if this is set without using Synapse's media repository. + +Example configuration: +```yaml +max_avatar_size: 10M +``` +--- +Config option: `allowed_avatar_mimetypes` + +The MIME types allowed for user avatars. Defaults to no restriction. + +Note that user avatar changes will not work if this is set without +using Synapse's media repository. + +Example configuration: +```yaml +allowed_avatar_mimetypes: ["image/png", "image/jpeg", "image/gif"] +``` +--- +Config option: `redaction_retention_period` + +How long to keep redacted events in unredacted form in the database. After +this period redacted events get replaced with their redacted form in the DB. + +Defaults to `7d`. Set to `null` to disable. + +Example configuration: +```yaml +redaction_retention_period: 28d +``` +--- +Config option: `user_ips_max_age` + +How long to track users' last seen time and IPs in the database. + +Defaults to `28d`. Set to `null` to disable clearing out of old rows. + +Example configuration: +```yaml +user_ips_max_age: 14d +``` +--- +Config option: `request_token_inhibit_3pid_errors` + +Inhibits the `/requestToken` endpoints from returning an error that might leak +information about whether an e-mail address is in use or not on this +homeserver. Defaults to false. +Note that for some endpoints the error situation is the e-mail already being +used, and for others the error is entering the e-mail being unused. +If this option is enabled, instead of returning an error, these endpoints will +act as if no error happened and return a fake session ID ('sid') to clients. + +Example configuration: +```yaml +request_token_inhibit_3pid_errors: true +``` +--- +Config option: `next_link_domain_whitelist` + +A list of domains that the domain portion of `next_link` parameters +must match. + +This parameter is optionally provided by clients while requesting +validation of an email or phone number, and maps to a link that +users will be automatically redirected to after validation +succeeds. Clients can make use this parameter to aid the validation +process. + +The whitelist is applied whether the homeserver or an identity server is handling validation. + +The default value is no whitelist functionality; all domains are +allowed. Setting this value to an empty list will instead disallow +all domains. + +Example configuration: +```yaml +next_link_domain_whitelist: ["matrix.org"] +``` +--- +Config option: `templates` and `custom_template_directory` + +These options define templates to use when generating email or HTML page contents. +The `custom_template_directory` determines which directory Synapse will try to +find template files in to use to generate email or HTML page contents. +If not set, or a file is not found within the template directory, a default +template from within the Synapse package will be used. + +See [here](../../templates.md) for more +information about using custom templates. + +Example configuration: +```yaml +templates: + custom_template_directory: /path/to/custom/templates/ +``` +--- +Config option: `retention` + +This option and the associated options determine message retention policy at the +server level. + +Room admins and mods can define a retention period for their rooms using the +`m.room.retention` state event, and server admins can cap this period by setting +the `allowed_lifetime_min` and `allowed_lifetime_max` config options. + +If this feature is enabled, Synapse will regularly look for and purge events +which are older than the room's maximum retention period. Synapse will also +filter events received over federation so that events that should have been +purged are ignored and not stored again. + +The message retention policies feature is disabled by default. + +This setting has the following sub-options: +* `default_policy`: Default retention policy. If set, Synapse will apply it to rooms that lack the + 'm.room.retention' state event. This option is further specified by the + `min_lifetime` and `max_lifetime` sub-options associated with it. Note that the + value of `min_lifetime` doesn't matter much because Synapse doesn't take it into account yet. + +* `allowed_lifetime_min` and `allowed_lifetime_max`: Retention policy limits. If + set, and the state of a room contains a `m.room.retention` event in its state + which contains a `min_lifetime` or a `max_lifetime` that's out of these bounds, + Synapse will cap the room's policy to these limits when running purge jobs. + +* `purge_jobs` and the associated `shortest_max_lifetime` and `longest_max_lifetime` sub-options: + Server admins can define the settings of the background jobs purging the + events whose lifetime has expired under the `purge_jobs` section. + + If no configuration is provided for this option, a single job will be set up to delete + expired events in every room daily. + + Each job's configuration defines which range of message lifetimes the job + takes care of. For example, if `shortest_max_lifetime` is '2d' and + `longest_max_lifetime` is '3d', the job will handle purging expired events in + rooms whose state defines a `max_lifetime` that's both higher than 2 days, and + lower than or equal to 3 days. Both the minimum and the maximum value of a + range are optional, e.g. a job with no `shortest_max_lifetime` and a + `longest_max_lifetime` of '3d' will handle every room with a retention policy + whose `max_lifetime` is lower than or equal to three days. + + The rationale for this per-job configuration is that some rooms might have a + retention policy with a low `max_lifetime`, where history needs to be purged + of outdated messages on a more frequent basis than for the rest of the rooms + (e.g. every 12h), but not want that purge to be performed by a job that's + iterating over every room it knows, which could be heavy on the server. + + If any purge job is configured, it is strongly recommended to have at least + a single job with neither `shortest_max_lifetime` nor `longest_max_lifetime` + set, or one job without `shortest_max_lifetime` and one job without + `longest_max_lifetime` set. Otherwise some rooms might be ignored, even if + `allowed_lifetime_min` and `allowed_lifetime_max` are set, because capping a + room's policy to these values is done after the policies are retrieved from + Synapse's database (which is done using the range specified in a purge job's + configuration). + +Example configuration: +```yaml +retention: + enabled: true + default_policy: + min_lifetime: 1d + max_lifetime: 1y + allowed_lifetime_min: 1d + allowed_lifetime_max: 1y + purge_jobs: + - longest_max_lifetime: 3d + interval: 12h + - shortest_max_lifetime: 3d + interval: 1d +``` +--- +## TLS ## + +Options related to TLS. + +--- +Config option: `tls_certificate_path` + +This option specifies a PEM-encoded X509 certificate for TLS. +This certificate, as of Synapse 1.0, will need to be a valid and verifiable +certificate, signed by a recognised Certificate Authority. Defaults to none. + +Be sure to use a `.pem` file that includes the full certificate chain including +any intermediate certificates (for instance, if using certbot, use +`fullchain.pem` as your certificate, not `cert.pem`). + +Example configuration: +```yaml +tls_certificate_path: "CONFDIR/SERVERNAME.tls.crt" +``` +--- +Config option: `tls_private_key_path` + +PEM-encoded private key for TLS. Defaults to none. + +Example configuration: +```yaml +tls_private_key_path: "CONFDIR/SERVERNAME.tls.key" +``` +--- +Config option: `federation_verify_certificates` +Whether to verify TLS server certificates for outbound federation requests. + +Defaults to true. To disable certificate verification, set the option to false. + +Example configuration: +```yaml +federation_verify_certificates: false +``` +--- +Config option: `federation_client_minimum_tls_version` + +The minimum TLS version that will be used for outbound federation requests. + +Defaults to `1`. Configurable to `1`, `1.1`, `1.2`, or `1.3`. Note +that setting this value higher than `1.2` will prevent federation to most +of the public Matrix network: only configure it to `1.3` if you have an +entirely private federation setup and you can ensure TLS 1.3 support. + +Example configuration: +```yaml +federation_client_minimum_tls_version: 1.2 +``` +--- +Config option: `federation_certificate_verification_whitelist` + +Skip federation certificate verification on a given whitelist +of domains. + +This setting should only be used in very specific cases, such as +federation over Tor hidden services and similar. For private networks +of homeservers, you likely want to use a private CA instead. + +Only effective if `federation_verify_certicates` is `true`. + +Example configuration: +```yaml +federation_certificate_verification_whitelist: + - lon.example.com + - "*.domain.com" + - "*.onion" +``` +--- +Config option: `federation_custom_ca_list` + +List of custom certificate authorities for federation traffic. + +This setting should only normally be used within a private network of +homeservers. + +Note that this list will replace those that are provided by your +operating environment. Certificates must be in PEM format. + +Example configuration: +```yaml +federation_custom_ca_list: + - myCA1.pem + - myCA2.pem + - myCA3.pem +``` +--- +## Federation ## + +Options related to federation. + +--- +Config option: `federation_domain_whitelist` + +Restrict federation to the given whitelist of domains. +N.B. we recommend also firewalling your federation listener to limit +inbound federation traffic as early as possible, rather than relying +purely on this application-layer restriction. If not specified, the +default is to whitelist everything. + +Example configuration: +```yaml +federation_domain_whitelist: + - lon.example.com + - nyc.example.com + - syd.example.com +``` +--- +Config option: `federation_metrics_domains` + +Report prometheus metrics on the age of PDUs being sent to and received from +the given domains. This can be used to give an idea of "delay" on inbound +and outbound federation, though be aware that any delay can be due to problems +at either end or with the intermediate network. + +By default, no domains are monitored in this way. + +Example configuration: +```yaml +federation_metrics_domains: + - matrix.org + - example.com +``` +--- +Config option: `allow_profile_lookup_over_federation` + +Set to false to disable profile lookup over federation. By default, the +Federation API allows other homeservers to obtain profile data of any user +on this homeserver. + +Example configuration: +```yaml +allow_profile_lookup_over_federation: false +``` +--- +Config option: `allow_device_name_lookup_over_federation` + +Set this option to false to disable device display name lookup over federation. By default, the +Federation API allows other homeservers to obtain device display names of any user +on this homeserver. + +Example configuration: +```yaml +allow_device_name_lookup_over_federation: false +``` +--- +## Caching ## + +Options related to caching + +--- +Config option: `event_cache_size` + +The number of events to cache in memory. Not affected by +`caches.global_factor`. Defaults to 10K. + +Example configuration: +```yaml +event_cache_size: 15K +``` +--- +Config option: `cache` and associated values + +A cache 'factor' is a multiplier that can be applied to each of +Synapse's caches in order to increase or decrease the maximum +number of entries that can be stored. + +Caching can be configured through the following sub-options: + +* `global_factor`: Controls the global cache factor, which is the default cache factor + for all caches if a specific factor for that cache is not otherwise + set. + + This can also be set by the `SYNAPSE_CACHE_FACTOR` environment + variable. Setting by environment variable takes priority over + setting through the config file. + + Defaults to 0.5, which will halve the size of all caches. + +* `per_cache_factors`: A dictionary of cache name to cache factor for that individual + cache. Overrides the global cache factor for a given cache. + + These can also be set through environment variables comprised + of `SYNAPSE_CACHE_FACTOR_` + the name of the cache in capital + letters and underscores. Setting by environment variable + takes priority over setting through the config file. + Ex. `SYNAPSE_CACHE_FACTOR_GET_USERS_WHO_SHARE_ROOM_WITH_USER=2.0` + + Some caches have '*' and other characters that are not + alphanumeric or underscores. These caches can be named with or + without the special characters stripped. For example, to specify + the cache factor for `*stateGroupCache*` via an environment + variable would be `SYNAPSE_CACHE_FACTOR_STATEGROUPCACHE=2.0`. + +* `expire_caches`: Controls whether cache entries are evicted after a specified time + period. Defaults to true. Set to false to disable this feature. Note that never expiring + caches may result in excessive memory usage. + +* `cache_entry_ttl`: If `expire_caches` is enabled, this flag controls how long an entry can + be in a cache without having been accessed before being evicted. + Defaults to 30m. + +* `sync_response_cache_duration`: Controls how long the results of a /sync request are + cached for after a successful response is returned. A higher duration can help clients + with intermittent connections, at the cost of higher memory usage. + By default, this is zero, which means that sync responses are not cached + at all. + + +Example configuration: +```yaml +caches: + global_factor: 1.0 + per_cache_factors: + get_users_who_share_room_with_user: 2.0 + expire_caches: false + sync_response_cache_duration: 2m +``` +--- +## Database ## +Config options related to database settings. + +--- +Config option: `database` + +The `database` setting defines the database that synapse uses to store all of +its data. + +Associated sub-options: + +* `name`: this option specifies the database engine to use: either `sqlite3` (for SQLite) + or `psycopg2` (for PostgreSQL). If no name is specified Synapse will default to SQLite. + +* `txn_limit` gives the maximum number of transactions to run per connection + before reconnecting. Defaults to 0, which means no limit. + +* `allow_unsafe_locale` is an option specific to Postgres. Under the default behavior, Synapse will refuse to + start if the postgres db is set to a non-C locale. You can override this behavior (which is *not* recommended) + by setting `allow_unsafe_locale` to true. Note that doing so may corrupt your database. You can find more information + [here](../../postgres.md#fixing-incorrect-collate-or-ctype) and [here](https://wiki.postgresql.org/wiki/Locale_data_changes). + +* `args` gives options which are passed through to the database engine, + except for options starting with `cp_`, which are used to configure the Twisted + connection pool. For a reference to valid arguments, see: + * for [sqlite](https://docs.python.org/3/library/sqlite3.html#sqlite3.connect) + * for [postgres](https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-PARAMKEYWORDS) + * for [the connection pool](https://twistedmatrix.com/documents/current/api/twisted.enterprise.adbapi.ConnectionPool.html#__init__) + +For more information on using Synapse with Postgres, +see [here](../../postgres.md). + +Example SQLite configuration: +``` +database: + name: sqlite3 + args: + database: /path/to/homeserver.db +``` + +Example Postgres configuration: +``` +database: + name: psycopg2 + txn_limit: 10000 + args: + user: synapse_user + password: secretpassword + database: synapse + host: localhost + port: 5432 + cp_min: 5 + cp_max: 10 +``` +--- +## Logging ## +Config options related to logging. + +--- +Config option: `log_config` + +This option specifies a yaml python logging config file as described [here](https://docs.python.org/3.7/library/logging.config.html#configuration-dictionary-schema). + +Example configuration: +```yaml +log_config: "CONFDIR/SERVERNAME.log.config" +``` +--- +## Ratelimiting ## +Options related to ratelimiting in Synapse. + +Each ratelimiting configuration is made of two parameters: + - `per_second`: number of requests a client can send per second. + - `burst_count`: number of requests a client can send before being throttled. +--- +Config option: `rc_message` + + +Ratelimiting settings for client messaging. + +This is a ratelimiting option for messages that ratelimits sending based on the account the client +is using. It defaults to: `per_second: 0.2`, `burst_count: 10`. + +Example configuration: +```yaml +rc_message: + per_second: 0.5 + burst_count: 15 +``` +--- +Config option: `rc_registration` + +This option ratelimits registration requests based on the client's IP address. +It defaults to `per_second: 0.17`, `burst_count: 3`. + +Example configuration: +```yaml +rc_registration: + per_second: 0.15 + burst_count: 2 +``` +--- +Config option: `rc_registration_token_validity` + +This option checks the validity of registration tokens that ratelimits requests based on +the client's IP address. +Defaults to `per_second: 0.1`, `burst_count: 5`. + +Example configuration: +```yaml +rc_registration_token_validity: + per_second: 0.3 + burst_count: 6 +``` +--- +Config option: `rc_login` + +This option specifies several limits for login: +* `address` ratelimits login requests based on the client's IP + address. Defaults to `per_second: 0.17`, `burst_count: 3`. + +* `account` ratelimits login requests based on the account the + client is attempting to log into. Defaults to `per_second: 0.17`, + `burst_count: 3`. + +* `failted_attempts` ratelimits login requests based on the account the + client is attempting to log into, based on the amount of failed login + attempts for this account. Defaults to `per_second: 0.17`, `burst_count: 3`. + +Example configuration: +```yaml +rc_login: + address: + per_second: 0.15 + burst_count: 5 + account: + per_second: 0.18 + burst_count: 4 + failed_attempts: + per_second: 0.19 + burst_count: 7 +``` +--- +Config option: `rc_admin_redaction` + +This option sets ratelimiting redactions by room admins. If this is not explicitly +set then it uses the same ratelimiting as per `rc_message`. This is useful +to allow room admins to deal with abuse quickly. + +Example configuration: +```yaml +rc_admin_redaction: + per_second: 1 + burst_count: 50 +``` +--- +Config option: `rc_joins` + +This option allows for ratelimiting number of rooms a user can join. This setting has the following sub-options: + +* `local`: ratelimits when users are joining rooms the server is already in. + Defaults to `per_second: 0.1`, `burst_count: 10`. + +* `remote`: ratelimits when users are trying to join rooms not on the server (which + can be more computationally expensive than restricting locally). Defaults to + `per_second: 0.01`, `burst_count: 10` + +Example configuration: +```yaml +rc_joins: + local: + per_second: 0.2 + burst_count: 15 + remote: + per_second: 0.03 + burst_count: 12 +``` +--- +Config option: `rc_3pid_validation` + +This option ratelimits how often a user or IP can attempt to validate a 3PID. +Defaults to `per_second: 0.003`, `burst_count: 5`. + +Example configuration: +```yaml +rc_3pid_validation: + per_second: 0.003 + burst_count: 5 +``` +--- +Config option: `rc_invites` + +This option sets ratelimiting how often invites can be sent in a room or to a +specific user. `per_room` defaults to `per_second: 0.3`, `burst_count: 10` and +`per_user` defaults to `per_second: 0.003`, `burst_count: 5`. + +Example configuration: +```yaml +rc_invites: + per_room: + per_second: 0.5 + burst_count: 5 + per_user: + per_second: 0.004 + burst_count: 3 +``` +--- +Config option: `rc_third_party_invite` + +This option ratelimits 3PID invites (i.e. invites sent to a third-party ID +such as an email address or a phone number) based on the account that's +sending the invite. Defaults to `per_second: 0.2`, `burst_count: 10`. + +Example configuration: +```yaml +rc_third_party_invite: + per_second: 0.2 + burst_count: 10 +``` +--- +Config option: `rc_federation` + +Defines limits on federation requests. + +The `rc_federation` configuration has the following sub-options: +* `window_size`: window size in milliseconds. Defaults to 1000. +* `sleep_limit`: number of federation requests from a single server in + a window before the server will delay processing the request. Defaults to 10. +* `sleep_delay`: duration in milliseconds to delay processing events + from remote servers by if they go over the sleep limit. Defaults to 500. +* `reject_limit`: maximum number of concurrent federation requests + allowed from a single server. Defaults to 50. +* `concurrent`: number of federation requests to concurrently process + from a single server. Defaults to 3. + +Example configuration: +```yaml +rc_federation: + window_size: 750 + sleep_limit: 15 + sleep_delay: 400 + reject_limit: 40 + concurrent: 5 +``` +--- +Config option: `federation_rr_transactions_per_room_per_second` + +Sets outgoing federation transaction frequency for sending read-receipts, +per-room. + +If we end up trying to send out more read-receipts, they will get buffered up +into fewer transactions. Defaults to 50. + +Example configuration: +```yaml +federation_rr_transactions_per_room_per_second: 40 +``` +--- +## Media Store ## +Config options relating to Synapse media store. + +--- +Config option: `enable_media_repo` + +Enable the media store service in the Synapse master. Defaults to true. +Set to false if you are using a separate media store worker. + +Example configuration: +```yaml +enable_media_repo: false +``` +--- +Config option: `media_store_path` + +Directory where uploaded images and attachments are stored. + +Example configuration: +```yaml +media_store_path: "DATADIR/media_store" +``` +--- +Config option: `media_storage_providers` + +Media storage providers allow media to be stored in different +locations. Defaults to none. Associated sub-options are: +* `module`: type of resource, e.g. `file_system`. +* `store_local`: whether to store newly uploaded local files +* `store_remote`: whether to store newly downloaded local files +* `store_synchronous`: whether to wait for successful storage for local uploads +* `config`: sets a path to the resource through the `directory` option + +Example configuration: +```yaml +media_storage_providers: + - module: file_system + store_local: false + store_remote: false + store_synchronous: false + config: + directory: /mnt/some/other/directory +``` +--- +Config option: `max_upload_size` + +The largest allowed upload size in bytes. + +If you are using a reverse proxy you may also need to set this value in +your reverse proxy's config. Defaults to 50M. Notably Nginx has a small max body size by default. +See [here](../../reverse_proxy.md) for more on using a reverse proxy with Synapse. + +Example configuration: +```yaml +max_upload_size: 60M +``` +--- +Config option: `max_image_pixels` + +Maximum number of pixels that will be thumbnailed. Defaults to 32M. + +Example configuration: +```yaml +max_image_pixels: 35M +``` +--- +Config option: `dynamic_thumbnails` + +Whether to generate new thumbnails on the fly to precisely match +the resolution requested by the client. If true then whenever +a new resolution is requested by the client the server will +generate a new thumbnail. If false the server will pick a thumbnail +from a precalculated list. Defaults to false. + +Example configuration: +```yaml +dynamic_thumbnails: true +``` +--- +Config option: `thumbnail_sizes` + +List of thumbnails to precalculate when an image is uploaded. Associated sub-options are: +* `width` +* `height` +* `method`: i.e. `crop`, `scale`, etc. + +Example configuration: +```yaml +thumbnail_sizes: + - width: 32 + height: 32 + method: crop + - width: 96 + height: 96 + method: crop + - width: 320 + height: 240 + method: scale + - width: 640 + height: 480 + method: scale + - width: 800 + height: 600 + method: scale +``` +Config option: `url_preview_enabled` + +This setting determines whether the preview URL API is enabled. +It is disabled by default. Set to true to enable. If enabled you must specify a +`url_preview_ip_range_blacklist` blacklist. + +Example configuration: +```yaml +url_preview_enabled: true +``` +--- +Config option: `url_preview_ip_range_blacklist` + +List of IP address CIDR ranges that the URL preview spider is denied +from accessing. There are no defaults: you must explicitly +specify a list for URL previewing to work. You should specify any +internal services in your network that you do not want synapse to try +to connect to, otherwise anyone in any Matrix room could cause your +synapse to issue arbitrary GET requests to your internal services, +causing serious security issues. + +(0.0.0.0 and :: are always blacklisted, whether or not they are explicitly +listed here, since they correspond to unroutable addresses.) + +This must be specified if `url_preview_enabled` is set. It is recommended that +you use the following example list as a starting point. + +Note: The value is ignored when an HTTP proxy is in use. + +Example configuration: +```yaml +url_preview_ip_range_blacklist: + - '127.0.0.0/8' + - '10.0.0.0/8' + - '172.16.0.0/12' + - '192.168.0.0/16' + - '100.64.0.0/10' + - '192.0.0.0/24' + - '169.254.0.0/16' + - '192.88.99.0/24' + - '198.18.0.0/15' + - '192.0.2.0/24' + - '198.51.100.0/24' + - '203.0.113.0/24' + - '224.0.0.0/4' + - '::1/128' + - 'fe80::/10' + - 'fc00::/7' + - '2001:db8::/32' + - 'ff00::/8' + - 'fec0::/10' +``` +---- +Config option: `url_preview_ip_range_whitelist` + +This option sets a list of IP address CIDR ranges that the URL preview spider is allowed +to access even if they are specified in `url_preview_ip_range_blacklist`. +This is useful for specifying exceptions to wide-ranging blacklisted +target IP ranges - e.g. for enabling URL previews for a specific private +website only visible in your network. Defaults to none. + +Example configuration: +```yaml +url_preview_ip_range_whitelist: + - '192.168.1.1' +``` +--- +Config option: `url_preview_url_blacklist` + +Optional list of URL matches that the URL preview spider is +denied from accessing. You should use `url_preview_ip_range_blacklist` +in preference to this, otherwise someone could define a public DNS +entry that points to a private IP address and circumvent the blacklist. +This is more useful if you know there is an entire shape of URL that +you know that will never want synapse to try to spider. + +Each list entry is a dictionary of url component attributes as returned +by urlparse.urlsplit as applied to the absolute form of the URL. See +[here](https://docs.python.org/2/library/urlparse.html#urlparse.urlsplit) for more +information. Some examples are: + +* `username` +* `netloc` +* `scheme` +* `path` + +The values of the dictionary are treated as a filename match pattern +applied to that component of URLs, unless they start with a ^ in which +case they are treated as a regular expression match. If all the +specified component matches for a given list item succeed, the URL is +blacklisted. + +Example configuration: +```yaml +url_preview_url_blacklist: + # blacklist any URL with a username in its URI + - username: '*' + + # blacklist all *.google.com URLs + - netloc: 'google.com' + - netloc: '*.google.com' + + # blacklist all plain HTTP URLs + - scheme: 'http' + + # blacklist http(s)://www.acme.com/foo + - netloc: 'www.acme.com' + path: '/foo' + + # blacklist any URL with a literal IPv4 address + - netloc: '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$' +``` +--- +Config option: `max_spider_size` + +The largest allowed URL preview spidering size in bytes. Defaults to 10M. + +Example configuration: +```yaml +max_spider_size: 8M +``` +--- +Config option: `url_preview_language` + +A list of values for the Accept-Language HTTP header used when +downloading webpages during URL preview generation. This allows +Synapse to specify the preferred languages that URL previews should +be in when communicating with remote servers. + +Each value is a IETF language tag; a 2-3 letter identifier for a +language, optionally followed by subtags separated by '-', specifying +a country or region variant. + +Multiple values can be provided, and a weight can be added to each by +using quality value syntax (;q=). '*' translates to any language. + +Defaults to "en". + +Example configuration: +```yaml + url_preview_accept_language: + - en-UK + - en-US;q=0.9 + - fr;q=0.8 + - *;q=0.7 +``` +---- +Config option: `oembed` + +oEmbed allows for easier embedding content from a website. It can be +used for generating URLs previews of services which support it. A default list of oEmbed providers +is included with Synapse. Set `disable_default_providers` to true to disable using +these default oEmbed URLs. Use `additional_providers` to specify additional files with oEmbed configuration (each +should be in the form of providers.json). By default this list is empty. + +Example configuration: +```yaml +oembed: + disable_default_providers: true + additional_providers: + - oembed/my_providers.json +``` +--- +## Captcha ## + +See [here](../../CAPTCHA_SETUP.md) for full details on setting up captcha. + +--- +Config option: `recaptcha_public_key` + +This homeserver's ReCAPTCHA public key. Must be specified if `enable_registration_captcha` is +enabled. + +Example configuration: +```yaml +recaptcha_public_key: "YOUR_PUBLIC_KEY" +``` +--- +Config option: `recaptcha_private_key` + +This homeserver's ReCAPTCHA private key. Must be specified if `enable_registration_captcha` is +enabled. + +Example configuration: +```yaml +recaptcha_private_key: "YOUR_PRIVATE_KEY" +``` +--- +Config option: `enable_registration_captcha` + +Set to true to enable ReCaptcha checks when registering, preventing signup +unless a captcha is answered. Requires a valid ReCaptcha public/private key. +Defaults to false. + +Example configuration: +```yaml +enable_registration_captcha: true +``` +--- +Config option: `recaptcha_siteverify_api` + +The API endpoint to use for verifying `m.login.recaptcha` responses. +Defaults to `https://www.recaptcha.net/recaptcha/api/siteverify`. + +Example configuration: +```yaml +recaptcha_siteverify_api: "https://my.recaptcha.site" +``` +--- +## TURN ## +Options related to adding a TURN server to Synapse. + +--- +Config option: `turn_uris` + +The public URIs of the TURN server to give to clients. + +Example configuration: +```yaml +turn_uris: [turn:example.org] +``` +--- +Config option: `turn_shared_secret` + +The shared secret used to compute passwords for the TURN server. + +Example configuration: +```yaml +turn_shared_secret: "YOUR_SHARED_SECRET" +``` +---- +Config options: `turn_username` and `turn_password` + +The Username and password if the TURN server needs them and does not use a token. + +Example configuration: +```yaml +turn_username: "TURNSERVER_USERNAME" +turn_password: "TURNSERVER_PASSWORD" +``` +--- +Config option: `turn_user_lifetime` + +How long generated TURN credentials last. Defaults to 1h. + +Example configuration: +```yaml +turn_user_lifetime: 2h +``` +--- +Config option: `turn_allow_guests` + +Whether guests should be allowed to use the TURN server. This defaults to true, otherwise +VoIP will be unreliable for guests. However, it does introduce a slight security risk as +it allows users to connect to arbitrary endpoints without having first signed up for a valid account (e.g. by passing a CAPTCHA). + +Example configuration: +```yaml +turn_allow_guests: false +``` +--- +## Registration ## + +Registration can be rate-limited using the parameters in the [Ratelimiting](#ratelimiting) section of this manual. + +--- +Config option: `enable_registration` + +Enable registration for new users. Defaults to false. It is highly recommended that if you enable registration, +you use either captcha, email, or token-based verification to verify that new users are not bots. In order to enable registration +without any verification, you must also set `enable_registration_without_verification` to true. + +Example configuration: +```yaml +enable_registration: true +``` +--- +Config option: `enable_registration_without_verification` +Enable registration without email or captcha verification. Note: this option is *not* recommended, +as registration without verification is a known vector for spam and abuse. Defaults to false. Has no effect +unless `enable_registration` is also enabled. + +Example configuration: +```yaml +enable_registration_without_verification: true +``` +--- +Config option: `session_lifetime` + +Time that a user's session remains valid for, after they log in. + +Note that this is not currently compatible with guest logins. + +Note also that this is calculated at login time: changes are not applied retrospectively to users who have already +logged in. + +By default, this is infinite. + +Example configuration: +```yaml +session_lifetime: 24h +``` +---- +Config option: `refresh_access_token_lifetime` + +Time that an access token remains valid for, if the session is using refresh tokens. + +For more information about refresh tokens, please see the [manual](user_authentication/refresh_tokens.md). + +Note that this only applies to clients which advertise support for refresh tokens. + +Note also that this is calculated at login time and refresh time: changes are not applied to +existing sessions until they are refreshed. + +By default, this is 5 minutes. + +Example configuration: +```yaml +refreshable_access_token_lifetime: 10m +``` +--- +Config option: `refresh_token_lifetime: 24h` + +Time that a refresh token remains valid for (provided that it is not +exchanged for another one first). +This option can be used to automatically log-out inactive sessions. +Please see the manual for more information. + +Note also that this is calculated at login time and refresh time: +changes are not applied to existing sessions until they are refreshed. + +By default, this is infinite. + +Example configuration: +```yaml +refresh_token_lifetime: 24h +``` +--- +Config option: `nonrefreshable_access_token_lifetime` + +Time that an access token remains valid for, if the session is NOT +using refresh tokens. + +Please note that not all clients support refresh tokens, so setting +this to a short value may be inconvenient for some users who will +then be logged out frequently. + +Note also that this is calculated at login time: changes are not applied +retrospectively to existing sessions for users that have already logged in. + +By default, this is infinite. + +Example configuration: +```yaml +nonrefreshable_access_token_lifetime: 24h +``` +--- +Config option: `registrations_require_3pid` + +If this is set, the user must provide all of the specified types of 3PID when registering. + +Example configuration: +```yaml +registrations_require_3pid: + - email + - msisdn +``` +--- +Config option: `disable_msisdn_registration` + +Explicitly disable asking for MSISDNs from the registration +flow (overrides `registrations_require_3pid` if MSISDNs are set as required). + +Example configuration: +```yaml +disable_msisdn_registration: true +``` +--- +Config option: `allowed_local_3pids` + +Mandate that users are only allowed to associate certain formats of +3PIDs with accounts on this server, as specified by the `medium` and `pattern` sub-options. + +Example configuration: +```yaml +allowed_local_3pids: + - medium: email + pattern: '^[^@]+@matrix\.org$' + - medium: email + pattern: '^[^@]+@vector\.im$' + - medium: msisdn + pattern: '\+44' +``` +--- +Config option: `enable_3pid_lookup` + +Enable 3PIDs lookup requests to identity servers from this server. Defaults to true. + +Example configuration: +```yaml +enable_3pid_lookup: false +``` +--- +Config option: `registration_requires_token` + +Require users to submit a token during registration. +Tokens can be managed using the admin [API](../administration/admin_api/registration_tokens.md). +Note that `enable_registration` must be set to true. +Disabling this option will not delete any tokens previously generated. +Defaults to false. Set to true to enable. + +Example configuration: +```yaml +registration_requires_token: true +``` +--- +Config option: `registration_shared_secret` + +If set, allows registration of standard or admin accounts by anyone who +has the shared secret, even if registration is otherwise disabled. + +Example configuration: +```yaml +registration_shared_secret: +``` +--- +Config option: `bcrypt_rounds` + +Set the number of bcrypt rounds used to generate password hash. +Larger numbers increase the work factor needed to generate the hash. +The default number is 12 (which equates to 2^12 rounds). +N.B. that increasing this will exponentially increase the time required +to register or login - e.g. 24 => 2^24 rounds which will take >20 mins. +Example configuration: +```yaml +bcrypt_rounds: 14 +``` +--- +Config option: `allow_guest_access` + +Allows users to register as guests without a password/email/etc, and +participate in rooms hosted on this server which have been made +accessible to anonymous users. Defaults to false. + +Example configuration: +```yaml +allow_guest_access: true +``` +--- +Config option: `default_identity_server` + +The identity server which we suggest that clients should use when users log +in on this server. + +(By default, no suggestion is made, so it is left up to the client. +This setting is ignored unless `public_baseurl` is also explicitly set.) + +Example configuration: +```yaml +default_identity_server: https://matrix.org +``` +--- +Config option: `account_threepid_delegates` + +Handle threepid (email/phone etc) registration and password resets through a set of +*trusted* identity servers. Note that this allows the configured identity server to +reset passwords for accounts! + +Be aware that if `email` is not set, and SMTP options have not been +configured in the email config block, registration and user password resets via +email will be globally disabled. + +Additionally, if `msisdn` is not set, registration and password resets via msisdn +will be disabled regardless, and users will not be able to associate an msisdn +identifier to their account. This is due to Synapse currently not supporting +any method of sending SMS messages on its own. + +To enable using an identity server for operations regarding a particular third-party +identifier type, set the value to the URL of that identity server as shown in the +examples below. + +Servers handling the these requests must answer the `/requestToken` endpoints defined +by the Matrix Identity Service API [specification](https://matrix.org/docs/spec/identity_service/latest). + +Example configuration: +```yaml +account_threepid_delegates: + email: https://example.com # Delegate email sending to example.com + msisdn: http://localhost:8090 # Delegate SMS sending to this local process +``` +--- +Config option: `enable_set_displayname` + +Whether users are allowed to change their displayname after it has +been initially set. Useful when provisioning users based on the +contents of a third-party directory. + +Does not apply to server administrators. Defaults to true. + +Example configuration: +```yaml +enable_set_displayname: false +``` +--- +Config option: `enable_set_avatar_url` + +Whether users are allowed to change their avatar after it has been +initially set. Useful when provisioning users based on the contents +of a third-party directory. + +Does not apply to server administrators. Defaults to true. + +Example configuration: +```yaml +enable_set_avatar_url: false +``` +--- +Config option: `enable_3pid_changes` + +Whether users can change the third-party IDs associated with their accounts +(email address and msisdn). + +Defaults to true. + +Example configuration: +```yaml +enable_3pid_changes: false +``` +--- +Config option: `auto_join_rooms` + +Users who register on this homeserver will automatically be joined +to the rooms listed under this option. + +By default, any room aliases included in this list will be created +as a publicly joinable room when the first user registers for the +homeserver. If the room already exists, make certain it is a publicly joinable +room, i.e. the join rule of the room must be set to 'public'. You can find more options +relating to auto-joining rooms below. + +Example configuration: +```yaml +auto_join_rooms: + - "#exampleroom:example.com" + - "#anotherexampleroom:example.com" +``` +--- +Config option: `autocreate_auto_join_rooms` + +Where `auto_join_rooms` are specified, setting this flag ensures that +the rooms exist by creating them when the first user on the +homeserver registers. + +By default the auto-created rooms are publicly joinable from any federated +server. Use the `autocreate_auto_join_rooms_federated` and +`autocreate_auto_join_room_preset` settings to customise this behaviour. + +Setting to false means that if the rooms are not manually created, +users cannot be auto-joined since they do not exist. + +Defaults to true. + +Example configuration: +```yaml +autocreate_auto_join_rooms: false +``` +--- +Config option: `autocreate_auto_join_rooms_federated` + +Whether the rooms listen in `auto_join_rooms` that are auto-created are available +via federation. Only has an effect if `autocreate_auto_join_rooms` is true. + +Note that whether a room is federated cannot be modified after +creation. + +Defaults to true: the room will be joinable from other servers. +Set to false to prevent users from other homeservers from +joining these rooms. + +Example configuration: +```yaml +autocreate_auto_join_rooms_federated: false +``` +--- +Config option: `autocreate_auto_join_room_preset` + +The room preset to use when auto-creating one of `auto_join_rooms`. Only has an +effect if `autocreate_auto_join_rooms` is true. + +Possible values for this option are: +* "public_chat": the room is joinable by anyone, including + federated servers if `autocreate_auto_join_rooms_federated` is true (the default). +* "private_chat": an invitation is required to join these rooms. +* "trusted_private_chat": an invitation is required to join this room and the invitee is + assigned a power level of 100 upon joining the room. + +If a value of "private_chat" or "trusted_private_chat" is used then +`auto_join_mxid_localpart` must also be configured. + +Defaults to "public_chat". + +Example configuration: +```yaml +autocreate_auto_join_room_preset: private_chat +``` +--- +Config option: `auto_join_mxid_localpart` + +The local part of the user id which is used to create `auto_join_rooms` if +`autocreate_auto_join_rooms` is true. If this is not provided then the +initial user account that registers will be used to create the rooms. + +The user id is also used to invite new users to any auto-join rooms which +are set to invite-only. + +It *must* be configured if `autocreate_auto_join_room_preset` is set to +"private_chat" or "trusted_private_chat". + +Note that this must be specified in order for new users to be correctly +invited to any auto-join rooms which have been set to invite-only (either +at the time of creation or subsequently). + +Note that, if the room already exists, this user must be joined and +have the appropriate permissions to invite new members. + +Example configuration: +```yaml +auto_join_mxid_localpart: system +``` +--- +Config option: `auto_join_rooms_for_guests` + +When `auto_join_rooms` is specified, setting this flag to false prevents +guest accounts from being automatically joined to the rooms. + +Defaults to true. + +Example configuration: +```yaml +auto_join_rooms_for_guests: false +``` +--- +Config option: `inhibit_user_in_use_error` + +Whether to inhibit errors raised when registering a new account if the user ID +already exists. If turned on, requests to `/register/available` will always +show a user ID as available, and Synapse won't raise an error when starting +a registration with a user ID that already exists. However, Synapse will still +raise an error if the registration completes and the username conflicts. + +Defaults to false. + +Example configuration: +```yaml +inhibit_user_in_use_error: true +``` +--- +## Metrics ### +Config options related to metrics. + +--- +Config option: `enable_metrics` + +Set to true to enable collection and rendering of performance metrics. +Defaults to false. + +Example configuration: +```yaml +enable_metrics: true +``` +--- +Config option: `sentry` + +Use this option to enable sentry integration. Provide the DSN assigned to you by sentry +with the `dsn` setting. + +NOTE: While attempts are made to ensure that the logs don't contain +any sensitive information, this cannot be guaranteed. By enabling +this option the sentry server may therefore receive sensitive +information, and it in turn may then disseminate sensitive information +through insecure notification channels if so configured. + +Example configuration: +```yaml +sentry: + dsn: "..." +``` +--- +Config option: `metrics_flags` + +Flags to enable Prometheus metrics which are not suitable to be +enabled by default, either for performance reasons or limited use. +Currently the only option is `known_servers`, which publishes +`synapse_federation_known_servers`, a gauge of the number of +servers this homeserver knows about, including itself. May cause +performance problems on large homeservers. + +Example configuration: +```yaml +metrics_flags: + known_servers: true +``` +--- +Config option: `report_stats` + +Whether or not to report anonymized homeserver usage statistics. This is originally +set when generating the config. Set this option to true or false to change the current +behavior. + +Example configuration: +```yaml +report_stats: true +``` +--- +Config option: `report_stats_endpoint` + +The endpoint to report the anonymized homeserver usage statistics to. +Defaults to https://matrix.org/report-usage-stats/push + +Example configuration: +```yaml +report_stats_endpoint: https://example.com/report-usage-stats/push +``` +--- +## API Configuration ## +Config settings related to the client/server API + +--- +Config option: `room_prejoin_state:` + +Controls for the state that is shared with users who receive an invite +to a room. By default, the following state event types are shared with users who +receive invites to the room: +- m.room.join_rules +- m.room.canonical_alias +- m.room.avatar +- m.room.encryption +- m.room.name +- m.room.create +- m.room.topic + +To change the default behavior, use the following sub-options: +* `disable_default_event_types`: set to true to disable the above defaults. If this + is enabled, only the event types listed in `additional_event_types` are shared. + Defaults to false. +* `additional_event_types`: Additional state event types to share with users when they are invited + to a room. By default, this list is empty (so only the default event types are shared). + +Example configuration: +```yaml +room_prejoin_state: + disable_default_event_types: true + additional_event_types: + - org.example.custom.event.type + - m.room.join_rules +``` +--- +Config option: `track_puppeted_user_ips` + +We record the IP address of clients used to access the API for various +reasons, including displaying it to the user in the "Where you're signed in" +dialog. + +By default, when puppeting another user via the admin API, the client IP +address is recorded against the user who created the access token (ie, the +admin user), and *not* the puppeted user. + +Set this option to true to also record the IP address against the puppeted +user. (This also means that the puppeted user will count as an "active" user +for the purpose of monthly active user tracking - see `limit_usage_by_mau` etc +above.) + +Example configuration: +```yaml +track_puppeted_user_ips: true +``` +--- +Config option: `app_service_config_files` + +A list of application service config files to use. + +Example configuration: +```yaml +app_service_config_files: + - app_service_1.yaml + - app_service_2.yaml +``` +--- +Config option: `track_appservice_user_ips` + +Defaults to false. Set to true to enable tracking of application service IP addresses. +Implicitly enables MAU tracking for application service users. + +Example configuration: +```yaml +track_appservice_user_ips: true +``` +--- +Config option: `macaroon_secret_key` + +A secret which is used to sign access tokens. If none is specified, +the `registration_shared_secret` is used, if one is given; otherwise, +a secret key is derived from the signing key. + +Example configuration: +```yaml +macaroon_secret_key: +``` +--- +Config option: `form_secret` + +A secret which is used to calculate HMACs for form values, to stop +falsification of values. Must be specified for the User Consent +forms to work. + +Example configuration: +```yaml +form_secret: +``` +--- +## Signing Keys ## +Config options relating to signing keys + +--- +Config option: `signing_key_path` + +Path to the signing key to sign messages with. + +Example configuration: +```yaml +signing_key_path: "CONFDIR/SERVERNAME.signing.key" +``` +--- +Config option: `old_signing_keys` + +The keys that the server used to sign messages with but won't use +to sign new messages. For each key, `key` should be the base64-encoded public key, and +`expired_ts`should be the time (in milliseconds since the unix epoch) that +it was last used. + +It is possible to build an entry from an old `signing.key` file using the +`export_signing_key` script which is provided with synapse. + +Example configuration: +```yaml +old_signing_keys: + "ed25519:id": { key: "base64string", expired_ts: 123456789123 } +``` +--- +Config option: `key_refresh_interval` + +How long key response published by this server is valid for. +Used to set the `valid_until_ts` in `/key/v2` APIs. +Determines how quickly servers will query to check which keys +are still valid. Defaults to 1d. + +Example configuration: +```yaml +key_refresh_interval: 2d +``` +--- +Config option: `trusted_key_servers:` + +The trusted servers to download signing keys from. + +When we need to fetch a signing key, each server is tried in parallel. + +Normally, the connection to the key server is validated via TLS certificates. +Additional security can be provided by configuring a `verify key`, which +will make synapse check that the response is signed by that key. + +This setting supercedes an older setting named `perspectives`. The old format +is still supported for backwards-compatibility, but it is deprecated. + +`trusted_key_servers` defaults to matrix.org, but using it will generate a +warning on start-up. To suppress this warning, set +`suppress_key_server_warning` to true. + +Options for each entry in the list include: +* `server_name`: the name of the server. Required. +* `verify_keys`: an optional map from key id to base64-encoded public key. + If specified, we will check that the response is signed by at least + one of the given keys. +* `accept_keys_insecurely`: a boolean. Normally, if `verify_keys` is unset, + and `federation_verify_certificates` is not `true`, synapse will refuse + to start, because this would allow anyone who can spoof DNS responses + to masquerade as the trusted key server. If you know what you are doing + and are sure that your network environment provides a secure connection + to the key server, you can set this to `true` to override this behaviour. + +Example configuration #1: +```yaml +trusted_key_servers: + - server_name: "my_trusted_server.example.com" + verify_keys: + "ed25519:auto": "abcdefghijklmnopqrstuvwxyzabcdefghijklmopqr" + - server_name: "my_other_trusted_server.example.com" +``` +Example configuration #2: +```yaml +trusted_key_servers: + - server_name: "matrix.org" +``` +--- +Config option: `suppress_key_server_warning` + +Set the following to true to disable the warning that is emitted when the +`trusted_key_servers` include 'matrix.org'. See above. + +Example configuration: +```yaml +suppress_key_server_warning: true +``` +--- +Config option: `key_server_signing_keys_path` + +The signing keys to use when acting as a trusted key server. If not specified +defaults to the server signing key. + +Can contain multiple keys, one per line. + +Example configuration: +```yaml +key_server_signing_keys_path: "key_server_signing_keys.key" +``` +--- +## Single sign-on integration ## + +The following settings can be used to make Synapse use a single sign-on +provider for authentication, instead of its internal password database. + +You will probably also want to set the following options to false to +disable the regular login/registration flows: + * `enable_registration` + * `password_config.enabled` + +You will also want to investigate the settings under the "sso" configuration +section below. + +--- +Config option: `saml2_config` + +Enable SAML2 for registration and login. Uses pysaml2. To learn more about pysaml and +to find a full list options for configuring pysaml, read the docs [here](https://pysaml2.readthedocs.io/en/latest/). + +At least one of `sp_config` or `config_path` must be set in this section to +enable SAML login. You can either put your entire pysaml config inline using the `sp_config` +option, or you can specify a path to a psyaml config file with the sub-option `config_path`. +This setting has the following sub-options: + +* `sp_config`: the configuration for the pysaml2 Service Provider. See pysaml2 docs for format of config. + Default values will be used for the `entityid` and `service` settings, + so it is not normally necessary to specify them unless you need to + override them. Here are a few useful sub-options for configuring pysaml: + * `metadata`: Point this to the IdP's metadata. You must provide either a local + file via the `local` attribute or (preferably) a URL via the + `remote` attribute. + * `accepted_time_diff: 3`: Allowed clock difference in seconds between the homeserver and IdP. + Defaults to 0. + * `service`: By default, the user has to go to our login page first. If you'd like + to allow IdP-initiated login, set `allow_unsolicited` to true under `sp` in the `service` + section. +* `config_path`: specify a separate pysaml2 configuration file thusly: + `config_path: "CONFDIR/sp_conf.py"` +* `saml_session_lifetime`: The lifetime of a SAML session. This defines how long a user has to + complete the authentication process, if `allow_unsolicited` is unset. The default is 15 minutes. +* `user_mapping_provider`: Using this option, an external module can be provided as a + custom solution to mapping attributes returned from a saml provider onto a matrix user. The + `user_mapping_provider` has the following attributes: + * `module`: The custom module's class. + * `config`: Custom configuration values for the module. Use the values provided in the + example if you are using the built-in user_mapping_provider, or provide your own + config values for a custom class if you are using one. This section will be passed as a Python + dictionary to the module's `parse_config` method. The built-in provider takes the following two + options: + * `mxid_source_attribute`: The SAML attribute (after mapping via the attribute maps) to use + to derive the Matrix ID from. It is 'uid' by default. Note: This used to be configured by the + `saml2_config.mxid_source_attribute option`. If that is still defined, its value will be used instead. + * `mxid_mapping`: The mapping system to use for mapping the saml attribute onto a + matrix ID. Options include: `hexencode` (which maps unpermitted characters to '=xx') + and `dotreplace` (which replaces unpermitted characters with '.'). + The default is `hexencode`. Note: This used to be configured by the + `saml2_config.mxid_mapping option`. If that is still defined, its value will be used instead. +* `grandfathered_mxid_source_attribute`: In previous versions of synapse, the mapping from SAML attribute to + MXID was always calculated dynamically rather than stored in a table. For backwards- compatibility, we will look for `user_ids` + matching such a pattern before creating a new account. This setting controls the SAML attribute which will be used for this + backwards-compatibility lookup. Typically it should be 'uid', but if the attribute maps are changed, it may be necessary to change it. + The default is 'uid'. +* `attribute_requirements`: It is possible to configure Synapse to only allow logins if SAML attributes + match particular values. The requirements can be listed under + `attribute_requirements` as shown in the example. All of the listed attributes must + match for the login to be permitted. +* `idp_entityid`: If the metadata XML contains multiple IdP entities then the `idp_entityid` + option must be set to the entity to redirect users to. + Most deployments only have a single IdP entity and so should omit this option. + + +Once SAML support is enabled, a metadata file will be exposed at +`https://:/_synapse/client/saml2/metadata.xml`, which you may be able to +use to configure your SAML IdP with. Alternatively, you can manually configure +the IdP to use an ACS location of +`https://:/_synapse/client/saml2/authn_response`. + +Example configuration: +```yaml +saml2_config: + sp_config: + metadata: + local: ["saml2/idp.xml"] + remote: + - url: https://our_idp/metadata.xml + accepted_time_diff: 3 + + service: + sp: + allow_unsolicited: true + + # The examples below are just used to generate our metadata xml, and you + # may well not need them, depending on your setup. Alternatively you + # may need a whole lot more detail - see the pysaml2 docs! + description: ["My awesome SP", "en"] + name: ["Test SP", "en"] + + ui_info: + display_name: + - lang: en + text: "Display Name is the descriptive name of your service." + description: + - lang: en + text: "Description should be a short paragraph explaining the purpose of the service." + information_url: + - lang: en + text: "https://example.com/terms-of-service" + privacy_statement_url: + - lang: en + text: "https://example.com/privacy-policy" + keywords: + - lang: en + text: ["Matrix", "Element"] + logo: + - lang: en + text: "https://example.com/logo.svg" + width: "200" + height: "80" + + organization: + name: Example com + display_name: + - ["Example co", "en"] + url: "http://example.com" + + contact_person: + - given_name: Bob + sur_name: "the Sysadmin" + email_address": ["admin@example.com"] + contact_type": technical + + saml_session_lifetime: 5m + + user_mapping_provider: + # Below options are intended for the built-in provider, they should be + # changed if using a custom module. + config: + mxid_source_attribute: displayName + mxid_mapping: dotreplace + + grandfathered_mxid_source_attribute: upn + + attribute_requirements: + - attribute: userGroup + value: "staff" + - attribute: department + value: "sales" + + idp_entityid: 'https://our_idp/entityid' +``` +--- +Config option: `oidc_providers` + +List of OpenID Connect (OIDC) / OAuth 2.0 identity providers, for registration +and login. See [here](../../openid.md) +for information on how to configure these options. + +For backwards compatibility, it is also possible to configure a single OIDC +provider via an `oidc_config` setting. This is now deprecated and admins are +advised to migrate to the `oidc_providers` format. (When doing that migration, +use `oidc` for the `idp_id` to ensure that existing users continue to be +recognised.) + +Options for each entry include: +* `idp_id`: a unique identifier for this identity provider. Used internally + by Synapse; should be a single word such as 'github'. + Note that, if this is changed, users authenticating via that provider + will no longer be recognised as the same user! + (Use "oidc" here if you are migrating from an old `oidc_config` configuration.) + +* `idp_name`: A user-facing name for this identity provider, which is used to + offer the user a choice of login mechanisms. + +* `idp_icon`: An optional icon for this identity provider, which is presented + by clients and Synapse's own IdP picker page. If given, must be an + MXC URI of the format mxc:///. (An easy way to + obtain such an MXC URI is to upload an image to an (unencrypted) room + and then copy the "url" from the source of the event.) + +* `idp_brand`: An optional brand for this identity provider, allowing clients + to style the login flow according to the identity provider in question. + See the [spec](https://spec.matrix.org/latest/) for possible options here. + +* `discover`: set to false to disable the use of the OIDC discovery mechanism + to discover endpoints. Defaults to true. + +* `issuer`: Required. The OIDC issuer. Used to validate tokens and (if discovery + is enabled) to discover the provider's endpoints. + +* `client_id`: Required. oauth2 client id to use. + +* `client_secret`: oauth2 client secret to use. May be omitted if + `client_secret_jwt_key` is given, or if `client_auth_method` is 'none'. + +* `client_secret_jwt_key`: Alternative to client_secret: details of a key used + to create a JSON Web Token to be used as an OAuth2 client secret. If + given, must be a dictionary with the following properties: + + * `key`: a pem-encoded signing key. Must be a suitable key for the + algorithm specified. Required unless `key_file` is given. + + * `key_file`: the path to file containing a pem-encoded signing key file. + Required unless `key` is given. + + * `jwt_header`: a dictionary giving properties to include in the JWT + header. Must include the key `alg`, giving the algorithm used to + sign the JWT, such as "ES256", using the JWA identifiers in + RFC7518. + + * `jwt_payload`: an optional dictionary giving properties to include in + the JWT payload. Normally this should include an `iss` key. + +* `client_auth_method`: auth method to use when exchanging the token. Valid + values are `client_secret_basic` (default), `client_secret_post` and + `none`. + +* `scopes`: list of scopes to request. This should normally include the "openid" + scope. Defaults to ["openid"]. + +* `authorization_endpoint`: the oauth2 authorization endpoint. Required if + provider discovery is disabled. + +* `token_endpoint`: the oauth2 token endpoint. Required if provider discovery is + disabled. + +* `userinfo_endpoint`: the OIDC userinfo endpoint. Required if discovery is + disabled and the 'openid' scope is not requested. + +* `jwks_uri`: URI where to fetch the JWKS. Required if discovery is disabled and + the 'openid' scope is used. + +* `skip_verification`: set to 'true' to skip metadata verification. Use this if + you are connecting to a provider that is not OpenID Connect compliant. + Defaults to false. Avoid this in production. + +* `user_profile_method`: Whether to fetch the user profile from the userinfo + endpoint, or to rely on the data returned in the id_token from the `token_endpoint`. + Valid values are: `auto` or `userinfo_endpoint`. + Defaults to `auto`, which uses the userinfo endpoint if `openid` is + not included in `scopes`. Set to `userinfo_endpoint` to always use the + userinfo endpoint. + +* `allow_existing_users`: set to true to allow a user logging in via OIDC to + match a pre-existing account instead of failing. This could be used if + switching from password logins to OIDC. Defaults to false. + +* `user_mapping_provider`: Configuration for how attributes returned from a OIDC + provider are mapped onto a matrix user. This setting has the following + sub-properties: + + * `module`: The class name of a custom mapping module. Default is + `synapse.handlers.oidc.JinjaOidcMappingProvider`. + See https://matrix-org.github.io/synapse/latest/sso_mapping_providers.html#openid-mapping-providers + for information on implementing a custom mapping provider. + + * `config`: Configuration for the mapping provider module. This section will + be passed as a Python dictionary to the user mapping provider + module's `parse_config` method. + + For the default provider, the following settings are available: + + * subject_claim: name of the claim containing a unique identifier + for the user. Defaults to 'sub', which OpenID Connect + compliant providers should provide. + + * `localpart_template`: Jinja2 template for the localpart of the MXID. + If this is not set, the user will be prompted to choose their + own username (see the documentation for the `sso_auth_account_details.html` + template). This template can use the `localpart_from_email` filter. + + * `confirm_localpart`: Whether to prompt the user to validate (or + change) the generated localpart (see the documentation for the + 'sso_auth_account_details.html' template), instead of + registering the account right away. + + * `display_name_template`: Jinja2 template for the display name to set + on first login. If unset, no displayname will be set. + + * `email_template`: Jinja2 template for the email address of the user. + If unset, no email address will be added to the account. + + * `extra_attributes`: a map of Jinja2 templates for extra attributes + to send back to the client during login. Note that these are non-standard and clients will ignore them + without modifications. + + When rendering, the Jinja2 templates are given a 'user' variable, + which is set to the claims returned by the UserInfo Endpoint and/or + in the ID Token. + + +It is possible to configure Synapse to only allow logins if certain attributes +match particular values in the OIDC userinfo. The requirements can be listed under +`attribute_requirements` as shown here: +```yaml +attribute_requirements: + - attribute: family_name + value: "Stephensson" + - attribute: groups + value: "admin" +``` +All of the listed attributes must match for the login to be permitted. Additional attributes can be added to +userinfo by expanding the `scopes` section of the OIDC config to retrieve +additional information from the OIDC provider. + +If the OIDC claim is a list, then the attribute must match any value in the list. +Otherwise, it must exactly match the value of the claim. Using the example +above, the `family_name` claim MUST be "Stephensson", but the `groups` +claim MUST contain "admin". + +Example configuration: +```yaml +oidc_providers: + # Generic example + # + - idp_id: my_idp + idp_name: "My OpenID provider" + idp_icon: "mxc://example.com/mediaid" + discover: false + issuer: "https://accounts.example.com/" + client_id: "provided-by-your-issuer" + client_secret: "provided-by-your-issuer" + client_auth_method: client_secret_post + scopes: ["openid", "profile"] + authorization_endpoint: "https://accounts.example.com/oauth2/auth" + token_endpoint: "https://accounts.example.com/oauth2/token" + userinfo_endpoint: "https://accounts.example.com/userinfo" + jwks_uri: "https://accounts.example.com/.well-known/jwks.json" + skip_verification: true + user_mapping_provider: + config: + subject_claim: "id" + localpart_template: "{{ user.login }}" + display_name_template: "{{ user.name }}" + email_template: "{{ user.email }}" + attribute_requirements: + - attribute: userGroup + value: "synapseUsers" +``` +--- +Config option: `cas_config` + +Enable Central Authentication Service (CAS) for registration and login. +Has the following sub-options: +* `enabled`: Set this to true to enable authorization against a CAS server. + Defaults to false. +* `server_url`: The URL of the CAS authorization endpoint. +* `displayname_attribute`: The attribute of the CAS response to use as the display name. + If no name is given here, no displayname will be set. +* `required_attributes`: It is possible to configure Synapse to only allow logins if CAS attributes + match particular values. All of the keys given below must exist + and the values must match the given value. Alternately if the given value + is `None` then any value is allowed (the attribute just must exist). + All of the listed attributes must match for the login to be permitted. + +Example configuration: +```yaml +cas_config: + enabled: true + server_url: "https://cas-server.com" + displayname_attribute: name + required_attributes: + userGroup: "staff" + department: None +``` +--- +Config option: `sso` + +Additional settings to use with single-sign on systems such as OpenID Connect, +SAML2 and CAS. + +Server admins can configure custom templates for pages related to SSO. See +[here](../../templates.md) for more information. + +Options include: +* `client_whitelist`: A list of client URLs which are whitelisted so that the user does not + have to confirm giving access to their account to the URL. Any client + whose URL starts with an entry in the following list will not be subject + to an additional confirmation step after the SSO login is completed. + WARNING: An entry such as "https://my.client" is insecure, because it + will also match "https://my.client.evil.site", exposing your users to + phishing attacks from evil.site. To avoid this, include a slash after the + hostname: "https://my.client/". + The login fallback page (used by clients that don't natively support the + required login flows) is whitelisted in addition to any URLs in this list. + By default, this list contains only the login fallback page. +* `update_profile_information`: Use this setting to keep a user's profile fields in sync with information from + the identity provider. Currently only syncing the displayname is supported. Fields + are checked on every SSO login, and are updated if necessary. + Note that enabling this option will override user profile information, + regardless of whether users have opted-out of syncing that + information when first signing in. Defaults to false. + + +Example configuration: +```yaml +sso: + client_whitelist: + - https://riot.im/develop + - https://my.custom.client/ + update_profile_information: true +``` +--- +Config option: `jwt_config` + +JSON web token integration. The following settings can be used to make +Synapse JSON web tokens for authentication, instead of its internal +password database. + +Each JSON Web Token needs to contain a "sub" (subject) claim, which is +used as the localpart of the mxid. + +Additionally, the expiration time ("exp"), not before time ("nbf"), +and issued at ("iat") claims are validated if present. + +Note that this is a non-standard login type and client support is +expected to be non-existent. + +See [here](../../jwt.md) for more. + +Additional sub-options for this setting include: +* `enabled`: Set to true to enable authorization using JSON web + tokens. Defaults to false. +* `secret`: This is either the private shared secret or the public key used to + decode the contents of the JSON web token. Required if `enabled` is set to true. +* `algorithm`: The algorithm used to sign the JSON web token. Supported algorithms are listed at + https://pyjwt.readthedocs.io/en/latest/algorithms.html Required if `enabled` is set to true. +* `subject_claim`: Name of the claim containing a unique identifier for the user. + Optional, defaults to `sub`. +* `issuer`: The issuer to validate the "iss" claim against. Optional. If provided the + "iss" claim will be required and validated for all JSON web tokens. +* `audiences`: A list of audiences to validate the "aud" claim against. Optional. + If provided the "aud" claim will be required and validated for all JSON web tokens. + Note that if the "aud" claim is included in a JSON web token then + validation will fail without configuring audiences. + +Example configuration: +```yaml +jwt_config: + enabled: true + secret: "provided-by-your-issuer" + algorithm: "provided-by-your-issuer" + subject_claim: "name_of_claim" + issuer: "provided-by-your-issuer" + audiences: + - "provided-by-your-issuer" +``` +--- +Config option: `password_config` + +Use this setting to enable password-based logins. + +This setting has the following sub-options: +* `enabled`: Defaults to true. +* `localdb_enabled`: Set to false to disable authentication against the local password + database. This is ignored if `enabled` is false, and is only useful + if you have other `password_providers`. Defaults to true. +* `pepper`: Set the value here to a secret random string for extra security. # Uncomment and change to a secret random string for extra security. + DO NOT CHANGE THIS AFTER INITIAL SETUP! +* `policy`: Define and enforce a password policy, such as minimum lengths for passwords, etc. + Each parameter is optional. This is an implementation of MSC2000. Parameters are as follows: + * `enabled`: Defaults to false. Set to true to enable. + * `minimum_length`: Minimum accepted length for a password. Defaults to 0. + * `require_digit`: Whether a password must contain at least one digit. + Defaults to false. + * `require_symbol`: Whether a password must contain at least one symbol. + A symbol is any character that's not a number or a letter. Defaults to false. + * `require_lowercase`: Whether a password must contain at least one lowercase letter. + Defaults to false. + * `require_uppercase`: Whether a password must contain at least one uppercase letter. + Defaults to false. + + +Example configuration: +```yaml +password_config: + enabled: false + localdb_enabled: false + pepper: "EVEN_MORE_SECRET" + + policy: + enabled: true + minimum_length: 15 + require_digit: true + require_symbol: true + require_lowercase: true + require_uppercase: true +``` +--- +Config option: `ui_auth` + +The amount of time to allow a user-interactive authentication session to be active. + +This defaults to 0, meaning the user is queried for their credentials +before every action, but this can be overridden to allow a single +validation to be re-used. This weakens the protections afforded by +the user-interactive authentication process, by allowing for multiple +(and potentially different) operations to use the same validation session. + +This is ignored for potentially "dangerous" operations (including +deactivating an account, modifying an account password, and +adding a 3PID). + +Use the `session_timeout` sub-option here to change the time allowed for credential validation. + +Example configuration: +```yaml +ui_auth: + session_timeout: "15s" +``` +--- +Config option: `email` + +Configuration for sending emails from Synapse. + +Server admins can configure custom templates for email content. See +[here](../../templates.md) for more information. + +This setting has the following sub-options: +* `smtp_host`: The hostname of the outgoing SMTP server to use. Defaults to 'localhost'. +* `smtp_port`: The port on the mail server for outgoing SMTP. Defaults to 25. +* `smtp_user` and `smtp_pass`: Username/password for authentication to the SMTP server. By default, no + authentication is attempted. +* `require_transport_security`: Set to true to require TLS transport security for SMTP. + By default, Synapse will connect over plain text, and will then switch to + TLS via STARTTLS *if the SMTP server supports it*. If this option is set, + Synapse will refuse to connect unless the server supports STARTTLS. +* `enable_tls`: By default, if the server supports TLS, it will be used, and the server + must present a certificate that is valid for 'smtp_host'. If this option + is set to false, TLS will not be used. +* `notif_from`: defines the "From" address to use when sending emails. + It must be set if email sending is enabled. The placeholder '%(app)s' will be replaced by the application name, + which is normally set in `app_name`, but may be overridden by the + Matrix client application. Note that the placeholder must be written '%(app)s', including the + trailing 's'. +* `app_name`: `app_name` defines the default value for '%(app)s' in `notif_from` and email + subjects. It defaults to 'Matrix'. +* `enable_notifs`: Set to true to enable sending emails for messages that the user + has missed. Disabled by default. +* `notif_for_new_users`: Set to false to disable automatic subscription to email + notifications for new users. Enabled by default. +* `client_base_url`: Custom URL for client links within the email notifications. By default + links will be based on "https://matrix.to". (This setting used to be called `riot_base_url`; + the old name is still supported for backwards-compatibility but is now deprecated.) +* `validation_token_lifetime`: Configures the time that a validation email will expire after sending. + Defaults to 1h. +* `invite_client_location`: The web client location to direct users to during an invite. This is passed + to the identity server as the `org.matrix.web_client_location` key. Defaults + to unset, giving no guidance to the identity server. +* `subjects`: Subjects to use when sending emails from Synapse. The placeholder '%(app)s' will + be replaced with the value of the `app_name` setting, or by a value dictated by the Matrix client application. + In addition, each subject can use the following placeholders: '%(person)s', which will be replaced by the displayname + of the user(s) that sent the message(s), e.g. "Alice and Bob", and '%(room)s', which will be replaced by the name of the room the + message(s) have been sent to, e.g. "My super room". In addition, emails related to account administration will + can use the '%(server_name)s' placeholder, which will be replaced by the value of the + `server_name` setting in your Synapse configuration. + + Here is a list of subjects for notification emails that can be set: + * `message_from_person_in_room`: Subject to use to notify about one message from one or more user(s) in a + room which has a name. Defaults to "[%(app)s] You have a message on %(app)s from %(person)s in the %(room)s room..." + * `message_from_person`: Subject to use to notify about one message from one or more user(s) in a + room which doesn't have a name. Defaults to "[%(app)s] You have a message on %(app)s from %(person)s..." + * `messages_from_person`: Subject to use to notify about multiple messages from one or more users in + a room which doesn't have a name. Defaults to "[%(app)s] You have messages on %(app)s from %(person)s..." + * `messages_in_room`: Subject to use to notify about multiple messages in a room which has a + name. Defaults to "[%(app)s] You have messages on %(app)s in the %(room)s room..." + * `messages_in_room_and_others`: Subject to use to notify about multiple messages in multiple rooms. + Defaults to "[%(app)s] You have messages on %(app)s in the %(room)s room and others..." + * `messages_from_person_and_others`: Subject to use to notify about multiple messages from multiple persons in + multiple rooms. This is similar to the setting above except it's used when + the room in which the notification was triggered has no name. Defaults to + "[%(app)s] You have messages on %(app)s from %(person)s and others..." + * `invite_from_person_to_room`: Subject to use to notify about an invite to a room which has a name. + Defaults to "[%(app)s] %(person)s has invited you to join the %(room)s room on %(app)s..." + * `invite_from_person`: Subject to use to notify about an invite to a room which doesn't have a + name. Defaults to "[%(app)s] %(person)s has invited you to chat on %(app)s..." + * `password_reset`: Subject to use when sending a password reset email. Defaults to "[%(server_name)s] Password reset" + * `email_validation`: Subject to use when sending a verification email to assert an address's + ownership. Defaults to "[%(server_name)s] Validate your email" + +Example configuration: +```yaml +email: + smtp_host: mail.server + smtp_port: 587 + smtp_user: "exampleusername" + smtp_pass: "examplepassword" + require_transport_security: true + enable_tls: false + notif_from: "Your Friendly %(app)s homeserver " + app_name: my_branded_matrix_server + enable_notifs: true + notif_for_new_users: false + client_base_url: "http://localhost/riot" + validation_token_lifetime: 15m + invite_client_location: https://app.element.io + + subjects: + message_from_person_in_room: "[%(app)s] You have a message on %(app)s from %(person)s in the %(room)s room..." + message_from_person: "[%(app)s] You have a message on %(app)s from %(person)s..." + messages_from_person: "[%(app)s] You have messages on %(app)s from %(person)s..." + messages_in_room: "[%(app)s] You have messages on %(app)s in the %(room)s room..." + messages_in_room_and_others: "[%(app)s] You have messages on %(app)s in the %(room)s room and others..." + messages_from_person_and_others: "[%(app)s] You have messages on %(app)s from %(person)s and others..." + invite_from_person_to_room: "[%(app)s] %(person)s has invited you to join the %(room)s room on %(app)s..." + invite_from_person: "[%(app)s] %(person)s has invited you to chat on %(app)s..." + password_reset: "[%(server_name)s] Password reset" + email_validation: "[%(server_name)s] Validate your email" +``` +--- +## Push ## +Configuration settings related to push notifications + +--- +Config option: `push` + +This setting defines options for push notifications. + +This option has a number of sub-options. They are as follows: +* `include_content`: Clients requesting push notifications can either have the body of + the message sent in the notification poke along with other details + like the sender, or just the event ID and room ID (`event_id_only`). + If clients choose the to have the body sent, this option controls whether the + notification request includes the content of the event (other details + like the sender are still included). If `event_id_only` is enabled, it + has no effect. + For modern android devices the notification content will still appear + because it is loaded by the app. iPhone, however will send a + notification saying only that a message arrived and who it came from. + Defaults to true. Set to false to only include the event ID and room ID in push notification payloads. +* `group_unread_count_by_room: false`: When a push notification is received, an unread count is also sent. + This number can either be calculated as the number of unread messages for the user, or the number of *rooms* the + user has unread messages in. Defaults to true, meaning push clients will see the number of + rooms with unread messages in them. Set to false to instead send the number + of unread messages. + +Example configuration: +```yaml +push: + include_content: false + group_unread_count_by_room: false +``` +--- +## Rooms ## +Config options relating to rooms. + +--- +Config option: `encryption_enabled_by_default` + +Controls whether locally-created rooms should be end-to-end encrypted by +default. + +Possible options are "all", "invite", and "off". They are defined as: + +* "all": any locally-created room +* "invite": any room created with the `private_chat` or `trusted_private_chat` + room creation presets +* "off": this option will take no effect + +The default value is "off". + +Note that this option will only affect rooms created after it is set. It +will also not affect rooms created by other servers. + +Example configuration: +```yaml +encryption_enabled_by_default_for_room_type: invite +``` +--- +Config option: `enable_group_creation` + +Set to true to allow non-server-admin users to create groups on this server + +Example configuration: +```yaml +enable_group_creation: true +``` +--- +Config option: `group_creation_prefix` + +If enabled/present, non-server admins can only create groups with local parts +starting with this prefix. + +Example configuration: +```yaml +group_creation_prefix: "unofficial_" +``` +--- +Config option: `user_directory` + +This setting defines options related to the user directory. + +This option has the following sub-options: +* `enabled`: Defines whether users can search the user directory. If false then + empty responses are returned to all queries. Defaults to true. +* `search_all_users`: Defines whether to search all users visible to your HS when searching + the user directory. If false, search results will only contain users + visible in public rooms and users sharing a room with the requester. + Defaults to false. + NB. If you set this to true, and the last time the user_directory search + indexes were (re)built was before Synapse 1.44, you'll have to + rebuild the indexes in order to search through all known users. + These indexes are built the first time Synapse starts; admins can + manually trigger a rebuild via API following the instructions at + https://matrix-org.github.io/synapse/latest/usage/administration/admin_api/background_updates.html#run + Set to true to return search results containing all known users, even if that + user does not share a room with the requester. +* `prefer_local_users`: Defines whether to prefer local users in search query results. + If set to true, local users are more likely to appear above remote users when searching the + user directory. Defaults to false. + +Example configuration: +```yaml +user_directory: + enabled: false + search_all_users: true + prefer_local_users: true +``` +--- +Config option: `user_consent` + +For detailed instructions on user consent configuration, see [here](../../consent_tracking.md). + +Parts of this section are required if enabling the `consent` resource under +`listeners`, in particular `template_dir` and `version`. # TODO: link `listeners` + +* `template_dir`: gives the location of the templates for the HTML forms. + This directory should contain one subdirectory per language (eg, `en`, `fr`), + and each language directory should contain the policy document (named as + .html) and a success page (success.html). + +* `version`: specifies the 'current' version of the policy document. It defines + the version to be served by the consent resource if there is no 'v' + parameter. + +* `server_notice_content`: if enabled, will send a user a "Server Notice" + asking them to consent to the privacy policy. The `server_notices` section ##TODO: link + must also be configured for this to work. Notices will *not* be sent to + guest users unless `send_server_notice_to_guests` is set to true. + +* `block_events_error`, if set, will block any attempts to send events + until the user consents to the privacy policy. The value of the setting is + used as the text of the error. + +* `require_at_registration`, if enabled, will add a step to the registration + process, similar to how captcha works. Users will be required to accept the + policy before their account is created. + +* `policy_name` is the display name of the policy users will see when registering + for an account. Has no effect unless `require_at_registration` is enabled. + Defaults to "Privacy Policy". + +Example configuration: +```yaml +user_consent: + template_dir: res/templates/privacy + version: 1.0 + server_notice_content: + msgtype: m.text + body: >- + To continue using this homeserver you must review and agree to the + terms and conditions at %(consent_uri)s + send_server_notice_to_guests: true + block_events_error: >- + To continue using this homeserver you must review and agree to the + terms and conditions at %(consent_uri)s + require_at_registration: false + policy_name: Privacy Policy +``` +--- +Config option: `stats` + +Settings for local room and user statistics collection. See [here](../../room_and_user_statistics.md) +for more. + +* `enabled`: Set to false to disable room and user statistics. Note that doing + so may cause certain features (such as the room directory) not to work + correctly. Defaults to true. + +Example configuration: +```yaml +stats: + enabled: false +``` +--- +Config option: `server_notices` + +Use this setting to enable a room which can be used to send notices +from the server to users. It is a special room which users cannot leave; notices +in the room come from a special "notices" user id. + +If you use this setting, you *must* define the `system_mxid_localpart` +sub-setting, which defines the id of the user which will be used to send the +notices. + +Sub-options for this setting include: +* `system_mxid_display_name`: set the display name of the "notices" user +* `system_mxid_avatar_url`: set the avatar for the "notices" user +* `room_name`: set the room name of the server notices room + +Example configuration: +```yaml +server_notices: + system_mxid_localpart: notices + system_mxid_display_name: "Server Notices" + system_mxid_avatar_url: "mxc://server.com/oumMVlgDnLYFaPVkExemNVVZ" + room_name: "Server Notices" +``` +--- +Config option: `enable_room_list_search` + +Set to false to disable searching the public room list. When disabled +blocks searching local and remote room lists for local and remote +users by always returning an empty list for all queries. Defaults to true. + +Example configuration: +```yaml +enable_room_list_search: false +``` +--- +Config option: `alias_creation` + +The `alias_creation` option controls who is allowed to create aliases +on this server. + +The format of this option is a list of rules that contain globs that +match against user_id, room_id and the new alias (fully qualified with +server name). The action in the first rule that matches is taken, +which can currently either be "allow" or "deny". + +Missing user_id/room_id/alias fields default to "*". + +If no rules match the request is denied. An empty list means no one +can create aliases. + +Options for the rules include: +* `user_id`: Matches against the creator of the alias. Defaults to "*". +* `alias`: Matches against the alias being created. Defaults to "*". +* `room_id`: Matches against the room ID the alias is being pointed at. Defaults to "*" +* `action`: Whether to "allow" or "deny" the request if the rule matches. Defaults to allow. + +Example configuration: +```yaml +alias_creation_rules: + - user_id: "bad_user" + alias: "spammy_alias" + room_id: "*" + action: deny +``` +--- +Config options: `room_list_publication_rules` + +The `room_list_publication_rules` option controls who can publish and +which rooms can be published in the public room list. + +The format of this option is the same as that for +`alias_creation_rules`. + +If the room has one or more aliases associated with it, only one of +the aliases needs to match the alias rule. If there are no aliases +then only rules with `alias: *` match. + +If no rules match the request is denied. An empty list means no one +can publish rooms. + +Options for the rules include: +* `user_id`: Matches against the creator of the alias. Defaults to "*". +* `alias`: Matches against any current local or canonical aliases associated with the room. Defaults to "*". +* `room_id`: Matches against the room ID being published. Defaults to "*". +* `action`: Whether to "allow" or "deny" the request if the rule matches. Defaults to allow. + +Example configuration: +```yaml +room_list_publication_rules: + - user_id: "*" + alias: "*" + room_id: "*" + action: allow +``` +--- +## Opentracing ## +Configuration options related to Opentracing support. + +--- +Config option: `opentracing` + +These settings enable and configure opentracing, which implements distributed tracing. +This allows you to observe the causal chains of events across servers +including requests, key lookups etc., across any server running +synapse or any other services which support opentracing +(specifically those implemented with Jaeger). + +Sub-options include: +* `enabled`: whether tracing is enabled. Set to true to enable. Disabled by default. +* `homeserver_whitelist`: The list of homeservers we wish to send and receive span contexts and span baggage. + See [here](../../opentracing.md) for more. + This is a list of regexes which are matched against the `server_name` of the homeserver. + By default, it is empty, so no servers are matched. +* `force_tracing_for_users`: # A list of the matrix IDs of users whose requests will always be traced, + even if the tracing system would otherwise drop the traces due to probabilistic sampling. + By default, the list is empty. +* `jaeger_config`: Jaeger can be configured to sample traces at different rates. + All configuration options provided by Jaeger can be set here. Jaeger's configuration is + mostly related to trace sampling which is documented [here](https://www.jaegertracing.io/docs/latest/sampling/). + +Example configuration: +```yaml +opentracing: + enabled: true + homeserver_whitelist: + - ".*" + force_tracing_for_users: + - "@user1:server_name" + - "@user2:server_name" + + jaeger_config: + sampler: + type: const + param: 1 + logging: + false +``` +--- +## Workers ## +Configuration options related to workers. + +--- +Config option: `send_federation` + +Controls sending of outbound federation transactions on the main process. +Set to false if using a federation sender worker. Defaults to true. + +Example configuration: +```yaml +send_federation: false +``` +--- +Config option: `federation_sender_instances` + +It is possible to run multiple federation sender workers, in which case the +work is balanced across them. Use this setting to list the senders. + +This configuration setting must be shared between all federation sender workers, and if +changed all federation sender workers must be stopped at the same time and then +started, to ensure that all instances are running with the same config (otherwise +events may be dropped). + +Example configuration: +```yaml +federation_sender_instances: + - federation_sender1 +``` +--- +Config option: `instance_map` + +When using workers this should be a map from worker name to the +HTTP replication listener of the worker, if configured. + +Example configuration: +```yaml +instance_map: + worker1: + host: localhost + port: 8034 +``` +--- +Config option: `stream_writers` + +Experimental: When using workers you can define which workers should +handle event persistence and typing notifications. Any worker +specified here must also be in the `instance_map`. + +Example configuration: +```yaml +stream_writers: + events: worker1 + typing: worker1 +``` +--- +Config option: `run_background_task_on` + +The worker that is used to run background tasks (e.g. cleaning up expired +data). If not provided this defaults to the main process. + +Example configuration: +```yaml +run_background_tasks_on: worker1 +``` +--- +Config option: `worker_replication_secret` + +A shared secret used by the replication APIs to authenticate HTTP requests +from workers. + +By default this is unused and traffic is not authenticated. + +Example configuration: +```yaml +worker_replication_secret: "secret_secret" +``` +Config option: `redis` + +Configuration for Redis when using workers. This *must* be enabled when +using workers (unless using old style direct TCP configuration). +This setting has the following sub-options: +* `enabled`: whether to use Redis support. Defaults to false. +* `host` and `port`: Optional host and port to use to connect to redis. Defaults to + localhost and 6379 +* `password`: Optional password if configured on the Redis instance. + +Example configuration: +```yaml +redis: + enabled: true + host: localhost + port: 6379 + password: +``` +## Background Updates ## +Configuration settings related to background updates. + +--- +Config option: `background_updates` + +Background updates are database updates that are run in the background in batches. +The duration, minimum batch size, default batch size, whether to sleep between batches and if so, how long to +sleep can all be configured. This is helpful to speed up or slow down the updates. +This setting has the following sub-options: +* `background_update_duration_ms`: How long in milliseconds to run a batch of background updates for. Defaults to 100. + Set a different time to change the default. +* `sleep_enabled`: Whether to sleep between updates. Defaults to true. Set to false to change the default. +* `sleep_duration_ms`: If sleeping between updates, how long in milliseconds to sleep for. Defaults to 1000. + Set a duration to change the default. +* `min_batch_size`: Minimum size a batch of background updates can be. Must be greater than 0. Defaults to 1. + Set a size to change the default. +* `default_batch_size`: The batch size to use for the first iteration of a new background update. The default is 100. + Set a size to change the default. + +Example configuration: +```yaml +background_updates: + background_update_duration_ms: 500 + sleep_enabled: false + sleep_duration_ms: 300 + min_batch_size: 10 + default_batch_size: 50 +``` \ No newline at end of file From f8d3ee95706100e982f30f4c04fc3df8de4d4212 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 18 Apr 2022 12:41:55 -0600 Subject: [PATCH 30/60] Fix grammatical error in error message (#12483) * Fix grammatical error in error message * changelog --- changelog.d/12483.misc | 1 + synapse/storage/databases/main/state.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/12483.misc diff --git a/changelog.d/12483.misc b/changelog.d/12483.misc new file mode 100644 index 0000000000..88c6e3e465 --- /dev/null +++ b/changelog.d/12483.misc @@ -0,0 +1 @@ +Fix grammatical error in federation error response when the room version of a room is unknown. diff --git a/synapse/storage/databases/main/state.py b/synapse/storage/databases/main/state.py index eba35f3700..7a1b013fa3 100644 --- a/synapse/storage/databases/main/state.py +++ b/synapse/storage/databases/main/state.py @@ -130,7 +130,7 @@ class StateGroupWorkerStore(EventsWorkerStore, SQLBaseStore): ) if room_version is None: - raise NotFoundError("Could not room_version for %s" % (room_id,)) + raise NotFoundError("Could not find room_version for %s" % (room_id,)) return room_version From 88fe72cc1eceb61d7bfd41c45fe250738f6693a7 Mon Sep 17 00:00:00 2001 From: "Olivier Wilkinson (reivilibre)" Date: Tue, 19 Apr 2022 11:00:37 +0100 Subject: [PATCH 31/60] 1.57.0 --- CHANGES.md | 10 ++++++++-- debian/changelog | 6 ++++++ pyproject.toml | 2 +- synapse/__init__.py | 2 +- 4 files changed, 16 insertions(+), 4 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 748f6833ca..a45f7a1fca 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,10 +1,16 @@ -Synapse 1.57.0rc1 (2022-04-12) -============================== +Synapse 1.57.0 (2022-04-19) +=========================== This version includes a [change](https://github.com/matrix-org/synapse/pull/12209) to the way transaction IDs are managed for application services. If your deployment uses a dedicated worker for application service traffic, **it must be stopped** when the database is upgraded (which normally happens when the main process is upgraded), to ensure the change is made safely without any risk of reusing transaction IDs. See the [upgrade notes](https://github.com/matrix-org/synapse/blob/develop/docs/upgrade.md#upgrading-to-v1570) for more details. +No significant changes since 1.57.0rc1. + + +Synapse 1.57.0rc1 (2022-04-12) +============================== + Features -------- diff --git a/debian/changelog b/debian/changelog index 67215aa080..71dcf9de8e 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.57.0) stable; urgency=medium + + * New synapse release 1.57.0. + + -- Synapse Packaging team Tue, 19 Apr 2022 10:58:42 +0100 + matrix-synapse-py3 (1.57.0~rc1) stable; urgency=medium * New synapse release 1.57.0~rc1. diff --git a/pyproject.toml b/pyproject.toml index 2a3637a7f0..7f58c37e3f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,7 +54,7 @@ skip_gitignore = true [tool.poetry] name = "matrix-synapse" -version = "1.56.0" +version = "1.57.0" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] license = "Apache-2.0" diff --git a/synapse/__init__.py b/synapse/__init__.py index 7cdd921366..bf88f1d93d 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -68,7 +68,7 @@ try: except ImportError: pass -__version__ = "1.57.0rc1" +__version__ = "1.57.0" if bool(os.environ.get("SYNAPSE_TEST_PATCH_LOG_CONTEXTS", False)): # We import here so that we don't have to install a bunch of deps when From 9f512ff53715287088daa34d8927826892a0d5e2 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 19 Apr 2022 11:25:12 +0100 Subject: [PATCH 32/60] fix typo in upgrade notes --- docs/upgrade.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/upgrade.md b/docs/upgrade.md index 55a6ff7cae..023872490e 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -96,7 +96,7 @@ worker for application service traffic, **it must be stopped** when the database without any risk of reusing transaction IDs. Deployments which do not use separate worker processes can be upgraded as normal. Similarly, -deployments where no applciation services are in use can be upgraded as normal. +deployments where no application services are in use can be upgraded as normal.
Recovering from an incorrect upgrade From dfc7646504cef3e4ff396c36089e1c6f1b1634de Mon Sep 17 00:00:00 2001 From: "Olivier Wilkinson (reivilibre)" Date: Tue, 19 Apr 2022 11:29:51 +0100 Subject: [PATCH 33/60] Link to specific version of upgrade notes --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index a45f7a1fca..74a5dbf424 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -3,7 +3,7 @@ Synapse 1.57.0 (2022-04-19) This version includes a [change](https://github.com/matrix-org/synapse/pull/12209) to the way transaction IDs are managed for application services. If your deployment uses a dedicated worker for application service traffic, **it must be stopped** when the database is upgraded (which normally happens when the main process is upgraded), to ensure the change is made safely without any risk of reusing transaction IDs. -See the [upgrade notes](https://github.com/matrix-org/synapse/blob/develop/docs/upgrade.md#upgrading-to-v1570) for more details. +See the [upgrade notes](https://github.com/matrix-org/synapse/blob/v1.57.0rc1/docs/upgrade.md#upgrading-to-v1570) for more details. No significant changes since 1.57.0rc1. From b121a3ad2b75f94002b085e1945110fc446a560a Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 19 Apr 2022 12:17:29 +0100 Subject: [PATCH 34/60] Back out implementation of MSC2314 (#12474) MSC2314 has now been closed, so we're backing out its implementation, which originally happened in #6176. Unfortunately it's not a direct revert, as that PR mixed in a bunch of unrelated changes to tests etc. --- changelog.d/12474.misc | 1 + synapse/federation/federation_server.py | 26 ++++-------- .../federation/transport/server/federation.py | 2 +- sytest-blacklist | 4 -- tests/federation/test_federation_server.py | 41 +------------------ 5 files changed, 13 insertions(+), 61 deletions(-) create mode 100644 changelog.d/12474.misc diff --git a/changelog.d/12474.misc b/changelog.d/12474.misc new file mode 100644 index 0000000000..5292108b39 --- /dev/null +++ b/changelog.d/12474.misc @@ -0,0 +1 @@ +Back out experimental implementation of [MSC2314](https://github.com/matrix-org/matrix-spec-proposals/pull/2314). diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index e67af6463f..beab1227b8 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -515,7 +515,7 @@ class FederationServer(FederationBase): ) async def on_room_state_request( - self, origin: str, room_id: str, event_id: Optional[str] + self, origin: str, room_id: str, event_id: str ) -> Tuple[int, JsonDict]: origin_host, _ = parse_server_name(origin) await self.check_server_matches_acl(origin_host, room_id) @@ -530,18 +530,13 @@ class FederationServer(FederationBase): # - but that's non-trivial to get right, and anyway somewhat defeats # the point of the linearizer. async with self._server_linearizer.queue((origin, room_id)): - resp: JsonDict = dict( - await self._state_resp_cache.wrap( - (room_id, event_id), - self._on_context_state_request_compute, - room_id, - event_id, - ) + resp = await self._state_resp_cache.wrap( + (room_id, event_id), + self._on_context_state_request_compute, + room_id, + event_id, ) - room_version = await self.store.get_room_version_id(room_id) - resp["room_version"] = room_version - return 200, resp async def on_state_ids_request( @@ -574,14 +569,11 @@ class FederationServer(FederationBase): return {"pdu_ids": state_ids, "auth_chain_ids": list(auth_chain_ids)} async def _on_context_state_request_compute( - self, room_id: str, event_id: Optional[str] + self, room_id: str, event_id: str ) -> Dict[str, list]: pdus: Collection[EventBase] - if event_id: - event_ids = await self.handler.get_state_ids_for_pdu(room_id, event_id) - pdus = await self.store.get_events_as_list(event_ids) - else: - pdus = (await self.state.get_current_state(room_id)).values() + event_ids = await self.handler.get_state_ids_for_pdu(room_id, event_id) + pdus = await self.store.get_events_as_list(event_ids) auth_chain = await self.store.get_auth_chain( room_id, [pdu.event_id for pdu in pdus] diff --git a/synapse/federation/transport/server/federation.py b/synapse/federation/transport/server/federation.py index aed3d5069c..6fbc7b5f15 100644 --- a/synapse/federation/transport/server/federation.py +++ b/synapse/federation/transport/server/federation.py @@ -160,7 +160,7 @@ class FederationStateV1Servlet(BaseFederationServerServlet): return await self.handler.on_room_state_request( origin, room_id, - parse_string_from_args(query, "event_id", None, required=False), + parse_string_from_args(query, "event_id", None, required=True), ) diff --git a/sytest-blacklist b/sytest-blacklist index 57e603a4a6..d5fa36cec7 100644 --- a/sytest-blacklist +++ b/sytest-blacklist @@ -21,10 +21,6 @@ Newly created users see their own presence in /initialSync (SYT-34) # Blacklisted due to https://github.com/matrix-org/synapse/issues/1396 Should reject keys claiming to belong to a different user -# Blacklisted due to https://github.com/matrix-org/matrix-doc/pull/2314 removing -# this requirement from the spec -Inbound federation of state requires event_id as a mandatory paramater - # Blacklisted until MSC2753 is implemented Local users can peek into world_readable rooms by room ID We can't peek into rooms with shared history_visibility diff --git a/tests/federation/test_federation_server.py b/tests/federation/test_federation_server.py index 30e7e5093a..b19365b81a 100644 --- a/tests/federation/test_federation_server.py +++ b/tests/federation/test_federation_server.py @@ -104,58 +104,21 @@ class ServerACLsTestCase(unittest.TestCase): class StateQueryTests(unittest.FederatingHomeserverTestCase): - servlets = [ admin.register_servlets, room.register_servlets, login.register_servlets, ] - def test_without_event_id(self): - """ - Querying v1/state/ without an event ID will return the current - known state. - """ - u1 = self.register_user("u1", "pass") - u1_token = self.login("u1", "pass") - - room_1 = self.helper.create_room_as(u1, tok=u1_token) - self.inject_room_member(room_1, "@user:other.example.com", "join") - - channel = self.make_signed_federation_request( - "GET", "/_matrix/federation/v1/state/%s" % (room_1,) - ) - self.assertEqual(200, channel.code, channel.result) - - self.assertEqual( - channel.json_body["room_version"], - self.hs.config.server.default_room_version.identifier, - ) - - members = set( - map( - lambda x: x["state_key"], - filter( - lambda x: x["type"] == "m.room.member", channel.json_body["pdus"] - ), - ) - ) - - self.assertEqual(members, {"@user:other.example.com", u1}) - self.assertEqual(len(channel.json_body["pdus"]), 6) - def test_needs_to_be_in_room(self): - """ - Querying v1/state/ requires the server - be in the room to provide data. - """ + """/v1/state/ requires the server to be in the room""" u1 = self.register_user("u1", "pass") u1_token = self.login("u1", "pass") room_1 = self.helper.create_room_as(u1, tok=u1_token) channel = self.make_signed_federation_request( - "GET", "/_matrix/federation/v1/state/%s" % (room_1,) + "GET", "/_matrix/federation/v1/state/%s?event_id=xyz" % (room_1,) ) self.assertEqual(403, channel.code, channel.result) self.assertEqual(channel.json_body["errcode"], "M_FORBIDDEN") From dbe016e2585e2761e3c5fa63c4857a5ef8ef580f Mon Sep 17 00:00:00 2001 From: "Olivier Wilkinson (reivilibre)" Date: Tue, 19 Apr 2022 13:33:36 +0100 Subject: [PATCH 35/60] Remove 'Non-maintainer upload' line from Debian changelog --- debian/changelog | 1 - 1 file changed, 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index a44f7d98ff..b45f6a77a9 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,6 +1,5 @@ matrix-synapse-py3 (1.58.0+nmu1) UNRELEASED; urgency=medium - * Non-maintainer upload. * Use poetry to manage the bundled virtualenv included with this package. -- Synapse Packaging Team Wed, 30 Mar 2022 12:21:43 +0100 From 7dec4ce7e4bc9c330e5ec4c31be475156c4539e1 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 19 Apr 2022 13:45:27 +0100 Subject: [PATCH 36/60] fix typo in debian changelog --- debian/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index b45f6a77a9..a46bb2bba3 100644 --- a/debian/changelog +++ b/debian/changelog @@ -2,7 +2,7 @@ matrix-synapse-py3 (1.58.0+nmu1) UNRELEASED; urgency=medium * Use poetry to manage the bundled virtualenv included with this package. - -- Synapse Packaging Team Wed, 30 Mar 2022 12:21:43 +0100 + -- Synapse Packaging team Wed, 30 Mar 2022 12:21:43 +0100 matrix-synapse-py3 (1.57.0) stable; urgency=medium From fbdee86004a9b2e7f52be584b7b62efa434fc8be Mon Sep 17 00:00:00 2001 From: Dirk Klimpel <5740567+dklimpel@users.noreply.github.com> Date: Tue, 19 Apr 2022 15:00:41 +0200 Subject: [PATCH 37/60] Fix a link in `README.rst` (#12495) * Fix a link in `README.rst` * newsfile --- README.rst | 2 +- changelog.d/12495.doc | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/12495.doc diff --git a/README.rst b/README.rst index 595fb5ff62..8a14401d65 100644 --- a/README.rst +++ b/README.rst @@ -312,7 +312,7 @@ We recommend using the demo which starts 3 federated instances running on ports (to stop, you can use `./demo/stop.sh`) -See the [demo documentation](https://matrix-org.github.io/synapse/develop/development/demo.html) +See the `demo documentation `_ for more information. If you just want to start a single instance of the app and run it directly:: diff --git a/changelog.d/12495.doc b/changelog.d/12495.doc new file mode 100644 index 0000000000..afa0111675 --- /dev/null +++ b/changelog.d/12495.doc @@ -0,0 +1 @@ +Fix a broken link in `README.rst`. From a1f87f57ff7b5971e0e3450ec7761cf8dc4e01d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Christian=20Gr=C3=BCnhage?= Date: Tue, 19 Apr 2022 17:23:53 +0200 Subject: [PATCH 38/60] Implement MSC3383: include destination in X-Matrix auth header (#11398) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Jan Christian Grünhage Co-authored-by: Marcus Hoffmann --- changelog.d/11398.feature | 1 + scripts-dev/federation_client.py | 7 +++- synapse/federation/transport/server/_base.py | 39 ++++++++++++++++---- synapse/http/matrixfederationclient.py | 12 +++++- 4 files changed, 48 insertions(+), 11 deletions(-) create mode 100644 changelog.d/11398.feature diff --git a/changelog.d/11398.feature b/changelog.d/11398.feature new file mode 100644 index 0000000000..a910f4da14 --- /dev/null +++ b/changelog.d/11398.feature @@ -0,0 +1 @@ +Implement [MSC3383](https://github.com/matrix-org/matrix-spec-proposals/pull/3383) for including the destination in server-to-server authentication headers. Contributed by @Bubu and @jcgruenhage for Famedly GmbH. diff --git a/scripts-dev/federation_client.py b/scripts-dev/federation_client.py index c72e19f61d..079d2f5ed0 100755 --- a/scripts-dev/federation_client.py +++ b/scripts-dev/federation_client.py @@ -124,7 +124,12 @@ def request( authorization_headers = [] for key, sig in signed_json["signatures"][origin_name].items(): - header = 'X-Matrix origin=%s,key="%s",sig="%s"' % (origin_name, key, sig) + header = 'X-Matrix origin=%s,key="%s",sig="%s",destination="%s"' % ( + origin_name, + key, + sig, + destination, + ) authorization_headers.append(header.encode("ascii")) print("Authorization: %s" % header, file=sys.stderr) diff --git a/synapse/federation/transport/server/_base.py b/synapse/federation/transport/server/_base.py index 2529dee613..d629a3ecb5 100644 --- a/synapse/federation/transport/server/_base.py +++ b/synapse/federation/transport/server/_base.py @@ -16,7 +16,8 @@ import functools import logging import re import time -from typing import TYPE_CHECKING, Any, Awaitable, Callable, Optional, Tuple, cast +from http import HTTPStatus +from typing import TYPE_CHECKING, Any, Awaitable, Callable, Dict, Optional, Tuple, cast from synapse.api.errors import Codes, FederationDeniedError, SynapseError from synapse.api.urls import FEDERATION_V1_PREFIX @@ -86,15 +87,24 @@ class Authenticator: if not auth_headers: raise NoAuthenticationError( - 401, "Missing Authorization headers", Codes.UNAUTHORIZED + HTTPStatus.UNAUTHORIZED, + "Missing Authorization headers", + Codes.UNAUTHORIZED, ) for auth in auth_headers: if auth.startswith(b"X-Matrix"): - (origin, key, sig) = _parse_auth_header(auth) + (origin, key, sig, destination) = _parse_auth_header(auth) json_request["origin"] = origin json_request["signatures"].setdefault(origin, {})[key] = sig + # if the origin_server sent a destination along it needs to match our own server_name + if destination is not None and destination != self.server_name: + raise AuthenticationError( + HTTPStatus.UNAUTHORIZED, + "Destination mismatch in auth header", + Codes.UNAUTHORIZED, + ) if ( self.federation_domain_whitelist is not None and origin not in self.federation_domain_whitelist @@ -103,7 +113,9 @@ class Authenticator: if origin is None or not json_request["signatures"]: raise NoAuthenticationError( - 401, "Missing Authorization headers", Codes.UNAUTHORIZED + HTTPStatus.UNAUTHORIZED, + "Missing Authorization headers", + Codes.UNAUTHORIZED, ) await self.keyring.verify_json_for_server( @@ -142,13 +154,14 @@ class Authenticator: logger.exception("Error resetting retry timings on %s", origin) -def _parse_auth_header(header_bytes: bytes) -> Tuple[str, str, str]: +def _parse_auth_header(header_bytes: bytes) -> Tuple[str, str, str, Optional[str]]: """Parse an X-Matrix auth header Args: header_bytes: header value Returns: + origin, key id, signature, destination. origin, key id, signature. Raises: @@ -157,7 +170,9 @@ def _parse_auth_header(header_bytes: bytes) -> Tuple[str, str, str]: try: header_str = header_bytes.decode("utf-8") params = header_str.split(" ")[1].split(",") - param_dict = {k: v for k, v in (kv.split("=", maxsplit=1) for kv in params)} + param_dict: Dict[str, str] = { + k: v for k, v in [param.split("=", maxsplit=1) for param in params] + } def strip_quotes(value: str) -> str: if value.startswith('"'): @@ -172,7 +187,15 @@ def _parse_auth_header(header_bytes: bytes) -> Tuple[str, str, str]: key = strip_quotes(param_dict["key"]) sig = strip_quotes(param_dict["sig"]) - return origin, key, sig + + # get the destination server_name from the auth header if it exists + destination = param_dict.get("destination") + if destination is not None: + destination = strip_quotes(destination) + else: + destination = None + + return origin, key, sig, destination except Exception as e: logger.warning( "Error parsing auth header '%s': %s", @@ -180,7 +203,7 @@ def _parse_auth_header(header_bytes: bytes) -> Tuple[str, str, str]: e, ) raise AuthenticationError( - 400, "Malformed Authorization header", Codes.UNAUTHORIZED + HTTPStatus.BAD_REQUEST, "Malformed Authorization header", Codes.UNAUTHORIZED ) diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 5097b3ca57..e686445955 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -704,6 +704,9 @@ class MatrixFederationHttpClient: Returns: A list of headers to be added as "Authorization:" headers """ + if destination is None and destination_is is None: + raise ValueError("destination and destination_is cannot both be None!") + request: JsonDict = { "method": method.decode("ascii"), "uri": url_bytes.decode("ascii"), @@ -726,8 +729,13 @@ class MatrixFederationHttpClient: for key, sig in request["signatures"][self.server_name].items(): auth_headers.append( ( - 'X-Matrix origin=%s,key="%s",sig="%s"' - % (self.server_name, key, sig) + 'X-Matrix origin=%s,key="%s",sig="%s",destination="%s"' + % ( + self.server_name, + key, + sig, + request.get("destination") or request["destination_is"], + ) ).encode("ascii") ) return auth_headers From 798deb3a10e59cfa7b46994417f0c56c4a0bc9f7 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 19 Apr 2022 16:41:52 +0100 Subject: [PATCH 39/60] Fix typo in deb changelogs from release script (#12497) The release script used to incorrectly write `New synapse release 1.57.0~rc1.` instead of `New synapse release 1.57.0rc1.` --- changelog.d/12497.misc | 1 + scripts-dev/release.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/12497.misc diff --git a/changelog.d/12497.misc b/changelog.d/12497.misc new file mode 100644 index 0000000000..17a661ec61 --- /dev/null +++ b/changelog.d/12497.misc @@ -0,0 +1 @@ +Fix a minor typo in the Debian changelogs generated by the release script. diff --git a/scripts-dev/release.py b/scripts-dev/release.py index 518eaf417c..6f7cf6888d 100755 --- a/scripts-dev/release.py +++ b/scripts-dev/release.py @@ -230,7 +230,7 @@ def prepare(): debian_version = new_version run_until_successful( - f'dch -M -v {debian_version} "New synapse release {debian_version}."', + f'dch -M -v {debian_version} "New Synapse release {new_version}."', shell=True, ) run_until_successful('dch -M -r -D stable ""', shell=True) From b80bb7e4526bd0425b3cdd10fa5633a5a4b4e05f Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 19 Apr 2022 16:42:19 +0100 Subject: [PATCH 40/60] Fix `/room/.../event/...` to return the *original* event after any edits (#12476) This is what the MSC (now) requires. Fixes https://github.com/matrix-org/synapse/issues/10310. --- changelog.d/12476.bugfix | 1 + synapse/events/utils.py | 21 ++++--- synapse/rest/client/room.py | 4 +- tests/rest/client/test_relations.py | 92 +++++++++++++++++++---------- 4 files changed, 79 insertions(+), 39 deletions(-) create mode 100644 changelog.d/12476.bugfix diff --git a/changelog.d/12476.bugfix b/changelog.d/12476.bugfix new file mode 100644 index 0000000000..9ad6a71abd --- /dev/null +++ b/changelog.d/12476.bugfix @@ -0,0 +1 @@ +Fix a long-standing bug which incorrectly caused `GET /_matrix/client/r3/rooms/{roomId}/event/{eventId}` to return edited events rather than the original. diff --git a/synapse/events/utils.py b/synapse/events/utils.py index 43c3241fb0..2174b4a094 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -402,6 +402,7 @@ class EventClientSerializer: *, config: SerializeEventConfig = _DEFAULT_SERIALIZE_EVENT_CONFIG, bundle_aggregations: Optional[Dict[str, "BundledAggregations"]] = None, + apply_edits: bool = True, ) -> JsonDict: """Serializes a single event. @@ -409,10 +410,10 @@ class EventClientSerializer: event: The event being serialized. time_now: The current time in milliseconds config: Event serialization config - bundle_aggregations: Whether to include the bundled aggregations for this - event. Only applies to non-state events. (State events never include - bundled aggregations.) - + bundle_aggregations: A map from event_id to the aggregations to be bundled + into the event. + apply_edits: Whether the content of the event should be modified to reflect + any replacement in `bundle_aggregations[].replace`. Returns: The serialized event """ @@ -430,8 +431,9 @@ class EventClientSerializer: event, time_now, config, - bundle_aggregations[event.event_id], + event_aggregations, serialized_event, + apply_edits=apply_edits, ) return serialized_event @@ -470,6 +472,7 @@ class EventClientSerializer: config: SerializeEventConfig, aggregations: "BundledAggregations", serialized_event: JsonDict, + apply_edits: bool, ) -> None: """Potentially injects bundled aggregations into the unsigned portion of the serialized event. @@ -479,7 +482,8 @@ class EventClientSerializer: aggregations: The bundled aggregation to serialize. serialized_event: The serialized event which may be modified. config: Event serialization config - + apply_edits: Whether the content of the event should be modified to reflect + any replacement in `aggregations.replace`. """ serialized_aggregations = {} @@ -490,9 +494,10 @@ class EventClientSerializer: serialized_aggregations[RelationTypes.REFERENCE] = aggregations.references if aggregations.replace: - # If there is an edit, apply it to the event. + # If there is an edit, optionally apply it to the event. edit = aggregations.replace - self._apply_edit(event, serialized_event, edit) + if apply_edits: + self._apply_edit(event, serialized_event, edit) # Include information about it in the relations dict. serialized_aggregations[RelationTypes.REPLACE] = { diff --git a/synapse/rest/client/room.py b/synapse/rest/client/room.py index 47e152c8cc..937c323176 100644 --- a/synapse/rest/client/room.py +++ b/synapse/rest/client/room.py @@ -669,8 +669,10 @@ class RoomEventServlet(RestServlet): ) time_now = self.clock.time_msec() + # per MSC2676, /rooms/{roomId}/event/{eventId}, should return the + # *original* event, rather than the edited version event_dict = self._event_serializer.serialize_event( - event, time_now, bundle_aggregations=aggregations + event, time_now, bundle_aggregations=aggregations, apply_edits=False ) return 200, event_dict diff --git a/tests/rest/client/test_relations.py b/tests/rest/client/test_relations.py index 6fabada8b3..57d97bbb47 100644 --- a/tests/rest/client/test_relations.py +++ b/tests/rest/client/test_relations.py @@ -380,13 +380,16 @@ class RelationsTestCase(BaseRelationsTestCase): {"event_id": edit_event_id, "sender": self.user_id}, m_replace_dict ) + # /event should return the *original* event channel = self.make_request( "GET", f"/rooms/{self.room}/event/{self.parent_id}", access_token=self.user_token, ) self.assertEqual(200, channel.code, channel.json_body) - self.assertEqual(channel.json_body["content"], new_body) + self.assertEqual( + channel.json_body["content"], {"body": "Hi!", "msgtype": "m.text"} + ) assert_bundle(channel.json_body) # Request the room messages. @@ -399,6 +402,7 @@ class RelationsTestCase(BaseRelationsTestCase): assert_bundle(self._find_event_in_chunk(channel.json_body["chunk"])) # Request the room context. + # /context should return the edited event. channel = self.make_request( "GET", f"/rooms/{self.room}/context/{self.parent_id}", @@ -406,6 +410,7 @@ class RelationsTestCase(BaseRelationsTestCase): ) self.assertEqual(200, channel.code, channel.json_body) assert_bundle(channel.json_body["event"]) + self.assertEqual(channel.json_body["event"]["content"], new_body) # Request sync, but limit the timeline so it becomes limited (and includes # bundled aggregations). @@ -470,14 +475,14 @@ class RelationsTestCase(BaseRelationsTestCase): channel = self.make_request( "GET", - f"/rooms/{self.room}/event/{self.parent_id}", + f"/rooms/{self.room}/context/{self.parent_id}", access_token=self.user_token, ) self.assertEqual(200, channel.code, channel.json_body) - self.assertEqual(channel.json_body["content"], new_body) + self.assertEqual(channel.json_body["event"]["content"], new_body) - relations_dict = channel.json_body["unsigned"].get("m.relations") + relations_dict = channel.json_body["event"]["unsigned"].get("m.relations") self.assertIn(RelationTypes.REPLACE, relations_dict) m_replace_dict = relations_dict[RelationTypes.REPLACE] @@ -492,10 +497,9 @@ class RelationsTestCase(BaseRelationsTestCase): """Test that editing a reply works.""" # Create a reply to edit. + original_body = {"msgtype": "m.text", "body": "A reply!"} channel = self._send_relation( - RelationTypes.REFERENCE, - "m.room.message", - content={"msgtype": "m.text", "body": "A reply!"}, + RelationTypes.REFERENCE, "m.room.message", content=original_body ) reply = channel.json_body["event_id"] @@ -508,38 +512,54 @@ class RelationsTestCase(BaseRelationsTestCase): ) edit_event_id = channel.json_body["event_id"] + # /event returns the original event channel = self.make_request( "GET", f"/rooms/{self.room}/event/{reply}", access_token=self.user_token, ) self.assertEqual(200, channel.code, channel.json_body) + event_result = channel.json_body + self.assertDictContainsSubset(original_body, event_result["content"]) - # We expect to see the new body in the dict, as well as the reference - # metadata sill intact. - self.assertDictContainsSubset(new_body, channel.json_body["content"]) - self.assertDictContainsSubset( - { - "m.relates_to": { - "event_id": self.parent_id, - "rel_type": "m.reference", - } - }, - channel.json_body["content"], + # also check /context, which returns the *edited* event + channel = self.make_request( + "GET", + f"/rooms/{self.room}/context/{reply}", + access_token=self.user_token, ) + self.assertEqual(200, channel.code, channel.json_body) + context_result = channel.json_body["event"] - # We expect that the edit relation appears in the unsigned relations - # section. - relations_dict = channel.json_body["unsigned"].get("m.relations") - self.assertIn(RelationTypes.REPLACE, relations_dict) + # check that the relations are correct for both APIs + for result_event_dict, desc in ( + (event_result, "/event"), + (context_result, "/context"), + ): + # The reference metadata should still be intact. + self.assertDictContainsSubset( + { + "m.relates_to": { + "event_id": self.parent_id, + "rel_type": "m.reference", + } + }, + result_event_dict["content"], + desc, + ) - m_replace_dict = relations_dict[RelationTypes.REPLACE] - for key in ["event_id", "sender", "origin_server_ts"]: - self.assertIn(key, m_replace_dict) + # We expect that the edit relation appears in the unsigned relations + # section. + relations_dict = result_event_dict["unsigned"].get("m.relations") + self.assertIn(RelationTypes.REPLACE, relations_dict, desc) - self.assert_dict( - {"event_id": edit_event_id, "sender": self.user_id}, m_replace_dict - ) + m_replace_dict = relations_dict[RelationTypes.REPLACE] + for key in ["event_id", "sender", "origin_server_ts"]: + self.assertIn(key, m_replace_dict, desc) + + self.assert_dict( + {"event_id": edit_event_id, "sender": self.user_id}, m_replace_dict + ) def test_edit_thread(self) -> None: """Test that editing a thread works.""" @@ -605,19 +625,31 @@ class RelationsTestCase(BaseRelationsTestCase): ) # Request the original event. + # /event should return the original event. channel = self.make_request( "GET", f"/rooms/{self.room}/event/{self.parent_id}", access_token=self.user_token, ) self.assertEqual(200, channel.code, channel.json_body) - # The edit to the edit should be ignored. - self.assertEqual(channel.json_body["content"], new_body) + self.assertEqual( + channel.json_body["content"], {"body": "Hi!", "msgtype": "m.text"} + ) # The relations information should not include the edit to the edit. relations_dict = channel.json_body["unsigned"].get("m.relations") self.assertIn(RelationTypes.REPLACE, relations_dict) + # /context should return the event updated for the *first* edit + # (The edit to the edit should be ignored.) + channel = self.make_request( + "GET", + f"/rooms/{self.room}/context/{self.parent_id}", + access_token=self.user_token, + ) + self.assertEqual(200, channel.code, channel.json_body) + self.assertEqual(channel.json_body["event"]["content"], new_body) + m_replace_dict = relations_dict[RelationTypes.REPLACE] for key in ["event_id", "sender", "origin_server_ts"]: self.assertIn(key, m_replace_dict) From c1482a352acd35055db93f5d0b5de1bec8974fa0 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 19 Apr 2022 16:49:45 +0100 Subject: [PATCH 41/60] Fix returned count of delete extremities admin API (#12496) --- changelog.d/12496.bugfix | 1 + .../storage/databases/main/events_forward_extremities.py | 8 +++++--- 2 files changed, 6 insertions(+), 3 deletions(-) create mode 100644 changelog.d/12496.bugfix diff --git a/changelog.d/12496.bugfix b/changelog.d/12496.bugfix new file mode 100644 index 0000000000..a68df7c96a --- /dev/null +++ b/changelog.d/12496.bugfix @@ -0,0 +1 @@ +Fix bug where the admin API for [deleting forward extremities](https://github.com/matrix-org/synapse/blob/erikj/fix_delete_event_response_count/docs/admin_api/rooms.md#deleting-forward-extremities) would always return a count of 1 no matter how many extremities were deleted. Broke in v1.27.0. diff --git a/synapse/storage/databases/main/events_forward_extremities.py b/synapse/storage/databases/main/events_forward_extremities.py index 68901b4335..f851bff604 100644 --- a/synapse/storage/databases/main/events_forward_extremities.py +++ b/synapse/storage/databases/main/events_forward_extremities.py @@ -66,13 +66,15 @@ class EventForwardExtremitiesStore( """ txn.execute(sql, (event_id, room_id)) + + deleted_count = txn.rowcount logger.info( "Deleted %s extra forward extremities for room %s", - txn.rowcount, + deleted_count, room_id, ) - if txn.rowcount > 0: + if deleted_count > 0: # Invalidate the cache self._invalidate_cache_and_stream( txn, @@ -80,7 +82,7 @@ class EventForwardExtremitiesStore( (room_id,), ) - return txn.rowcount + return deleted_count return await self.db_pool.runInteraction( "delete_forward_extremities_for_room", From eed38c502760989026fc826ce49205c02636cebd Mon Sep 17 00:00:00 2001 From: David Robertson Date: Wed, 20 Apr 2022 12:48:44 +0100 Subject: [PATCH 42/60] Add CI job to act as a canary for testing against latest dependencies (#12472) Co-authored-by: Patrick Cloke --- ...latest_deps_build_failed_issue_template.md | 4 + .github/workflows/latest_deps.yml | 156 ++++++++++++++++++ changelog.d/12472.misc | 1 + 3 files changed, 161 insertions(+) create mode 100644 .ci/latest_deps_build_failed_issue_template.md create mode 100644 .github/workflows/latest_deps.yml create mode 100644 changelog.d/12472.misc diff --git a/.ci/latest_deps_build_failed_issue_template.md b/.ci/latest_deps_build_failed_issue_template.md new file mode 100644 index 0000000000..0525402503 --- /dev/null +++ b/.ci/latest_deps_build_failed_issue_template.md @@ -0,0 +1,4 @@ +--- +title: CI run against latest deps is failing +--- +See https://github.com/{{env.GITHUB_REPOSITORY}}/actions/runs/{{env.GITHUB_RUN_ID}} diff --git a/.github/workflows/latest_deps.yml b/.github/workflows/latest_deps.yml new file mode 100644 index 0000000000..1a61d179d9 --- /dev/null +++ b/.github/workflows/latest_deps.yml @@ -0,0 +1,156 @@ +# People who are freshly `pip install`ing from PyPI will pull in the latest versions of +# dependencies which match the broad requirements. Since most CI runs are against +# the locked poetry environment, run specifically against the latest dependencies to +# know if there's an upcoming breaking change. +# +# As an overview this workflow: +# - checks out develop, +# - installs from source, pulling in the dependencies like a fresh `pip install` would, and +# - runs mypy and test suites in that checkout. +# +# Based on the twisted trunk CI job. + +name: Latest dependencies + +on: + schedule: + - cron: 0 7 * * * + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + mypy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + # The dev dependencies aren't exposed in the wheel metadata (at least with current + # poetry-core versions), so we install with poetry. + - uses: matrix-org/setup-python-poetry@v1 + with: + python-version: "3.x" + poetry-version: "1.2.0b1" + # Dump installed versions for debugging. + - run: poetry run pip list > before.txt + # Upgrade all runtime dependencies only. This is intended to mimic a fresh + # `pip install matrix-synapse[all]` as closely as possible. + - run: poetry update --no-dev + - run: poetry run pip list > after.txt && (diff -u before.txt after.txt || true) + - run: poetry run mypy + trial: + runs-on: ubuntu-latest + strategy: + matrix: + include: + - database: "sqlite" + - database: "postgres" + postgres-version: "14" + + steps: + - uses: actions/checkout@v2 + - run: sudo apt-get -qq install xmlsec1 + - name: Set up PostgreSQL ${{ matrix.postgres-version }} + if: ${{ matrix.postgres-version }} + run: | + docker run -d -p 5432:5432 \ + -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_INITDB_ARGS="--lc-collate C --lc-ctype C --encoding UTF8" \ + postgres:${{ matrix.postgres-version }} + - uses: actions/setup-python@v2 + with: + python-version: "3.x" + - run: pip install .[all,test] + - name: Await PostgreSQL + if: ${{ matrix.postgres-version }} + timeout-minutes: 2 + run: until pg_isready -h localhost; do sleep 1; done + - run: python -m twisted.trial --jobs=2 tests + env: + SYNAPSE_POSTGRES: ${{ matrix.database == 'postgres' || '' }} + SYNAPSE_POSTGRES_HOST: localhost + SYNAPSE_POSTGRES_USER: postgres + SYNAPSE_POSTGRES_PASSWORD: postgres + - name: Dump logs + # Logs are most useful when the command fails, always include them. + if: ${{ always() }} + # Note: Dumps to workflow logs instead of using actions/upload-artifact + # This keeps logs colocated with failing jobs + # It also ignores find's exit code; this is a best effort affair + run: >- + find _trial_temp -name '*.log' + -exec echo "::group::{}" \; + -exec cat {} \; + -exec echo "::endgroup::" \; + || true + + + sytest: + runs-on: ubuntu-latest + container: + image: matrixdotorg/sytest-synapse:testing + volumes: + - ${{ github.workspace }}:/src + strategy: + fail-fast: false + matrix: + include: + - sytest-tag: focal + + - sytest-tag: focal + postgres: postgres + workers: workers + redis: redis + env: + POSTGRES: ${{ matrix.postgres && 1}} + WORKERS: ${{ matrix.workers && 1 }} + REDIS: ${{ matrix.redis && 1 }} + BLACKLIST: ${{ matrix.workers && 'synapse-blacklist-with-workers' }} + + steps: + - uses: actions/checkout@v2 + - name: Ensure sytest runs `pip install` + # Delete the lockfile so sytest will `pip install` rather than `poetry install` + run: rm /src/poetry.lock + working-directory: /src + - name: Prepare test blacklist + run: cat sytest-blacklist .ci/worker-blacklist > synapse-blacklist-with-workers + - name: Run SyTest + run: /bootstrap.sh synapse + working-directory: /src + - name: Summarise results.tap + if: ${{ always() }} + run: /sytest/scripts/tap_to_gha.pl /logs/results.tap + - name: Upload SyTest logs + uses: actions/upload-artifact@v2 + if: ${{ always() }} + with: + name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.*, ', ') }}) + path: | + /logs/results.tap + /logs/**/*.log* + + + # TODO: run complement (as with twisted trunk, see #12473). + + # open an issue if the build fails, so we know about it. + open-issue: + if: failure() + needs: + # TODO: should mypy be included here? It feels more brittle than the other two. + - mypy + - trial + - sytest + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - uses: JasonEtco/create-an-issue@5d9504915f79f9cc6d791934b8ef34f2353dd74d # v2.5.0, 2020-12-06 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + update_existing: true + filename: .ci/latest_deps_build_failed_issue_template.md + diff --git a/changelog.d/12472.misc b/changelog.d/12472.misc new file mode 100644 index 0000000000..ed306209cc --- /dev/null +++ b/changelog.d/12472.misc @@ -0,0 +1 @@ +Add a CI job which tests Synapse against the latest version of all dependencies. From 4bc8cb4669ddeb719a3a6de39b093fc3be8db6fe Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 20 Apr 2022 14:57:39 +0300 Subject: [PATCH 43/60] Implement MSC2815: allow room moderators to view redacted event content (#12427) Implements matrix-org/matrix-spec-proposals#2815 Signed-off-by: Tulir Asokan --- changelog.d/12427.feature | 1 + synapse/api/errors.py | 18 ++++++++ synapse/config/experimental.py | 3 ++ synapse/handlers/events.py | 15 +++++- synapse/rest/client/room.py | 46 ++++++++++++++++++- synapse/rest/client/versions.py | 2 + .../storage/databases/main/events_worker.py | 18 ++++++++ 7 files changed, 100 insertions(+), 3 deletions(-) create mode 100644 changelog.d/12427.feature diff --git a/changelog.d/12427.feature b/changelog.d/12427.feature new file mode 100644 index 0000000000..e6913c8c09 --- /dev/null +++ b/changelog.d/12427.feature @@ -0,0 +1 @@ +Implement [MSC2815](https://github.com/matrix-org/matrix-spec-proposals/pull/2815) to allow room moderators to view redacted event content. Contributed by @tulir. diff --git a/synapse/api/errors.py b/synapse/api/errors.py index e92db29f6d..cb3b7323d5 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -79,6 +79,8 @@ class Codes: UNABLE_AUTHORISE_JOIN = "M_UNABLE_TO_AUTHORISE_JOIN" UNABLE_TO_GRANT_JOIN = "M_UNABLE_TO_GRANT_JOIN" + UNREDACTED_CONTENT_DELETED = "FI.MAU.MSC2815_UNREDACTED_CONTENT_DELETED" + class CodeMessageException(RuntimeError): """An exception with integer code and message string attributes. @@ -483,6 +485,22 @@ class RequestSendFailed(RuntimeError): self.can_retry = can_retry +class UnredactedContentDeletedError(SynapseError): + def __init__(self, content_keep_ms: Optional[int] = None): + super().__init__( + 404, + "The content for that event has already been erased from the database", + errcode=Codes.UNREDACTED_CONTENT_DELETED, + ) + self.content_keep_ms = content_keep_ms + + def error_dict(self) -> "JsonDict": + extra = {} + if self.content_keep_ms is not None: + extra = {"fi.mau.msc2815.content_keep_ms": self.content_keep_ms} + return cs_error(self.msg, self.errcode, **extra) + + def cs_error(msg: str, code: str = Codes.UNKNOWN, **kwargs: Any) -> "JsonDict": """Utility method for constructing an error response for client-server interactions. diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py index 979059e723..421ed7481b 100644 --- a/synapse/config/experimental.py +++ b/synapse/config/experimental.py @@ -78,3 +78,6 @@ class ExperimentalConfig(Config): # MSC2654: Unread counts self.msc2654_enabled: bool = experimental.get("msc2654_enabled", False) + + # MSC2815 (allow room moderators to view redacted event content) + self.msc2815_enabled: bool = experimental.get("msc2815_enabled", False) diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py index e89c4df314..5b94b00bc3 100644 --- a/synapse/handlers/events.py +++ b/synapse/handlers/events.py @@ -21,6 +21,7 @@ from synapse.api.errors import AuthError, SynapseError from synapse.events import EventBase from synapse.events.utils import SerializeEventConfig from synapse.handlers.presence import format_user_presence_state +from synapse.storage.databases.main.events_worker import EventRedactBehaviour from synapse.streams.config import PaginationConfig from synapse.types import JsonDict, UserID from synapse.visibility import filter_events_for_client @@ -141,7 +142,11 @@ class EventHandler: self.storage = hs.get_storage() async def get_event( - self, user: UserID, room_id: Optional[str], event_id: str + self, + user: UserID, + room_id: Optional[str], + event_id: str, + show_redacted: bool = False, ) -> Optional[EventBase]: """Retrieve a single specified event. @@ -150,6 +155,7 @@ class EventHandler: room_id: The expected room id. We'll return None if the event's room does not match. event_id: The event ID to obtain. + show_redacted: Should the full content of redacted events be returned? Returns: An event, or None if there is no event matching this ID. Raises: @@ -157,7 +163,12 @@ class EventHandler: AuthError if the user does not have the rights to inspect this event. """ - event = await self.store.get_event(event_id, check_room_id=room_id) + redact_behaviour = ( + EventRedactBehaviour.AS_IS if show_redacted else EventRedactBehaviour.REDACT + ) + event = await self.store.get_event( + event_id, check_room_id=room_id, redact_behaviour=redact_behaviour + ) if not event: return None diff --git a/synapse/rest/client/room.py b/synapse/rest/client/room.py index 937c323176..906fe09e97 100644 --- a/synapse/rest/client/room.py +++ b/synapse/rest/client/room.py @@ -21,6 +21,7 @@ from urllib import parse as urlparse from twisted.web.server import Request +from synapse import event_auth from synapse.api.constants import EventTypes, Membership from synapse.api.errors import ( AuthError, @@ -29,6 +30,7 @@ from synapse.api.errors import ( MissingClientTokenError, ShadowBanError, SynapseError, + UnredactedContentDeletedError, ) from synapse.api.filtering import Filter from synapse.events.utils import format_event_for_client_v2 @@ -643,18 +645,55 @@ class RoomEventServlet(RestServlet): super().__init__() self.clock = hs.get_clock() self._store = hs.get_datastores().main + self._state = hs.get_state_handler() self.event_handler = hs.get_event_handler() self._event_serializer = hs.get_event_client_serializer() self._relations_handler = hs.get_relations_handler() self.auth = hs.get_auth() + self.content_keep_ms = hs.config.server.redaction_retention_period + self.msc2815_enabled = hs.config.experimental.msc2815_enabled async def on_GET( self, request: SynapseRequest, room_id: str, event_id: str ) -> Tuple[int, JsonDict]: requester = await self.auth.get_user_by_req(request, allow_guest=True) + + include_unredacted_content = self.msc2815_enabled and ( + parse_string( + request, + "fi.mau.msc2815.include_unredacted_content", + allowed_values=("true", "false"), + ) + == "true" + ) + if include_unredacted_content and not await self.auth.is_server_admin( + requester.user + ): + power_level_event = await self._state.get_current_state( + room_id, EventTypes.PowerLevels, "" + ) + + auth_events = {} + if power_level_event: + auth_events[(EventTypes.PowerLevels, "")] = power_level_event + + redact_level = event_auth.get_named_level(auth_events, "redact", 50) + user_level = event_auth.get_user_power_level( + requester.user.to_string(), auth_events + ) + if user_level < redact_level: + raise SynapseError( + 403, + "You don't have permission to view redacted events in this room.", + errcode=Codes.FORBIDDEN, + ) + try: event = await self.event_handler.get_event( - requester.user, room_id, event_id + requester.user, + room_id, + event_id, + show_redacted=include_unredacted_content, ) except AuthError: # This endpoint is supposed to return a 404 when the requester does @@ -663,6 +702,11 @@ class RoomEventServlet(RestServlet): raise SynapseError(404, "Event not found.", errcode=Codes.NOT_FOUND) if event: + if include_unredacted_content and await self._store.have_censored_event( + event_id + ): + raise UnredactedContentDeletedError(self.content_keep_ms) + # Ensure there are bundled aggregations available. aggregations = await self._relations_handler.get_bundled_aggregations( [event], requester.user.to_string() diff --git a/synapse/rest/client/versions.py b/synapse/rest/client/versions.py index 7b29026381..bfc1d4ee08 100644 --- a/synapse/rest/client/versions.py +++ b/synapse/rest/client/versions.py @@ -101,6 +101,8 @@ class VersionsRestServlet(RestServlet): "org.matrix.msc3030": self.config.experimental.msc3030_enabled, # Adds support for thread relations, per MSC3440. "org.matrix.msc3440.stable": True, # TODO: remove when "v1.3" is added above + # Allows moderators to fetch redacted event content as described in MSC2815 + "fi.mau.msc2815": self.config.experimental.msc2815_enabled, }, }, ) diff --git a/synapse/storage/databases/main/events_worker.py b/synapse/storage/databases/main/events_worker.py index 5288cdba03..60876204bd 100644 --- a/synapse/storage/databases/main/events_worker.py +++ b/synapse/storage/databases/main/events_worker.py @@ -303,6 +303,24 @@ class EventsWorkerStore(SQLBaseStore): desc="get_received_ts", ) + async def have_censored_event(self, event_id: str) -> bool: + """Check if an event has been censored, i.e. if the content of the event has been erased + from the database due to a redaction. + + Args: + event_id: The event ID that was redacted. + + Returns: + True if the event has been censored, False otherwise. + """ + censored_redactions_list = await self.db_pool.simple_select_onecol( + table="redactions", + keyvalues={"redacts": event_id}, + retcol="have_censored", + desc="get_have_censored", + ) + return any(censored_redactions_list) + # Inform mypy that if allow_none is False (the default) then get_event # always returns an EventBase. @overload From d0c1f4ca4c66d4b067ff5fd1be43285f54773d4f Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Wed, 20 Apr 2022 09:56:59 -0400 Subject: [PATCH 44/60] Remove unnecessary config overrides for MSC3666. (#12511) --- changelog.d/12511.misc | 1 + tests/rest/client/test_relations.py | 4 ---- 2 files changed, 1 insertion(+), 4 deletions(-) create mode 100644 changelog.d/12511.misc diff --git a/changelog.d/12511.misc b/changelog.d/12511.misc new file mode 100644 index 0000000000..a314bedfc4 --- /dev/null +++ b/changelog.d/12511.misc @@ -0,0 +1 @@ +Remove unnecessary configuration overrides in tests. diff --git a/tests/rest/client/test_relations.py b/tests/rest/client/test_relations.py index 57d97bbb47..65743cdf67 100644 --- a/tests/rest/client/test_relations.py +++ b/tests/rest/client/test_relations.py @@ -355,7 +355,6 @@ class RelationsTestCase(BaseRelationsTestCase): self.assertEqual(200, channel.code, channel.json_body) self.assertNotIn("m.relations", channel.json_body["unsigned"]) - @unittest.override_config({"experimental_features": {"msc3666_enabled": True}}) def test_edit(self) -> None: """Test that a simple edit works.""" @@ -998,7 +997,6 @@ class BundledAggregationsTestCase(BaseRelationsTestCase): ] assert_bundle(self._find_event_in_chunk(chunk)) - @unittest.override_config({"experimental_features": {"msc3666_enabled": True}}) def test_annotation(self) -> None: """ Test that annotations get correctly bundled. @@ -1023,7 +1021,6 @@ class BundledAggregationsTestCase(BaseRelationsTestCase): self._test_bundled_aggregations(RelationTypes.ANNOTATION, assert_annotations, 7) - @unittest.override_config({"experimental_features": {"msc3666_enabled": True}}) def test_reference(self) -> None: """ Test that references get correctly bundled. @@ -1042,7 +1039,6 @@ class BundledAggregationsTestCase(BaseRelationsTestCase): self._test_bundled_aggregations(RelationTypes.REFERENCE, assert_annotations, 7) - @unittest.override_config({"experimental_features": {"msc3666_enabled": True}}) def test_thread(self) -> None: """ Test that threads get correctly bundled. From ecef741add0adfdd59f960e8b46503a5604303bd Mon Sep 17 00:00:00 2001 From: David Robertson Date: Wed, 20 Apr 2022 15:18:21 +0100 Subject: [PATCH 45/60] Recommend poetry in docs (#12475) * Recommend poetry in docs - readme - contributor guide - upgrade notes - new dev cheat sheet for poetry Co-authored-by: Shay Co-authored-by: Patrick Cloke --- README.rst | 31 ++-- changelog.d/12475.doc | 1 + docs/development/contributing_guide.md | 46 ++--- docs/development/dependencies.md | 239 +++++++++++++++++++++++++ docs/upgrade.md | 38 ++-- 5 files changed, 303 insertions(+), 52 deletions(-) create mode 100644 changelog.d/12475.doc create mode 100644 docs/development/dependencies.md diff --git a/README.rst b/README.rst index 8a14401d65..d71d733679 100644 --- a/README.rst +++ b/README.rst @@ -293,24 +293,27 @@ directory of your choice:: git clone https://github.com/matrix-org/synapse.git cd synapse -Synapse has a number of external dependencies, that are easiest -to install using pip and a virtualenv:: +Synapse has a number of external dependencies. We maintain a fixed development +environment using [poetry](https://python-poetry.org/). First, install poetry. We recommend - python3 -m venv ./env - source ./env/bin/activate - pip install -e ".[all,dev]" + pip install --user pipx + pipx install poetry + +as described `here `_. +(See `poetry's installation docs ` +for other installation methods.) Then ask poetry to create a virtual environment +from the project and install Synapse's dependencies:: + + poetry install --extras "all test" This will run a process of downloading and installing all the needed -dependencies into a virtual env. If any dependencies fail to install, -try installing the failing modules individually:: - - pip install -e "module-name" +dependencies into a virtual env. We recommend using the demo which starts 3 federated instances running on ports `8080` - `8082` - ./demo/start.sh + poetry run ./demo/start.sh -(to stop, you can use `./demo/stop.sh`) +(to stop, you can use `poetry run ./demo/stop.sh`) See the `demo documentation `_ for more information. @@ -318,14 +321,14 @@ for more information. If you just want to start a single instance of the app and run it directly:: # Create the homeserver.yaml config once - python -m synapse.app.homeserver \ + poetry run synapse_homeserver \ --server-name my.domain.name \ --config-path homeserver.yaml \ --generate-config \ --report-stats=[yes|no] # Start the app - python -m synapse.app.homeserver --config-path homeserver.yaml + poetry run synapse_homeserver --config-path homeserver.yaml Running the unit tests @@ -334,7 +337,7 @@ Running the unit tests After getting up and running, you may wish to run Synapse's unit tests to check that everything is installed correctly:: - trial tests + poetry run trial tests This should end with a 'PASSED' result (note that exact numbers will differ):: diff --git a/changelog.d/12475.doc b/changelog.d/12475.doc new file mode 100644 index 0000000000..f4481d0613 --- /dev/null +++ b/changelog.d/12475.doc @@ -0,0 +1 @@ +Strongly recommend `poetry` for development. diff --git a/docs/development/contributing_guide.md b/docs/development/contributing_guide.md index 0d9cf60196..3b5c774018 100644 --- a/docs/development/contributing_guide.md +++ b/docs/development/contributing_guide.md @@ -48,19 +48,28 @@ can find many good git tutorials on the web. # 4. Install the dependencies -Once you have installed Python 3 and added the source, please open a terminal and -setup a *virtualenv*, as follows: +Synapse uses the [poetry](https://python-poetry.org/) project to manage its dependencies +and development environment. Once you have installed Python 3 and added the +source, you should install `poetry`. +Of their installation methods, we recommend +[installing `poetry` using `pipx`](https://python-poetry.org/docs/#installing-with-pipx), + +```shell +pip install --user pipx +pipx install poetry +``` + +but see poetry's [installation instructions](https://python-poetry.org/docs/#installation) +for other installation methods. + +Next, open a terminal and install dependencies as follows: ```sh cd path/where/you/have/cloned/the/repository -python3 -m venv ./env -source ./env/bin/activate -pip install wheel -pip install -e ".[all,dev]" -pip install tox +poetry install --extras all ``` -This will install the developer dependencies for the project. +This will install the runtime and developer dependencies for the project. # 5. Get in touch. @@ -117,11 +126,10 @@ The linters look at your code and do two things: - ensure that your code follows the coding style adopted by the project; - catch a number of errors in your code. -The linter takes no time at all to run as soon as you've [downloaded the dependencies into your python virtual environment](#4-install-the-dependencies). +The linter takes no time at all to run as soon as you've [downloaded the dependencies](#4-install-the-dependencies). ```sh -source ./env/bin/activate -./scripts-dev/lint.sh +poetry run ./scripts-dev/lint.sh ``` Note that this script *will modify your files* to fix styling errors. @@ -131,15 +139,13 @@ If you wish to restrict the linters to only the files changed since the last com (much faster!), you can instead run: ```sh -source ./env/bin/activate -./scripts-dev/lint.sh -d +poetry run ./scripts-dev/lint.sh -d ``` Or if you know exactly which files you wish to lint, you can instead run: ```sh -source ./env/bin/activate -./scripts-dev/lint.sh path/to/file1.py path/to/file2.py path/to/folder +poetry run ./scripts-dev/lint.sh path/to/file1.py path/to/file2.py path/to/folder ``` ## Run the unit tests (Twisted trial). @@ -148,16 +154,14 @@ The unit tests run parts of Synapse, including your changes, to see if anything was broken. They are slower than the linters but will typically catch more errors. ```sh -source ./env/bin/activate -trial tests +poetry run trial tests ``` If you wish to only run *some* unit tests, you may specify another module instead of `tests` - or a test class or a method: ```sh -source ./env/bin/activate -trial tests.rest.admin.test_room tests.handlers.test_admin.ExfiltrateData.test_invite +poetry run trial tests.rest.admin.test_room tests.handlers.test_admin.ExfiltrateData.test_invite ``` If your tests fail, you may wish to look at the logs (the default log level is `ERROR`): @@ -169,7 +173,7 @@ less _trial_temp/test.log To increase the log level for the tests, set `SYNAPSE_TEST_LOG_LEVEL`: ```sh -SYNAPSE_TEST_LOG_LEVEL=DEBUG trial tests +SYNAPSE_TEST_LOG_LEVEL=DEBUG poetry run trial tests ``` By default, tests will use an in-memory SQLite database for test data. For additional @@ -180,7 +184,7 @@ database state to be stored in a file named `test.db` under the trial process' working directory. Typically, this ends up being `_trial_temp/test.db`. For example: ```sh -SYNAPSE_TEST_PERSIST_SQLITE_DB=1 trial tests +SYNAPSE_TEST_PERSIST_SQLITE_DB=1 poetry run trial tests ``` The database file can then be inspected with: diff --git a/docs/development/dependencies.md b/docs/development/dependencies.md new file mode 100644 index 0000000000..8ef7d357d8 --- /dev/null +++ b/docs/development/dependencies.md @@ -0,0 +1,239 @@ +# Managing dependencies with Poetry + +This is a quick cheat sheet for developers on how to use [`poetry`](https://python-poetry.org/). + +# Background + +Synapse uses a variety of third-party Python packages to function as a homeserver. +Some of these are direct dependencies, listed in `pyproject.toml` under the +`[tool.poetry.dependencies]` section. The rest are transitive dependencies (the +things that our direct dependencies themselves depend on, and so on recursively.) + +We maintain a locked list of all our dependencies (transitive included) so that +we can track exactly which version of each dependency appears in a given release. +See [here](https://github.com/matrix-org/synapse/issues/11537#issue-1074469665) +for discussion of why we wanted this for Synapse. We chose to use +[`poetry`](https://python-poetry.org/) to manage this locked list; see +[this comment](https://github.com/matrix-org/synapse/issues/11537#issuecomment-1015975819) +for the reasoning. + +The locked dependencies get included in our "self-contained" releases: namely, +our docker images and our debian packages. We also use the locked dependencies +in development and our continuous integration. + +Separately, our "broad" dependencies—the version ranges specified in +`pyproject.toml`—are included as metadata in our "sdists" and "wheels" [uploaded +to PyPI](https://pypi.org/project/matrix-synapse). Installing from PyPI or from +the Synapse source tree directly will _not_ use the locked dependencies; instead, +they'll pull in the latest version of each package available at install time. + +## Example dependency + +An example may help. We have a broad dependency on +[`phonenumbers`](https://pypi.org/project/phonenumbers/), as declared in +this snippet from pyproject.toml [as of Synapse 1.57]( +https://github.com/matrix-org/synapse/blob/release-v1.57/pyproject.toml#L133 +): + +```toml +[tool.poetry.dependencies] +# ... +phonenumbers = ">=8.2.0" +``` + +In our lockfile this is +[pinned]( https://github.com/matrix-org/synapse/blob/dfc7646504cef3e4ff396c36089e1c6f1b1634de/poetry.lock#L679-L685) +to version 8.12.44, even though +[newer versions are available](https://pypi.org/project/phonenumbers/#history). + +```toml +[[package]] +name = "phonenumbers" +version = "8.12.44" +description = "Python version of Google's common library for parsing, formatting, storing and validating international phone numbers." +category = "main" +optional = false +python-versions = "*" +``` + +The lockfile also includes a +[cryptographic checksum](https://github.com/matrix-org/synapse/blob/release-v1.57/poetry.lock#L2178-L2181) +of the sdists and wheels provided for this version of `phonenumbers`. + +```toml +[metadata.files] +# ... +phonenumbers = [ + {file = "phonenumbers-8.12.44-py2.py3-none-any.whl", hash = "sha256:cc1299cf37b309ecab6214297663ab86cb3d64ae37fd5b88e904fe7983a874a6"}, + {file = "phonenumbers-8.12.44.tar.gz", hash = "sha256:26cfd0257d1704fe2f88caff2caabb70d16a877b1e65b6aae51f9fbbe10aa8ce"}, +] +``` + +We can see this pinned version inside the docker image for that release: + +``` +$ docker pull matrixdotorg/synapse:v1.57.0 +... +$ docker run --entrypoint pip matrixdotorg/synapse:v1.57.0 show phonenumbers +Name: phonenumbers +Version: 8.12.44 +Summary: Python version of Google's common library for parsing, formatting, storing and validating international phone numbers. +Home-page: https://github.com/daviddrysdale/python-phonenumbers +Author: David Drysdale +Author-email: dmd@lurklurk.org +License: Apache License 2.0 +Location: /usr/local/lib/python3.9/site-packages +Requires: +Required-by: matrix-synapse +``` + +Whereas the wheel metadata just contains the broad dependencies: + +``` +$ cd /tmp +$ wget https://files.pythonhosted.org/packages/ca/5e/d722d572cc5b3092402b783d6b7185901b444427633bd8a6b00ea0dd41b7/matrix_synapse-1.57.0rc1-py3-none-any.whl +... +$ unzip -c matrix_synapse-1.57.0rc1-py3-none-any.whl matrix_synapse-1.57.0rc1.dist-info/METADATA | grep phonenumbers +Requires-Dist: phonenumbers (>=8.2.0) +``` + +# Tooling recommendation: direnv + +[`direnv`](https://direnv.net/) is a tool for activating environments in your +shell inside a given directory. Its support for poetry is unofficial (a +community wiki recipe only), but works solidly in our experience. We thoroughly +recommend it for daily use. To use it: + +1. [Install `direnv`](https://direnv.net/docs/installation.html) - it's likely + packaged for your system already. +2. Teach direnv about poetry. The [shell config here](https://github.com/direnv/direnv/wiki/Python#poetry) + needs to be added to `~/.config/direnv/direnvrc` (or more generally `$XDG_CONFIG_HOME/direnv/direnvrc`). +3. Mark the synapse checkout as a poetry project: `echo layout poetry > .envrc`. +4. Convince yourself that you trust this `.envrc` configuration and project. + Then formally confirm this to `direnv` by running `direnv allow`. + +Then whenever you navigate to the synapse checkout, you should be able to run +e.g. `mypy` instead of `poetry run mypy`; `python` instead of +`poetry run python`; and your shell commands will automatically run in the +context of poetry's venv, without having to run `poetry shell` beforehand. + + +# How do I... + +## ...reset my venv to the locked environment? + +```shell +poetry install --extras all --remove-untracked +``` + +## ...run a command in the `poetry` virtualenv? + +Use `poetry run cmd args` when you need the python virtualenv context. +To avoid typing `poetry run` all the time, you can run `poetry shell` +to start a new shell in the poetry virtualenv context. Within `poetry shell`, +`python`, `pip`, `mypy`, `trial`, etc. are all run inside the project virtualenv +and isolated from the rest o the system. + +Roughly speaking, the translation from a traditional virtualenv is: +- `env/bin/activate` -> `poetry shell`, and +- `deactivate` -> close the terminal (Ctrl-D, `exit`, etc.) + +See also the direnv recommendation above, which makes `poetry run` and +`poetry shell` unnecessary. + + +## ...inspect the `poetry` virtualenv? + +Some suggestions: + +```shell +# Current env only +poetry env info +# All envs: this allows you to have e.g. a poetry managed venv for Python 3.7, +# and another for Python 3.10. +poetry env list --full-path +poetry run pip list +``` + +Note that `poetry show` describes the abstract *lock file* rather than your +on-disk environment. With that said, `poetry show --tree` can sometimes be +useful. + + +## ...add a new dependency? + +Either: +- manually update `pyproject.toml`; then `poetry lock --no-update`; or else +- `poetry add packagename`. See `poetry add --help`; note the `--dev`, + `--extras` and `--optional` flags in particular. + - **NB**: this specifies the new package with a version given by a "caret bound". This won't get forced to its lowest version in the old deps CI job: see [this TODO](https://github.com/matrix-org/synapse/blob/4e1374373857f2f7a911a31c50476342d9070681/.ci/scripts/test_old_deps.sh#L35-L39). + +Include the updated `pyproject.toml` and `poetry.lock` files in your commit. + +## ...remove a dependency? + +This is not done often and is untested, but + +```shell +poetry remove packagename +``` + +ought to do the trick. Alternatively, manually update `pyproject.toml` and +`poetry lock --no-update`. Include the updated `pyproject.toml` and poetry.lock` +files in your commit. + +## ...update the version range for an existing dependency? + +Best done by manually editing `pyproject.toml`, then `poetry lock --no-update`. +Include the updated `pyproject.toml` and `poetry.lock` in your commit. + +## ...update a dependency in the locked environment? + +Use + +```shell +poetry update packagename +``` + +to use the latest version of `packagename` in the locked environment, without +affecting the broad dependencies listed in the wheel. + +There doesn't seem to be a way to do this whilst locking a _specific_ version of +`packagename`. We can workaround this (crudely) as follows: + +```shell +poetry add packagename==1.2.3 +# This should update pyproject.lock. + +# Now undo the changes to pyproject.toml. For example +# git restore pyproject.toml + +# Get poetry to recompute the content-hash of pyproject.toml without changing +# the locked package versions. +poetry lock --no-update +``` + +Either way, include the updated `poetry.lock` file in your commit. + +## ...export a `requirements.txt` file? + +```shell +poetry export --extras all +``` + +Be wary of bugs in `poetry export` and `pip install -r requirements.txt`. + +Note: `poetry export` will be made a plugin in Poetry 1.2. Additional config may +be required. + +## ...build a test wheel? + +I usually use + +```shell +poetry run pip install build && poetry run python -m build +``` + +because [`build`](https://github.com/pypa/build) is a standardish tool which +doesn't require poetry. (It's what we use in CI too). However, you could try +`poetry build` too. diff --git a/docs/upgrade.md b/docs/upgrade.md index a0c797ea9f..3a8aeb0395 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -19,32 +19,36 @@ this document. packages](setup/installation.md#prebuilt-packages), you will need to follow the normal process for upgrading those packages. +- If Synapse was installed using pip then upgrade to the latest + version by running: + + ```bash + pip install --upgrade matrix-synapse + ``` + - If Synapse was installed from source, then: - 1. Activate the virtualenv before upgrading. For example, if - Synapse is installed in a virtualenv in `~/synapse/env` then + 1. Obtain the latest version of the source code. Git users can run + `git pull` to do this. + + 2. If you're running Synapse in a virtualenv, make sure to activate it before + upgrading. For example, if Synapse is installed in a virtualenv in `~/synapse/env` then run: ```bash source ~/synapse/env/bin/activate - ``` - - 2. If Synapse was installed using pip then upgrade to the latest - version by running: - - ```bash - pip install --upgrade matrix-synapse - ``` - - If Synapse was installed using git then upgrade to the latest - version by running: - - ```bash - git pull pip install --upgrade . ``` + Include any relevant extras between square brackets, e.g. `pip install --upgrade ".[postgres,oidc]"`. - 3. Restart Synapse: + 3. If you're using `poetry` to manage a Synapse installation, run: + ```bash + poetry install + ``` + Include any relevant extras with `--extras`, e.g. `poetry install --extras postgres --extras oidc`. + It's probably easiest to run `poetry install --extras all`. + + 4. Restart Synapse: ```bash synctl restart From 3e2e76ca1507ff6b0ed4e5095fd602b61a26704f Mon Sep 17 00:00:00 2001 From: reivilibre Date: Wed, 20 Apr 2022 15:22:53 +0100 Subject: [PATCH 46/60] Include version 0.2.0 of the Synapse LDAP Auth Provider module in the Docker image. (#12512) * poetry update matrix-synapse-ldap3 * Newsfile Signed-off-by: Olivier Wilkinson (reivilibre) --- changelog.d/12512.docker | 1 + poetry.lock | 12 ++++++++---- 2 files changed, 9 insertions(+), 4 deletions(-) create mode 100644 changelog.d/12512.docker diff --git a/changelog.d/12512.docker b/changelog.d/12512.docker new file mode 100644 index 0000000000..24c0f6a3b1 --- /dev/null +++ b/changelog.d/12512.docker @@ -0,0 +1 @@ +Include version 0.2.0 of the Synapse LDAP Auth Provider module in the Docker image. \ No newline at end of file diff --git a/poetry.lock b/poetry.lock index bbe8eba96d..c11e2c6848 100644 --- a/poetry.lock +++ b/poetry.lock @@ -558,17 +558,20 @@ test = ["tox", "twisted", "aiounittest"] [[package]] name = "matrix-synapse-ldap3" -version = "0.1.5" +version = "0.2.0" description = "An LDAP3 auth provider for Synapse" category = "main" optional = true -python-versions = "*" +python-versions = ">=3.7" [package.dependencies] ldap3 = ">=2.8" -service_identity = "*" +service-identity = "*" Twisted = ">=15.1.0" +[package.extras] +dev = ["matrix-synapse", "tox", "ldaptor", "mypy (==0.910)", "types-setuptools", "black (==21.9b0)", "flake8 (==4.0.1)", "isort (==5.9.3)"] + [[package]] name = "mccabe" version = "0.6.1" @@ -2084,7 +2087,8 @@ matrix-common = [ {file = "matrix_common-1.1.0.tar.gz", hash = "sha256:a8238748afc2b37079818367fed5156f355771b07c8ff0a175934f47e0ff3276"}, ] matrix-synapse-ldap3 = [ - {file = "matrix-synapse-ldap3-0.1.5.tar.gz", hash = "sha256:9fdf8df7c8ec756642aa0fea53b31c0b2f1924f70d7f049a2090b523125456fe"}, + {file = "matrix-synapse-ldap3-0.2.0.tar.gz", hash = "sha256:91a0715b43a41ec3033244174fca20846836da98fda711fb01687f7199eecd2e"}, + {file = "matrix_synapse_ldap3-0.2.0-py3-none-any.whl", hash = "sha256:0128ca7c3058987adc2e8a88463bb46879915bfd3d373309632813b353e30f9f"}, ] mccabe = [ {file = "mccabe-0.6.1-py2.py3-none-any.whl", hash = "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42"}, From 05e8a5d298011508e04ecf4b2f73aaa737ed6035 Mon Sep 17 00:00:00 2001 From: "Olivier Wilkinson (reivilibre)" Date: Wed, 20 Apr 2022 15:30:03 +0100 Subject: [PATCH 47/60] 1.57.1 --- CHANGES.md | 13 +++++++++++++ changelog.d/12512.docker | 1 - debian/changelog | 6 ++++++ pyproject.toml | 2 +- synapse/__init__.py | 2 +- 5 files changed, 21 insertions(+), 3 deletions(-) delete mode 100644 changelog.d/12512.docker diff --git a/CHANGES.md b/CHANGES.md index 74a5dbf424..63c66af5bf 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,16 @@ +Synapse 1.57.1 (2022-04-20) +=========================== + +This is a patch release that only affects the Docker image. It is only of interest to administrators using the LDAP module to authenticate their users. +If you have already upgraded to Synapse 1.57.0 without problem, then you have no need to upgrade to this patch release. + + +Updates to the Docker image +--------------------------- + +- Include version 0.2.0 of the Synapse LDAP Auth Provider module in the Docker image. ([\#12512](https://github.com/matrix-org/synapse/issues/12512)) + + Synapse 1.57.0 (2022-04-19) =========================== diff --git a/changelog.d/12512.docker b/changelog.d/12512.docker deleted file mode 100644 index 24c0f6a3b1..0000000000 --- a/changelog.d/12512.docker +++ /dev/null @@ -1 +0,0 @@ -Include version 0.2.0 of the Synapse LDAP Auth Provider module in the Docker image. \ No newline at end of file diff --git a/debian/changelog b/debian/changelog index 71dcf9de8e..2db6ed9491 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.57.1) stable; urgency=medium + + * New synapse release 1.57.1. + + -- Synapse Packaging team Wed, 20 Apr 2022 15:27:21 +0100 + matrix-synapse-py3 (1.57.0) stable; urgency=medium * New synapse release 1.57.0. diff --git a/pyproject.toml b/pyproject.toml index 7f58c37e3f..65f5a5f59f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,7 +54,7 @@ skip_gitignore = true [tool.poetry] name = "matrix-synapse" -version = "1.57.0" +version = "1.57.1" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] license = "Apache-2.0" diff --git a/synapse/__init__.py b/synapse/__init__.py index bf88f1d93d..b62eed66e2 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -68,7 +68,7 @@ try: except ImportError: pass -__version__ = "1.57.0" +__version__ = "1.57.1" if bool(os.environ.get("SYNAPSE_TEST_PATCH_LOG_CONTEXTS", False)): # We import here so that we don't have to install a bunch of deps when From f8f06fc773e6d1dd9674be77d3e6a3665830a33a Mon Sep 17 00:00:00 2001 From: "Olivier Wilkinson (reivilibre)" Date: Wed, 20 Apr 2022 15:48:05 +0100 Subject: [PATCH 48/60] Clarify changelog entry --- CHANGES.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 63c66af5bf..a7d2529b55 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,14 +1,16 @@ Synapse 1.57.1 (2022-04-20) =========================== -This is a patch release that only affects the Docker image. It is only of interest to administrators using the LDAP module to authenticate their users. +This is a patch release that only affects the Docker image. It is only of interest to administrators using [the LDAP module][LDAPModule] to authenticate their users. If you have already upgraded to Synapse 1.57.0 without problem, then you have no need to upgrade to this patch release. +[LDAPModule]: https://github.com/matrix-org/matrix-synapse-ldap3 + Updates to the Docker image --------------------------- -- Include version 0.2.0 of the Synapse LDAP Auth Provider module in the Docker image. ([\#12512](https://github.com/matrix-org/synapse/issues/12512)) +- Include version 0.2.0 of the Synapse LDAP Auth Provider module in the Docker image. This matches the version that was present in the Docker image for Synapse v1.56.0. ([\#12512](https://github.com/matrix-org/synapse/issues/12512)) Synapse 1.57.0 (2022-04-19) From 103f51d867dc3966e44d79e949e78380efef5e2d Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Wed, 20 Apr 2022 12:03:03 -0400 Subject: [PATCH 49/60] Fix Jinja templating error when generating thumbnail URLs. (#12510) scale is meant to be a constant string, not refer to a variable. --- changelog.d/12510.bugfix | 1 + synapse/res/templates/notif.html | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/12510.bugfix diff --git a/changelog.d/12510.bugfix b/changelog.d/12510.bugfix new file mode 100644 index 0000000000..d5856e982a --- /dev/null +++ b/changelog.d/12510.bugfix @@ -0,0 +1 @@ +Fix a long-standing bug where the image thumbanils embedded into email notifications were broken. diff --git a/synapse/res/templates/notif.html b/synapse/res/templates/notif.html index 0aaef97df8..7d86681fed 100644 --- a/synapse/res/templates/notif.html +++ b/synapse/res/templates/notif.html @@ -30,7 +30,7 @@ {%- elif message.msgtype == "m.notice" %} {{ message.body_text_html }} {%- elif message.msgtype == "m.image" and message.image_url %} - + {%- elif message.msgtype == "m.file" %} {{ message.body_text_plain }} {%- else %} From e5a76ec00b443c6e15ac8de2838a5c60a47a67ec Mon Sep 17 00:00:00 2001 From: David Robertson Date: Wed, 20 Apr 2022 17:33:20 +0100 Subject: [PATCH 50/60] Dump setuptools; correct pyproject version number (#12478) --- .dockerignore | 4 - .github/workflows/tests.yml | 2 +- .gitignore | 3 +- MANIFEST.in | 54 ---------- changelog.d/12337.feature | 1 + changelog.d/12478.misc | 1 + docker/Dockerfile | 6 +- mypy.ini | 1 - pyproject.toml | 4 +- setup.cfg | 9 -- setup.py | 183 --------------------------------- synapse/__init__.py | 4 +- synapse/python_dependencies.py | 151 --------------------------- 13 files changed, 11 insertions(+), 412 deletions(-) delete mode 100644 MANIFEST.in create mode 100644 changelog.d/12337.feature create mode 100644 changelog.d/12478.misc delete mode 100644 setup.cfg delete mode 100755 setup.py delete mode 100644 synapse/python_dependencies.py diff --git a/.dockerignore b/.dockerignore index a236760cf1..7809863ef3 100644 --- a/.dockerignore +++ b/.dockerignore @@ -8,8 +8,4 @@ !pyproject.toml !poetry.lock -# TODO: remove these once we have moved over to using poetry-core in pyproject.toml -!MANIFEST.in -!setup.py - **/__pycache__ diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7946b76dba..cad4cb6d77 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -15,7 +15,7 @@ jobs: steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 - - run: pip install -e . + - run: pip install . - run: scripts-dev/generate_sample_config.sh --check - run: scripts-dev/config-lint.sh diff --git a/.gitignore b/.gitignore index c011cd27a4..e58affb241 100644 --- a/.gitignore +++ b/.gitignore @@ -15,8 +15,7 @@ _trial_temp*/ .DS_Store __pycache__/ -# We do want the poetry lockfile. TODO: is there a good reason for ignoring -# '*.lock' above? If not, let's nuke it. +# We do want the poetry lockfile. !poetry.lock # stuff that is likely to exist when you run a server locally diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index d744c090ac..0000000000 --- a/MANIFEST.in +++ /dev/null @@ -1,54 +0,0 @@ -include LICENSE -include VERSION -include *.rst -include *.md -include demo/README -include demo/demo.tls.dh -include demo/*.py -include demo/*.sh - -include synapse/py.typed -recursive-include synapse/storage *.sql -recursive-include synapse/storage *.sql.postgres -recursive-include synapse/storage *.sql.sqlite -recursive-include synapse/storage *.py -recursive-include synapse/storage *.txt -recursive-include synapse/storage *.md - -recursive-include docs * -recursive-include scripts-dev * -recursive-include synapse *.pyi -recursive-include tests *.py -recursive-include tests *.pem -recursive-include tests *.p8 -recursive-include tests *.crt -recursive-include tests *.key - -recursive-include synapse/res * -recursive-include synapse/static *.css -recursive-include synapse/static *.gif -recursive-include synapse/static *.html -recursive-include synapse/static *.js - -exclude .codecov.yml -exclude .coveragerc -exclude .dockerignore -exclude .editorconfig -exclude Dockerfile -exclude mypy.ini -exclude sytest-blacklist -exclude test_postgresql.sh - -include book.toml -include pyproject.toml -recursive-include changelog.d * - -include .flake8 -prune .circleci -prune .github -prune .ci -prune contrib -prune debian -prune demo/etc -prune docker -prune stubs diff --git a/changelog.d/12337.feature b/changelog.d/12337.feature new file mode 100644 index 0000000000..6c4444c707 --- /dev/null +++ b/changelog.d/12337.feature @@ -0,0 +1 @@ +Use poetry to manage Synapse's dependencies. \ No newline at end of file diff --git a/changelog.d/12478.misc b/changelog.d/12478.misc new file mode 100644 index 0000000000..061a604a1e --- /dev/null +++ b/changelog.d/12478.misc @@ -0,0 +1 @@ +Use poetry-core instead of setuptools to build wheels. diff --git a/docker/Dockerfile b/docker/Dockerfile index 6f87702cc8..4523c60645 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -59,7 +59,7 @@ RUN --mount=type=cache,target=/root/.cache/pip \ WORKDIR /synapse # Copy just what we need to run `poetry export`... -COPY pyproject.toml poetry.lock README.rst /synapse/ +COPY pyproject.toml poetry.lock /synapse/ RUN /root/.local/bin/poetry export --extras all -o /synapse/requirements.txt @@ -98,9 +98,7 @@ RUN --mount=type=cache,target=/root/.cache/pip \ # Copy over the rest of the synapse source code. COPY synapse /synapse/synapse/ # ... and what we need to `pip install`. -# TODO: once pyproject.toml declares poetry-core as its build system, we'll need to copy -# pyproject.toml here, ditching setup.py and MANIFEST.in. -COPY setup.py MANIFEST.in README.rst /synapse/ +COPY pyproject.toml README.rst /synapse/ # Install the synapse package itself. RUN pip install --prefix="/install" --no-deps --no-warn-script-location /synapse diff --git a/mypy.ini b/mypy.ini index 5246f987c0..b2c3b1524c 100644 --- a/mypy.ini +++ b/mypy.ini @@ -13,7 +13,6 @@ no_implicit_optional = True files = docker/, scripts-dev/, - setup.py, synapse/, tests/ diff --git a/pyproject.toml b/pyproject.toml index 7f58c37e3f..c04857b791 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -280,5 +280,5 @@ twine = "*" towncrier = ">=18.6.0rc1" [build-system] -requires = ["setuptools"] -build-backend = "setuptools.build_meta" +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 6213f3265b..0000000000 --- a/setup.cfg +++ /dev/null @@ -1,9 +0,0 @@ -[check-manifest] -ignore = - .git-blame-ignore-revs - contrib - contrib/* - docs/* - pylint.cfg - tox.ini - diff --git a/setup.py b/setup.py deleted file mode 100755 index ecd30247ed..0000000000 --- a/setup.py +++ /dev/null @@ -1,183 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2014-2017 OpenMarket Ltd -# Copyright 2017 Vector Creations Ltd -# Copyright 2017-2018 New Vector Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import os -from typing import Any, Dict - -from setuptools import Command, find_packages, setup - -here = os.path.abspath(os.path.dirname(__file__)) - - -# Some notes on `setup.py test`: -# -# Once upon a time we used to try to make `setup.py test` run `tox` to run the -# tests. That's a bad idea for three reasons: -# -# 1: `setup.py test` is supposed to find out whether the tests work in the -# *current* environmentt, not whatever tox sets up. -# 2: Empirically, trying to install tox during the test run wasn't working ("No -# module named virtualenv"). -# 3: The tox documentation advises against it[1]. -# -# Even further back in time, we used to use setuptools_trial [2]. That has its -# own set of issues: for instance, it requires installation of Twisted to build -# an sdist (because the recommended mode of usage is to add it to -# `setup_requires`). That in turn means that in order to successfully run tox -# you have to have the python header files installed for whichever version of -# python tox uses (which is python3 on recent ubuntus, for example). -# -# So, for now at least, we stick with what appears to be the convention among -# Twisted projects, and don't attempt to do anything when someone runs -# `setup.py test`; instead we direct people to run `trial` directly if they -# care. -# -# [1]: http://tox.readthedocs.io/en/2.5.0/example/basic.html#integration-with-setup-py-test-command -# [2]: https://pypi.python.org/pypi/setuptools_trial -class TestCommand(Command): - def initialize_options(self): - pass - - def finalize_options(self): - pass - - def run(self): - print( - """Synapse's tests cannot be run via setup.py. To run them, try: - PYTHONPATH="." trial tests -""" - ) - - -def read_file(path_segments): - """Read a file from the package. Takes a list of strings to join to - make the path""" - file_path = os.path.join(here, *path_segments) - with open(file_path) as f: - return f.read() - - -def exec_file(path_segments): - """Execute a single python file to get the variables defined in it""" - result: Dict[str, Any] = {} - code = read_file(path_segments) - exec(code, result) - return result - - -version = exec_file(("synapse", "__init__.py"))["__version__"] -dependencies = exec_file(("synapse", "python_dependencies.py")) -long_description = read_file(("README.rst",)) - -REQUIREMENTS = dependencies["REQUIREMENTS"] -CONDITIONAL_REQUIREMENTS = dependencies["CONDITIONAL_REQUIREMENTS"] -ALL_OPTIONAL_REQUIREMENTS = dependencies["ALL_OPTIONAL_REQUIREMENTS"] - -# Make `pip install matrix-synapse[all]` install all the optional dependencies. -CONDITIONAL_REQUIREMENTS["all"] = list(ALL_OPTIONAL_REQUIREMENTS) - -# Developer dependencies should not get included in "all". -# -# We pin black so that our tests don't start failing on new releases. -CONDITIONAL_REQUIREMENTS["lint"] = [ - "isort==5.7.0", - "black==22.3.0", - "flake8-comprehensions", - "flake8-bugbear==21.3.2", - "flake8", -] - -CONDITIONAL_REQUIREMENTS["mypy"] = [ - "mypy==0.931", - "mypy-zope==0.3.5", - "types-bleach>=4.1.0", - "types-jsonschema>=3.2.0", - "types-opentracing>=2.4.2", - "types-Pillow>=8.3.4", - "types-psycopg2>=2.9.9", - "types-pyOpenSSL>=20.0.7", - "types-PyYAML>=5.4.10", - "types-requests>=2.26.0", - "types-setuptools>=57.4.0", -] - -# Dependencies which are exclusively required by unit test code. This is -# NOT a list of all modules that are necessary to run the unit tests. -# Tests assume that all optional dependencies are installed. -# -# parameterized_class decorator was introduced in parameterized 0.7.0 -CONDITIONAL_REQUIREMENTS["test"] = ["parameterized>=0.7.0", "idna>=2.5"] - -CONDITIONAL_REQUIREMENTS["dev"] = ( - CONDITIONAL_REQUIREMENTS["lint"] - + CONDITIONAL_REQUIREMENTS["mypy"] - + CONDITIONAL_REQUIREMENTS["test"] - + [ - # The following are used by the release script - "click==8.1.0", - "redbaron==0.9.2", - "GitPython==3.1.14", - "commonmark==0.9.1", - "pygithub==1.55", - # The following are executed as commands by the release script. - "twine", - "towncrier", - ] -) - -setup( - name="matrix-synapse", - version=version, - packages=find_packages(exclude=["tests", "tests.*"]), - description="Reference homeserver for the Matrix decentralised comms protocol", - install_requires=REQUIREMENTS, - extras_require=CONDITIONAL_REQUIREMENTS, - include_package_data=True, - zip_safe=False, - long_description=long_description, - long_description_content_type="text/x-rst", - python_requires="~=3.7", - entry_points={ - "console_scripts": [ - # Application - "synapse_homeserver = synapse.app.homeserver:main", - "synapse_worker = synapse.app.generic_worker:main", - "synctl = synapse._scripts.synctl:main", - # Scripts - "export_signing_key = synapse._scripts.export_signing_key:main", - "generate_config = synapse._scripts.generate_config:main", - "generate_log_config = synapse._scripts.generate_log_config:main", - "generate_signing_key = synapse._scripts.generate_signing_key:main", - "hash_password = synapse._scripts.hash_password:main", - "register_new_matrix_user = synapse._scripts.register_new_matrix_user:main", - "synapse_port_db = synapse._scripts.synapse_port_db:main", - "synapse_review_recent_signups = synapse._scripts.review_recent_signups:main", - "update_synapse_database = synapse._scripts.update_synapse_database:main", - ] - }, - classifiers=[ - "Development Status :: 5 - Production/Stable", - "Topic :: Communications :: Chat", - "License :: OSI Approved :: Apache Software License", - "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - ], - cmdclass={"test": TestCommand}, -) diff --git a/synapse/__init__.py b/synapse/__init__.py index bf88f1d93d..1613941759 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -20,6 +20,8 @@ import json import os import sys +from matrix_common.versionstring import get_distribution_version_string + # Check that we're not running on an unsupported Python version. if sys.version_info < (3, 7): print("Synapse requires Python 3.7 or above.") @@ -68,7 +70,7 @@ try: except ImportError: pass -__version__ = "1.57.0" +__version__ = get_distribution_version_string("matrix-synapse") if bool(os.environ.get("SYNAPSE_TEST_PATCH_LOG_CONTEXTS", False)): # We import here so that we don't have to install a bunch of deps when diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py deleted file mode 100644 index ec199a161d..0000000000 --- a/synapse/python_dependencies.py +++ /dev/null @@ -1,151 +0,0 @@ -# Copyright 2015, 2016 OpenMarket Ltd -# Copyright 2017 Vector Creations Ltd -# Copyright 2018 New Vector Ltd -# Copyright 2020 The Matrix.org Foundation C.I.C. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import itertools -import logging -from typing import Set - -logger = logging.getLogger(__name__) - - -# REQUIREMENTS is a simple list of requirement specifiers[1], and must be -# installed. It is passed to setup() as install_requires in setup.py. -# -# CONDITIONAL_REQUIREMENTS is the optional dependencies, represented as a dict -# of lists. The dict key is the optional dependency name and can be passed to -# pip when installing. The list is a series of requirement specifiers[1] to be -# installed when that optional dependency requirement is specified. It is passed -# to setup() as extras_require in setup.py -# -# Note that these both represent runtime dependencies (and the versions -# installed are checked at runtime). -# -# Also note that we replicate these constraints in the Synapse Dockerfile while -# pre-installing dependencies. If these constraints are updated here, the same -# change should be made in the Dockerfile. -# -# [1] https://pip.pypa.io/en/stable/reference/pip_install/#requirement-specifiers. - -REQUIREMENTS = [ - # we use the TYPE_CHECKER.redefine method added in jsonschema 3.0.0 - "jsonschema>=3.0.0", - # frozendict 2.1.2 is broken on Debian 10: https://github.com/Marco-Sulla/python-frozendict/issues/41 - "frozendict>=1,!=2.1.2", - "unpaddedbase64>=1.1.0", - "canonicaljson>=1.4.0", - # we use the type definitions added in signedjson 1.1. - "signedjson>=1.1.0", - "pynacl>=1.2.1", - # validating SSL certs for IP addresses requires service_identity 18.1. - "service_identity>=18.1.0", - # Twisted 18.9 introduces some logger improvements that the structured - # logger utilises - "Twisted[tls]>=18.9.0", - "treq>=15.1", - # Twisted has required pyopenssl 16.0 since about Twisted 16.6. - "pyopenssl>=16.0.0", - "pyyaml>=3.11", - "pyasn1>=0.1.9", - "pyasn1-modules>=0.0.7", - "bcrypt>=3.1.0", - "pillow>=5.4.0", - "sortedcontainers>=1.4.4", - "pymacaroons>=0.13.0", - "msgpack>=0.5.2", - "phonenumbers>=8.2.0", - # we use GaugeHistogramMetric, which was added in prom-client 0.4.0. - "prometheus_client>=0.4.0", - # we use `order`, which arrived in attrs 19.2.0. - # Note: 21.1.0 broke `/sync`, see #9936 - "attrs>=19.2.0,!=21.1.0", - "netaddr>=0.7.18", - # Jinja 2.x is incompatible with MarkupSafe>=2.1. To ensure that admins do not - # end up with a broken installation, with recent MarkupSafe but old Jinja, we - # add a lower bound to the Jinja2 dependency. - "Jinja2>=3.0", - "bleach>=1.4.3", - # We use `ParamSpec`, which was added in `typing-extensions` 3.10.0.0. - "typing-extensions>=3.10.0", - # We enforce that we have a `cryptography` version that bundles an `openssl` - # with the latest security patches. - "cryptography>=3.4.7", - # ijson 3.1.4 fixes a bug with "." in property names - "ijson>=3.1.4", - "matrix-common~=1.1.0", - # We need packaging.requirements.Requirement, added in 16.1. - "packaging>=16.1", - # At the time of writing, we only use functions from the version `importlib.metadata` - # which shipped in Python 3.8. This corresponds to version 1.4 of the backport. - "importlib_metadata>=1.4 ; python_version < '3.8'", -] - -CONDITIONAL_REQUIREMENTS = { - "matrix-synapse-ldap3": ["matrix-synapse-ldap3>=0.1"], - "postgres": [ - # we use execute_values with the fetch param, which arrived in psycopg 2.8. - "psycopg2>=2.8 ; platform_python_implementation != 'PyPy'", - "psycopg2cffi>=2.8 ; platform_python_implementation == 'PyPy'", - "psycopg2cffi-compat==1.1 ; platform_python_implementation == 'PyPy'", - ], - "saml2": [ - "pysaml2>=4.5.0", - ], - "oidc": ["authlib>=0.14.0"], - # systemd-python is necessary for logging to the systemd journal via - # `systemd.journal.JournalHandler`, as is documented in - # `contrib/systemd/log_config.yaml`. - "systemd": ["systemd-python>=231"], - "url_preview": ["lxml>=4.2.0"], - "sentry": ["sentry-sdk>=0.7.2"], - "opentracing": ["jaeger-client>=4.0.0", "opentracing>=2.2.0"], - "jwt": ["pyjwt>=1.6.4"], - # hiredis is not a *strict* dependency, but it makes things much faster. - # (if it is not installed, we fall back to slow code.) - "redis": ["txredisapi>=1.4.7", "hiredis"], - # Required to use experimental `caches.track_memory_usage` config option. - "cache_memory": ["pympler"], -} - -ALL_OPTIONAL_REQUIREMENTS: Set[str] = set() - -for name, optional_deps in CONDITIONAL_REQUIREMENTS.items(): - # Exclude systemd as it's a system-based requirement. - # Exclude lint as it's a dev-based requirement. - if name not in ["systemd"]: - ALL_OPTIONAL_REQUIREMENTS = set(optional_deps) | ALL_OPTIONAL_REQUIREMENTS - - -# ensure there are no double-quote characters in any of the deps (otherwise the -# 'pip install' incantation in DependencyException will break) -for dep in itertools.chain( - REQUIREMENTS, - *CONDITIONAL_REQUIREMENTS.values(), -): - if '"' in dep: - raise Exception( - "Dependency `%s` contains double-quote; use single-quotes instead" % (dep,) - ) - - -def list_requirements(): - return list(set(REQUIREMENTS) | ALL_OPTIONAL_REQUIREMENTS) - - -if __name__ == "__main__": - import sys - - sys.stdout.writelines(req + "\n" for req in list_requirements()) From 09b4f6e46d3cbc3d93c52a7b76db17f178167b3e Mon Sep 17 00:00:00 2001 From: David Robertson Date: Wed, 20 Apr 2022 19:16:49 +0100 Subject: [PATCH 51/60] Remove leftover references to setup.py (#12514) * Remove leftover references to setup.py Missed in #12478. * Changelog --- changelog.d/12514.misc | 1 + mypy.ini | 4 ++-- scripts-dev/lint.sh | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 changelog.d/12514.misc diff --git a/changelog.d/12514.misc b/changelog.d/12514.misc new file mode 100644 index 0000000000..061a604a1e --- /dev/null +++ b/changelog.d/12514.misc @@ -0,0 +1 @@ +Use poetry-core instead of setuptools to build wheels. diff --git a/mypy.ini b/mypy.ini index b2c3b1524c..a663bf6975 100644 --- a/mypy.ini +++ b/mypy.ini @@ -233,8 +233,8 @@ disallow_untyped_defs = True ;; The `typeshed` project maintains stubs here: ;; https://github.com/python/typeshed/tree/master/stubs ;; and for each package `foo` there's a corresponding `types-foo` package on PyPI, -;; which we can pull in as a dev dependency by adding to `setup.py`'s -;; `CONDITIONAL_REQUIREMENTS["mypy"]` list. +;; which we can pull in as a dev dependency by adding to `pyproject.toml`'s +;; `[tool.poetry.dev-dependencies]` list. [mypy-authlib.*] ignore_missing_imports = True diff --git a/scripts-dev/lint.sh b/scripts-dev/lint.sh index 91a704d982..377348b107 100755 --- a/scripts-dev/lint.sh +++ b/scripts-dev/lint.sh @@ -91,7 +91,7 @@ else files=( "synapse" "docker" "tests" "scripts-dev" - "contrib" "setup.py" "synmark" "stubs" ".ci" + "contrib" "synmark" "stubs" ".ci" ) fi fi From f5668f0b4a6cca659ae98d3cb3714692ba488e89 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Thu, 21 Apr 2022 07:42:03 +0100 Subject: [PATCH 52/60] Await un-partial-stating after a partial-state join (#12399) When we join a room via the faster-joins mechanism, we end up with "partial state" at some points on the event DAG. Many parts of the codebase need to wait for the full state to load. So, we implement a mechanism to keep track of which events have partial state, and wait for them to be fully-populated. --- changelog.d/12399.misc | 1 + docker/complement/conf/homeserver.yaml | 4 +- scripts-dev/complement.sh | 2 +- synapse/handlers/federation_event.py | 1 + .../storage/databases/main/events_worker.py | 10 +- synapse/storage/databases/main/state.py | 1 + synapse/storage/state.py | 28 +++- .../util/partial_state_events_tracker.py | 120 ++++++++++++++++++ tests/storage/util/__init__.py | 13 ++ .../util/test_partial_state_events_tracker.py | 117 +++++++++++++++++ 10 files changed, 291 insertions(+), 6 deletions(-) create mode 100644 changelog.d/12399.misc create mode 100644 synapse/storage/util/partial_state_events_tracker.py create mode 100644 tests/storage/util/__init__.py create mode 100644 tests/storage/util/test_partial_state_events_tracker.py diff --git a/changelog.d/12399.misc b/changelog.d/12399.misc new file mode 100644 index 0000000000..cd2e09626d --- /dev/null +++ b/changelog.d/12399.misc @@ -0,0 +1 @@ +Preparation for faster-room-join work: Implement a tracking mechanism to allow functions to wait for full room state to arrive. diff --git a/docker/complement/conf/homeserver.yaml b/docker/complement/conf/homeserver.yaml index c9d6a31240..174f87f52e 100644 --- a/docker/complement/conf/homeserver.yaml +++ b/docker/complement/conf/homeserver.yaml @@ -103,8 +103,10 @@ experimental_features: spaces_enabled: true # Enable history backfilling support msc2716_enabled: true - # server-side support for partial state in /send_join + # server-side support for partial state in /send_join responses msc3706_enabled: true + # client-side support for partial state in /send_join responses + faster_joins: true # Enable jump to date endpoint msc3030_enabled: true diff --git a/scripts-dev/complement.sh b/scripts-dev/complement.sh index d34e9f3554..e0feba05fa 100755 --- a/scripts-dev/complement.sh +++ b/scripts-dev/complement.sh @@ -64,4 +64,4 @@ docker build -t $COMPLEMENT_BASE_IMAGE -f "docker/complement/$COMPLEMENT_DOCKERF # Run the tests! echo "Images built; running complement" cd "$COMPLEMENT_DIR" -go test -v -tags synapse_blacklist,msc2716,msc3030 -count=1 "$@" ./tests/... +go test -v -tags synapse_blacklist,msc2716,msc3030,faster_joins -count=1 "$@" ./tests/... diff --git a/synapse/handlers/federation_event.py b/synapse/handlers/federation_event.py index 32bf02818c..693b544286 100644 --- a/synapse/handlers/federation_event.py +++ b/synapse/handlers/federation_event.py @@ -515,6 +515,7 @@ class FederationEventHandler: ) return await self._store.update_state_for_partial_state_event(event, context) + self._state_store.notify_event_un_partial_stated(event.event_id) async def backfill( self, dest: str, room_id: str, limit: int, extremities: Collection[str] diff --git a/synapse/storage/databases/main/events_worker.py b/synapse/storage/databases/main/events_worker.py index 60876204bd..6d6e146ff1 100644 --- a/synapse/storage/databases/main/events_worker.py +++ b/synapse/storage/databases/main/events_worker.py @@ -1974,7 +1974,15 @@ class EventsWorkerStore(SQLBaseStore): async def get_partial_state_events( self, event_ids: Collection[str] ) -> Dict[str, bool]: - """Checks which of the given events have partial state""" + """Checks which of the given events have partial state + + Args: + event_ids: the events we want to check for partial state. + + Returns: + a dict mapping from event id to partial-stateness. We return True for + any of the events which are unknown (or are outliers). + """ result = await self.db_pool.simple_select_many_batch( table="partial_state_events", column="event_id", diff --git a/synapse/storage/databases/main/state.py b/synapse/storage/databases/main/state.py index 7a1b013fa3..e653841fe5 100644 --- a/synapse/storage/databases/main/state.py +++ b/synapse/storage/databases/main/state.py @@ -396,6 +396,7 @@ class StateGroupWorkerStore(EventsWorkerStore, SQLBaseStore): ) # TODO(faster_joins): need to do something about workers here + txn.call_after(self.is_partial_state_event.invalidate, (event.event_id,)) txn.call_after( self._get_state_group_for_event.prefill, (event.event_id,), diff --git a/synapse/storage/state.py b/synapse/storage/state.py index cda194e8c8..d1d5859214 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -31,6 +31,7 @@ from frozendict import frozendict from synapse.api.constants import EventTypes from synapse.events import EventBase +from synapse.storage.util.partial_state_events_tracker import PartialStateEventsTracker from synapse.types import MutableStateMap, StateKey, StateMap if TYPE_CHECKING: @@ -542,6 +543,10 @@ class StateGroupStorage: def __init__(self, hs: "HomeServer", stores: "Databases"): self.stores = stores + self._partial_state_events_tracker = PartialStateEventsTracker(stores.main) + + def notify_event_un_partial_stated(self, event_id: str) -> None: + self._partial_state_events_tracker.notify_un_partial_stated(event_id) async def get_state_group_delta( self, state_group: int @@ -579,7 +584,7 @@ class StateGroupStorage: if not event_ids: return {} - event_to_groups = await self.stores.main._get_state_group_for_events(event_ids) + event_to_groups = await self._get_state_group_for_events(event_ids) groups = set(event_to_groups.values()) group_to_state = await self.stores.state._get_state_for_groups(groups) @@ -668,7 +673,7 @@ class StateGroupStorage: RuntimeError if we don't have a state group for one or more of the events (ie they are outliers or unknown) """ - event_to_groups = await self.stores.main._get_state_group_for_events(event_ids) + event_to_groups = await self._get_state_group_for_events(event_ids) groups = set(event_to_groups.values()) group_to_state = await self.stores.state._get_state_for_groups( @@ -709,7 +714,7 @@ class StateGroupStorage: RuntimeError if we don't have a state group for one or more of the events (ie they are outliers or unknown) """ - event_to_groups = await self.stores.main._get_state_group_for_events(event_ids) + event_to_groups = await self._get_state_group_for_events(event_ids) groups = set(event_to_groups.values()) group_to_state = await self.stores.state._get_state_for_groups( @@ -785,6 +790,23 @@ class StateGroupStorage: groups, state_filter or StateFilter.all() ) + async def _get_state_group_for_events( + self, + event_ids: Collection[str], + await_full_state: bool = True, + ) -> Mapping[str, int]: + """Returns mapping event_id -> state_group + + Args: + event_ids: events to get state groups for + await_full_state: if true, will block if we do not yet have complete + state at this event. + """ + if await_full_state: + await self._partial_state_events_tracker.await_full_state(event_ids) + + return await self.stores.main._get_state_group_for_events(event_ids) + async def store_state_group( self, event_id: str, diff --git a/synapse/storage/util/partial_state_events_tracker.py b/synapse/storage/util/partial_state_events_tracker.py new file mode 100644 index 0000000000..a61a951ef0 --- /dev/null +++ b/synapse/storage/util/partial_state_events_tracker.py @@ -0,0 +1,120 @@ +# Copyright 2022 The Matrix.org Foundation C.I.C. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +from collections import defaultdict +from typing import Collection, Dict, Set + +from twisted.internet import defer +from twisted.internet.defer import Deferred + +from synapse.logging.context import PreserveLoggingContext, make_deferred_yieldable +from synapse.storage.databases.main.events_worker import EventsWorkerStore +from synapse.util import unwrapFirstError + +logger = logging.getLogger(__name__) + + +class PartialStateEventsTracker: + """Keeps track of which events have partial state, after a partial-state join""" + + def __init__(self, store: EventsWorkerStore): + self._store = store + # a map from event id to a set of Deferreds which are waiting for that event to be + # un-partial-stated. + self._observers: Dict[str, Set[Deferred[None]]] = defaultdict(set) + + def notify_un_partial_stated(self, event_id: str) -> None: + """Notify that we now have full state for a given event + + Called by the state-resynchronization loop whenever we resynchronize the state + for a particular event. Unblocks any callers to await_full_state() for that + event. + + Args: + event_id: the event that now has full state. + """ + observers = self._observers.pop(event_id, None) + if not observers: + return + logger.info( + "Notifying %i things waiting for un-partial-stating of event %s", + len(observers), + event_id, + ) + with PreserveLoggingContext(): + for o in observers: + o.callback(None) + + async def await_full_state(self, event_ids: Collection[str]) -> None: + """Wait for all the given events to have full state. + + Args: + event_ids: the list of event ids that we want full state for + """ + # first try the happy path: if there are no partial-state events, we can return + # quickly + partial_state_event_ids = [ + ev + for ev, p in (await self._store.get_partial_state_events(event_ids)).items() + if p + ] + + if not partial_state_event_ids: + return + + logger.info( + "Awaiting un-partial-stating of events %s", + partial_state_event_ids, + stack_info=True, + ) + + # create an observer for each lazy-joined event + observers: Dict[str, Deferred[None]] = { + event_id: Deferred() for event_id in partial_state_event_ids + } + for event_id, observer in observers.items(): + self._observers[event_id].add(observer) + + try: + # some of them may have been un-lazy-joined between us checking the db and + # registering the observer, in which case we'd wait forever for the + # notification. Call back the observers now. + for event_id, partial in ( + await self._store.get_partial_state_events(observers.keys()) + ).items(): + # there may have been a call to notify_un_partial_stated during the + # db query, so the observers may already have been called. + if not partial and not observers[event_id].called: + observers[event_id].callback(None) + + await make_deferred_yieldable( + defer.gatherResults( + observers.values(), + consumeErrors=True, + ) + ).addErrback(unwrapFirstError) + logger.info("Events %s all un-partial-stated", observers.keys()) + finally: + # remove any observers we created. This should happen when the notification + # is received, but that might not happen for two reasons: + # (a) we're bailing out early on an exception (including us being + # cancelled during the await) + # (b) the event got de-lazy-joined before we set up the observer. + for event_id, observer in observers.items(): + observer_set = self._observers.get(event_id) + if observer_set: + observer_set.discard(observer) + if not observer_set: + del self._observers[event_id] diff --git a/tests/storage/util/__init__.py b/tests/storage/util/__init__.py new file mode 100644 index 0000000000..3a5f22c022 --- /dev/null +++ b/tests/storage/util/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2022 The Matrix.org Foundation C.I.C. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/storage/util/test_partial_state_events_tracker.py b/tests/storage/util/test_partial_state_events_tracker.py new file mode 100644 index 0000000000..303e190b6c --- /dev/null +++ b/tests/storage/util/test_partial_state_events_tracker.py @@ -0,0 +1,117 @@ +# Copyright 2022 The Matrix.org Foundation C.I.C. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Dict +from unittest import mock + +from twisted.internet.defer import CancelledError, ensureDeferred + +from synapse.storage.util.partial_state_events_tracker import PartialStateEventsTracker + +from tests.unittest import TestCase + + +class PartialStateEventsTrackerTestCase(TestCase): + def setUp(self) -> None: + # the results to be returned by the mocked get_partial_state_events + self._events_dict: Dict[str, bool] = {} + + async def get_partial_state_events(events): + return {e: self._events_dict[e] for e in events} + + self.mock_store = mock.Mock(spec_set=["get_partial_state_events"]) + self.mock_store.get_partial_state_events.side_effect = get_partial_state_events + + self.tracker = PartialStateEventsTracker(self.mock_store) + + def test_does_not_block_for_full_state_events(self): + self._events_dict = {"event1": False, "event2": False} + + self.successResultOf( + ensureDeferred(self.tracker.await_full_state(["event1", "event2"])) + ) + + self.mock_store.get_partial_state_events.assert_called_once_with( + ["event1", "event2"] + ) + + def test_blocks_for_partial_state_events(self): + self._events_dict = {"event1": True, "event2": False} + + d = ensureDeferred(self.tracker.await_full_state(["event1", "event2"])) + + # there should be no result yet + self.assertNoResult(d) + + # notifying that the event has been de-partial-stated should unblock + self.tracker.notify_un_partial_stated("event1") + self.successResultOf(d) + + def test_un_partial_state_race(self): + # if the event is un-partial-stated between the initial check and the + # registration of the listener, it should not block. + self._events_dict = {"event1": True, "event2": False} + + async def get_partial_state_events(events): + res = {e: self._events_dict[e] for e in events} + # change the result for next time + self._events_dict = {"event1": False, "event2": False} + return res + + self.mock_store.get_partial_state_events.side_effect = get_partial_state_events + + self.successResultOf( + ensureDeferred(self.tracker.await_full_state(["event1", "event2"])) + ) + + def test_un_partial_state_during_get_partial_state_events(self): + # we should correctly handle a call to notify_un_partial_stated during the + # second call to get_partial_state_events. + + self._events_dict = {"event1": True, "event2": False} + + async def get_partial_state_events1(events): + self.mock_store.get_partial_state_events.side_effect = ( + get_partial_state_events2 + ) + return {e: self._events_dict[e] for e in events} + + async def get_partial_state_events2(events): + self.tracker.notify_un_partial_stated("event1") + self._events_dict["event1"] = False + return {e: self._events_dict[e] for e in events} + + self.mock_store.get_partial_state_events.side_effect = get_partial_state_events1 + + self.successResultOf( + ensureDeferred(self.tracker.await_full_state(["event1", "event2"])) + ) + + def test_cancellation(self): + self._events_dict = {"event1": True, "event2": False} + + d1 = ensureDeferred(self.tracker.await_full_state(["event1", "event2"])) + self.assertNoResult(d1) + + d2 = ensureDeferred(self.tracker.await_full_state(["event1"])) + self.assertNoResult(d2) + + d1.cancel() + self.assertFailure(d1, CancelledError) + + # d2 should still be waiting! + self.assertNoResult(d2) + + self.tracker.notify_un_partial_stated("event1") + self.successResultOf(d2) From f46b223354895efee181507a96a3a9a72c933a4b Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Fri, 22 Apr 2022 14:23:40 +0100 Subject: [PATCH 53/60] turn-howto: fix some links --- docs/turn-howto.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/turn-howto.md b/docs/turn-howto.md index 3a2cd04e36..37a311ad9c 100644 --- a/docs/turn-howto.md +++ b/docs/turn-howto.md @@ -302,14 +302,14 @@ Here are a few things to try: (Understanding the output is beyond the scope of this document!) - * You can test your Matrix homeserver TURN setup with https://test.voip.librepush.net/. + * You can test your Matrix homeserver TURN setup with . Note that this test is not fully reliable yet, so don't be discouraged if the test fails. [Here](https://github.com/matrix-org/voip-tester) is the github repo of the source of the tester, where you can file bug reports. * There is a WebRTC test tool at - https://webrtc.github.io/samples/src/content/peerconnection/trickle-ice/. To + . To use it, you will need a username/password for your TURN server. You can either: From b82fff66df6ea050d4d8da880c2ea2840e3fce0d Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Fri, 22 Apr 2022 16:03:46 +0100 Subject: [PATCH 54/60] MSC3202: Fix device_unused_fallback_keys -> device_unused_fallback_key_types (#12520) * Fix device_unused_fallback_keys -> device_unused_fallback_key_types * changelog --- changelog.d/12520.bugfix | 1 + synapse/appservice/__init__.py | 2 +- synapse/appservice/api.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 changelog.d/12520.bugfix diff --git a/changelog.d/12520.bugfix b/changelog.d/12520.bugfix new file mode 100644 index 0000000000..c73005fde8 --- /dev/null +++ b/changelog.d/12520.bugfix @@ -0,0 +1 @@ +Fix a bug in the implementation of MSC3202 where Synapse would use the field name `device_unused_fallback_keys`, rather than `device_unused_fallback_key_types`. \ No newline at end of file diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py index d23d9221bc..a610fb785d 100644 --- a/synapse/appservice/__init__.py +++ b/synapse/appservice/__init__.py @@ -42,7 +42,7 @@ logger = logging.getLogger(__name__) # user ID -> {device ID -> {algorithm -> count}} TransactionOneTimeKeyCounts = Dict[str, Dict[str, Dict[str, int]]] -# Type for the `device_unused_fallback_keys` field in an appservice transaction +# Type for the `device_unused_fallback_key_types` field in an appservice transaction # user ID -> {device ID -> [algorithm]} TransactionUnusedFallbackKeys = Dict[str, Dict[str, List[str]]] diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index 0cdbb04bfb..adc6b074da 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -278,7 +278,7 @@ class ApplicationServiceApi(SimpleHttpClient): ] = one_time_key_counts if unused_fallback_keys: body[ - "org.matrix.msc3202.device_unused_fallback_keys" + "org.matrix.msc3202.device_unused_fallback_key_types" ] = unused_fallback_keys if device_list_summary: body["org.matrix.msc3202.device_lists"] = { From a50fb411b366576c3d4796b04c45385bdb7a6897 Mon Sep 17 00:00:00 2001 From: Sean Quah <8349537+squahtx@users.noreply.github.com> Date: Fri, 22 Apr 2022 18:20:06 +0100 Subject: [PATCH 55/60] Update `delay_cancellation` to accept any awaitable (#12468) This will mainly be useful when dealing with module callbacks, which are all typed as returning `Awaitable`s instead of coroutines or `Deferred`s. Signed-off-by: Sean Quah --- changelog.d/12468.misc | 1 + synapse/storage/database.py | 3 +- synapse/util/async_helpers.py | 52 ++++++++++++++++++++++++++------ tests/util/test_async_helpers.py | 33 ++++++++++++++++++-- 4 files changed, 75 insertions(+), 14 deletions(-) create mode 100644 changelog.d/12468.misc diff --git a/changelog.d/12468.misc b/changelog.d/12468.misc new file mode 100644 index 0000000000..3d5d25247f --- /dev/null +++ b/changelog.d/12468.misc @@ -0,0 +1 @@ +Update `delay_cancellation` to accept any awaitable, rather than just `Deferred`s. diff --git a/synapse/storage/database.py b/synapse/storage/database.py index 5eb545c86e..df1e9c1b83 100644 --- a/synapse/storage/database.py +++ b/synapse/storage/database.py @@ -41,7 +41,6 @@ from prometheus_client import Histogram from typing_extensions import Literal from twisted.enterprise import adbapi -from twisted.internet import defer from synapse.api.errors import StoreError from synapse.config.database import DatabaseConnectionConfig @@ -794,7 +793,7 @@ class DatabasePool: # We also wait until everything above is done before releasing the # `CancelledError`, so that logging contexts won't get used after they have been # finished. - return await delay_cancellation(defer.ensureDeferred(_runInteraction())) + return await delay_cancellation(_runInteraction()) async def runWithConnection( self, diff --git a/synapse/util/async_helpers.py b/synapse/util/async_helpers.py index 650e44de22..e27c5d298f 100644 --- a/synapse/util/async_helpers.py +++ b/synapse/util/async_helpers.py @@ -14,6 +14,7 @@ # limitations under the License. import abc +import asyncio import collections import inspect import itertools @@ -25,6 +26,7 @@ from typing import ( Awaitable, Callable, Collection, + Coroutine, Dict, Generic, Hashable, @@ -701,27 +703,57 @@ def stop_cancellation(deferred: "defer.Deferred[T]") -> "defer.Deferred[T]": return new_deferred -def delay_cancellation(deferred: "defer.Deferred[T]") -> "defer.Deferred[T]": - """Delay cancellation of a `Deferred` until it resolves. +@overload +def delay_cancellation(awaitable: "defer.Deferred[T]") -> "defer.Deferred[T]": + ... + + +@overload +def delay_cancellation(awaitable: Coroutine[Any, Any, T]) -> "defer.Deferred[T]": + ... + + +@overload +def delay_cancellation(awaitable: Awaitable[T]) -> Awaitable[T]: + ... + + +def delay_cancellation(awaitable: Awaitable[T]) -> Awaitable[T]: + """Delay cancellation of a coroutine or `Deferred` awaitable until it resolves. Has the same effect as `stop_cancellation`, but the returned `Deferred` will not - resolve with a `CancelledError` until the original `Deferred` resolves. + resolve with a `CancelledError` until the original awaitable resolves. Args: - deferred: The `Deferred` to protect against cancellation. May optionally follow - the Synapse logcontext rules. + deferred: The coroutine or `Deferred` to protect against cancellation. May + optionally follow the Synapse logcontext rules. Returns: - A new `Deferred`, which will contain the result of the original `Deferred`. - The new `Deferred` will not propagate cancellation through to the original. - When cancelled, the new `Deferred` will wait until the original `Deferred` - resolves before failing with a `CancelledError`. + A new `Deferred`, which will contain the result of the original coroutine or + `Deferred`. The new `Deferred` will not propagate cancellation through to the + original coroutine or `Deferred`. - The new `Deferred` will follow the Synapse logcontext rules if `deferred` + When cancelled, the new `Deferred` will wait until the original coroutine or + `Deferred` resolves before failing with a `CancelledError`. + + The new `Deferred` will follow the Synapse logcontext rules if `awaitable` follows the Synapse logcontext rules. Otherwise the new `Deferred` should be wrapped with `make_deferred_yieldable`. """ + # First, convert the awaitable into a `Deferred`. + if isinstance(awaitable, defer.Deferred): + deferred = awaitable + elif asyncio.iscoroutine(awaitable): + # Ideally we'd use `Deferred.fromCoroutine()` here, to save on redundant + # type-checking, but we'd need Twisted >= 21.2. + deferred = defer.ensureDeferred(awaitable) + else: + # We have no idea what to do with this awaitable. + # We assume it's already resolved, such as `DoneAwaitable`s or `Future`s from + # `make_awaitable`, and let the caller `await` it normally. + return awaitable + def handle_cancel(new_deferred: "defer.Deferred[T]") -> None: # before the new deferred is cancelled, we `pause` it to stop the cancellation # propagating. we then `unpause` it once the wrapped deferred completes, to diff --git a/tests/util/test_async_helpers.py b/tests/util/test_async_helpers.py index e5bc416de1..daacc54c72 100644 --- a/tests/util/test_async_helpers.py +++ b/tests/util/test_async_helpers.py @@ -382,7 +382,7 @@ class StopCancellationTests(TestCase): class DelayCancellationTests(TestCase): """Tests for the `delay_cancellation` function.""" - def test_cancellation(self): + def test_deferred_cancellation(self): """Test that cancellation of the new `Deferred` waits for the original.""" deferred: "Deferred[str]" = Deferred() wrapper_deferred = delay_cancellation(deferred) @@ -403,6 +403,35 @@ class DelayCancellationTests(TestCase): # Now that the original `Deferred` has failed, we should get a `CancelledError`. self.failureResultOf(wrapper_deferred, CancelledError) + def test_coroutine_cancellation(self): + """Test that cancellation of the new `Deferred` waits for the original.""" + blocking_deferred: "Deferred[None]" = Deferred() + completion_deferred: "Deferred[None]" = Deferred() + + async def task(): + await blocking_deferred + completion_deferred.callback(None) + # Raise an exception. Twisted should consume it, otherwise unwanted + # tracebacks will be printed in logs. + raise ValueError("abc") + + wrapper_deferred = delay_cancellation(task()) + + # Cancel the new `Deferred`. + wrapper_deferred.cancel() + self.assertNoResult(wrapper_deferred) + self.assertFalse( + blocking_deferred.called, "Cancellation was propagated too deep" + ) + self.assertFalse(completion_deferred.called) + + # Unblock the task. + blocking_deferred.callback(None) + self.assertTrue(completion_deferred.called) + + # Now that the original coroutine has failed, we should get a `CancelledError`. + self.failureResultOf(wrapper_deferred, CancelledError) + def test_suppresses_second_cancellation(self): """Test that a second cancellation is suppressed. @@ -451,7 +480,7 @@ class DelayCancellationTests(TestCase): async def outer(): with LoggingContext("c") as c: try: - await delay_cancellation(defer.ensureDeferred(inner())) + await delay_cancellation(inner()) self.fail("`CancelledError` was not raised") except CancelledError: self.assertEqual(c, current_context()) From a36a38b1ca987c39eb70d9bf5ea4bd6999876a78 Mon Sep 17 00:00:00 2001 From: Dirk Klimpel <5740567+dklimpel@users.noreply.github.com> Date: Mon, 25 Apr 2022 14:17:03 +0200 Subject: [PATCH 56/60] Add some example configurations for worker (#12492) Signed-off-by: Dirk Klimpel --- changelog.d/12492.doc | 1 + .../workers/background_worker.yaml | 8 ++++ .../workers/event_persister.yaml | 23 ++++++++++ .../workers/generic_worker.yaml | 3 +- docs/workers.md | 45 +++++++++---------- 5 files changed, 56 insertions(+), 24 deletions(-) create mode 100644 changelog.d/12492.doc create mode 100644 docs/systemd-with-workers/workers/background_worker.yaml create mode 100644 docs/systemd-with-workers/workers/event_persister.yaml diff --git a/changelog.d/12492.doc b/changelog.d/12492.doc new file mode 100644 index 0000000000..4a3e2f4f06 --- /dev/null +++ b/changelog.d/12492.doc @@ -0,0 +1 @@ +Add some example configurations for workers and update architectural diagram. diff --git a/docs/systemd-with-workers/workers/background_worker.yaml b/docs/systemd-with-workers/workers/background_worker.yaml new file mode 100644 index 0000000000..9fbfbda7db --- /dev/null +++ b/docs/systemd-with-workers/workers/background_worker.yaml @@ -0,0 +1,8 @@ +worker_app: synapse.app.generic_worker +worker_name: background_worker + +# The replication listener on the main synapse process. +worker_replication_host: 127.0.0.1 +worker_replication_http_port: 9093 + +worker_log_config: /etc/matrix-synapse/background-worker-log.yaml diff --git a/docs/systemd-with-workers/workers/event_persister.yaml b/docs/systemd-with-workers/workers/event_persister.yaml new file mode 100644 index 0000000000..9bc6997bad --- /dev/null +++ b/docs/systemd-with-workers/workers/event_persister.yaml @@ -0,0 +1,23 @@ +worker_app: synapse.app.generic_worker +worker_name: event_persister1 + +# The replication listener on the main synapse process. +worker_replication_host: 127.0.0.1 +worker_replication_http_port: 9093 + +worker_listeners: + - type: http + port: 8034 + resources: + - names: [replication] + + # Enable listener if this stream writer handles endpoints for the `typing` or + # `to_device` streams. Uses a different port to the `replication` listener to + # avoid exposing the `replication` listener publicly. + # + #- type: http + # port: 8035 + # resources: + # - names: [client] + +worker_log_config: /etc/matrix-synapse/event-persister-log.yaml diff --git a/docs/systemd-with-workers/workers/generic_worker.yaml b/docs/systemd-with-workers/workers/generic_worker.yaml index 8561e2cda5..a82f9c161f 100644 --- a/docs/systemd-with-workers/workers/generic_worker.yaml +++ b/docs/systemd-with-workers/workers/generic_worker.yaml @@ -1,12 +1,13 @@ worker_app: synapse.app.generic_worker worker_name: generic_worker1 +# The replication listener on the main synapse process. worker_replication_host: 127.0.0.1 worker_replication_http_port: 9093 worker_listeners: - type: http - port: 8011 + port: 8083 resources: - names: [client, federation] diff --git a/docs/workers.md b/docs/workers.md index 858411b15e..afdcd785e4 100644 --- a/docs/workers.md +++ b/docs/workers.md @@ -138,20 +138,7 @@ as the `listeners` option in the shared config. For example: ```yaml -worker_app: synapse.app.generic_worker -worker_name: worker1 - -# The replication listener on the main synapse process. -worker_replication_host: 127.0.0.1 -worker_replication_http_port: 9093 - -worker_listeners: - - type: http - port: 8083 - resources: - - names: [client, federation] - -worker_log_config: /home/matrix/synapse/config/worker1_log_config.yaml +{{#include systemd-with-workers/workers/generic_worker.yaml}} ``` ...is a full configuration for a generic worker instance, which will expose a @@ -363,6 +350,12 @@ stream_writers: events: event_persister1 ``` +An example for a stream writer instance: + +```yaml +{{#include systemd-with-workers/workers/event_persister.yaml}} +``` + Some of the streams have associated endpoints which, for maximum efficiency, should be routed to the workers handling that stream. See below for the currently supported streams and the endpoints associated with them: @@ -436,6 +429,12 @@ run_background_tasks_on: background_worker You might also wish to investigate the `update_user_directory` and `media_instance_running_background_jobs` settings. +An example for a dedicated background worker instance: + +```yaml +{{#include systemd-with-workers/workers/background_worker.yaml}} +``` + ### `synapse.app.pusher` Handles sending push notifications to sygnal and email. Doesn't handle any @@ -615,14 +614,14 @@ The following shows an example setup using Redis and a reverse proxy: | Main | | Generic | | Generic | | Event | | Process | | Worker 1 | | Worker 2 | | Persister | +--------------+ +--------------+ +--------------+ +--------------+ - ^ ^ | ^ | | ^ | ^ ^ - | | | | | | | | | | - | | | | | HTTP | | | | | - | +----------+<--|---|---------+ | | | | - | | +-------------|-->+----------+ | - | | | | - | | | | - v v v v -==================================================================== + ^ ^ | ^ | | ^ | | ^ ^ + | | | | | | | | | | | + | | | | | HTTP | | | | | | + | +----------+<--|---|---------+<--|---|---------+ | | + | | +-------------|-->+-------------+ | + | | | | + | | | | + v v v v +====================================================================== Redis pub/sub channel ``` From d9b71410c28320f67e4816257e56ac44492ffa28 Mon Sep 17 00:00:00 2001 From: villepeh <100730729+villepeh@users.noreply.github.com> Date: Mon, 25 Apr 2022 15:18:18 +0300 Subject: [PATCH 57/60] Add HAProxy delegation example to docs (#12501) Signed-off-by: Ville Petteri Huh --- changelog.d/12501.doc | 1 + docs/reverse_proxy.md | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+) create mode 100644 changelog.d/12501.doc diff --git a/changelog.d/12501.doc b/changelog.d/12501.doc new file mode 100644 index 0000000000..278193a69a --- /dev/null +++ b/changelog.d/12501.doc @@ -0,0 +1 @@ +Add HAProxy delegation example with CORS headers to docs. diff --git a/docs/reverse_proxy.md b/docs/reverse_proxy.md index 5a0c847951..69caa8a73e 100644 --- a/docs/reverse_proxy.md +++ b/docs/reverse_proxy.md @@ -206,6 +206,28 @@ backend matrix server matrix 127.0.0.1:8008 ``` + +[Delegation](delegate.md) example: +``` +frontend https + acl matrix-well-known-client-path path /.well-known/matrix/client + acl matrix-well-known-server-path path /.well-known/matrix/server + use_backend matrix-well-known-client if matrix-well-known-client-path + use_backend matrix-well-known-server if matrix-well-known-server-path + +backend matrix-well-known-client + http-after-response set-header Access-Control-Allow-Origin "*" + http-after-response set-header Access-Control-Allow-Methods "GET, POST, PUT, DELETE, OPTIONS" + http-after-response set-header Access-Control-Allow-Headers "Origin, X-Requested-With, Content-Type, Accept, Authorization" + http-request return status 200 content-type application/json string '{"m.homeserver":{"base_url":"https://matrix.example.com"},"m.identity_server":{"base_url":"https://identity.example.com"}}' + +backend matrix-well-known-server + http-after-response set-header Access-Control-Allow-Origin "*" + http-after-response set-header Access-Control-Allow-Methods "GET, POST, PUT, DELETE, OPTIONS" + http-after-response set-header Access-Control-Allow-Headers "Origin, X-Requested-With, Content-Type, Accept, Authorization" + http-request return status 200 content-type application/json string '{"m.server":"matrix.example.com:443"}' +``` + ### Relayd ``` From 185da8f0f2db8e4d502a904942cbd8a6840e27c8 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Mon, 25 Apr 2022 08:25:56 -0400 Subject: [PATCH 58/60] Misc. clean-ups to the relations code (#12519) * Corrects some typos / copy & paste errors in tests. * Clarifies docstrings. * Removes an unnecessary method. --- changelog.d/12519.misc | 1 + synapse/events/utils.py | 2 +- synapse/handlers/relations.py | 94 ++++++++++------------------- tests/rest/client/test_relations.py | 74 +++++++++++------------ 4 files changed, 68 insertions(+), 103 deletions(-) create mode 100644 changelog.d/12519.misc diff --git a/changelog.d/12519.misc b/changelog.d/12519.misc new file mode 100644 index 0000000000..9c023d8e3e --- /dev/null +++ b/changelog.d/12519.misc @@ -0,0 +1 @@ +Refactor the relations code for clarity. diff --git a/synapse/events/utils.py b/synapse/events/utils.py index 2174b4a094..f8d3ba5456 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -479,9 +479,9 @@ class EventClientSerializer: Args: event: The event being serialized. time_now: The current time in milliseconds + config: Event serialization config aggregations: The bundled aggregation to serialize. serialized_event: The serialized event which may be modified. - config: Event serialization config apply_edits: Whether the content of the event should be modified to reflect any replacement in `aggregations.replace`. """ diff --git a/synapse/handlers/relations.py b/synapse/handlers/relations.py index 0be2319577..5efb561273 100644 --- a/synapse/handlers/relations.py +++ b/synapse/handlers/relations.py @@ -256,64 +256,6 @@ class RelationsHandler: return filtered_results - async def _get_bundled_aggregation_for_event( - self, event: EventBase, ignored_users: FrozenSet[str] - ) -> Optional[BundledAggregations]: - """Generate bundled aggregations for an event. - - Note that this does not use a cache, but depends on cached methods. - - Args: - event: The event to calculate bundled aggregations for. - ignored_users: The users ignored by the requesting user. - - Returns: - The bundled aggregations for an event, if bundled aggregations are - enabled and the event can have bundled aggregations. - """ - - # Do not bundle aggregations for an event which represents an edit or an - # annotation. It does not make sense for them to have related events. - relates_to = event.content.get("m.relates_to") - if isinstance(relates_to, (dict, frozendict)): - relation_type = relates_to.get("rel_type") - if relation_type in (RelationTypes.ANNOTATION, RelationTypes.REPLACE): - return None - - event_id = event.event_id - room_id = event.room_id - - # The bundled aggregations to include, a mapping of relation type to a - # type-specific value. Some types include the direct return type here - # while others need more processing during serialization. - aggregations = BundledAggregations() - - annotations = await self.get_annotations_for_event( - event_id, room_id, ignored_users=ignored_users - ) - if annotations: - aggregations.annotations = {"chunk": annotations} - - references, next_token = await self.get_relations_for_event( - event_id, - event, - room_id, - RelationTypes.REFERENCE, - ignored_users=ignored_users, - ) - if references: - aggregations.references = { - "chunk": [{"event_id": event.event_id} for event in references] - } - - if next_token: - aggregations.references["next_batch"] = await next_token.to_string( - self._main_store - ) - - # Store the bundled aggregations in the event metadata for later use. - return aggregations - async def get_threads_for_events( self, event_ids: Collection[str], user_id: str, ignored_users: FrozenSet[str] ) -> Dict[str, _ThreadAggregation]: @@ -435,11 +377,39 @@ class RelationsHandler: # Fetch other relations per event. for event in events_by_id.values(): - event_result = await self._get_bundled_aggregation_for_event( - event, ignored_users + # Do not bundle aggregations for an event which represents an edit or an + # annotation. It does not make sense for them to have related events. + relates_to = event.content.get("m.relates_to") + if isinstance(relates_to, (dict, frozendict)): + relation_type = relates_to.get("rel_type") + if relation_type in (RelationTypes.ANNOTATION, RelationTypes.REPLACE): + continue + + annotations = await self.get_annotations_for_event( + event.event_id, event.room_id, ignored_users=ignored_users ) - if event_result: - results[event.event_id] = event_result + if annotations: + results.setdefault( + event.event_id, BundledAggregations() + ).annotations = {"chunk": annotations} + + references, next_token = await self.get_relations_for_event( + event.event_id, + event, + event.room_id, + RelationTypes.REFERENCE, + ignored_users=ignored_users, + ) + if references: + aggregations = results.setdefault(event.event_id, BundledAggregations()) + aggregations.references = { + "chunk": [{"event_id": ev.event_id} for ev in references] + } + + if next_token: + aggregations.references["next_batch"] = await next_token.to_string( + self._main_store + ) # Fetch any edits (but not for redacted events). # diff --git a/tests/rest/client/test_relations.py b/tests/rest/client/test_relations.py index 65743cdf67..39667e3225 100644 --- a/tests/rest/client/test_relations.py +++ b/tests/rest/client/test_relations.py @@ -560,43 +560,6 @@ class RelationsTestCase(BaseRelationsTestCase): {"event_id": edit_event_id, "sender": self.user_id}, m_replace_dict ) - def test_edit_thread(self) -> None: - """Test that editing a thread works.""" - - # Create a thread and edit the last event. - channel = self._send_relation( - RelationTypes.THREAD, - "m.room.message", - content={"msgtype": "m.text", "body": "A threaded reply!"}, - ) - threaded_event_id = channel.json_body["event_id"] - - new_body = {"msgtype": "m.text", "body": "I've been edited!"} - self._send_relation( - RelationTypes.REPLACE, - "m.room.message", - content={"msgtype": "m.text", "body": "foo", "m.new_content": new_body}, - parent_id=threaded_event_id, - ) - - # Fetch the thread root, to get the bundled aggregation for the thread. - channel = self.make_request( - "GET", - f"/rooms/{self.room}/event/{self.parent_id}", - access_token=self.user_token, - ) - self.assertEqual(200, channel.code, channel.json_body) - - # We expect that the edit message appears in the thread summary in the - # unsigned relations section. - relations_dict = channel.json_body["unsigned"].get("m.relations") - self.assertIn(RelationTypes.THREAD, relations_dict) - - thread_summary = relations_dict[RelationTypes.THREAD] - self.assertIn("latest_event", thread_summary) - latest_event_in_thread = thread_summary["latest_event"] - self.assertEqual(latest_event_in_thread["content"]["body"], "I've been edited!") - def test_edit_edit(self) -> None: """Test that an edit cannot be edited.""" new_body = {"msgtype": "m.text", "body": "Initial edit"} @@ -1047,7 +1010,7 @@ class BundledAggregationsTestCase(BaseRelationsTestCase): channel = self._send_relation(RelationTypes.THREAD, "m.room.test") thread_2 = channel.json_body["event_id"] - def assert_annotations(bundled_aggregations: JsonDict) -> None: + def assert_thread(bundled_aggregations: JsonDict) -> None: self.assertEqual(2, bundled_aggregations.get("count")) self.assertTrue(bundled_aggregations.get("current_user_participated")) # The latest thread event has some fields that don't matter. @@ -1066,7 +1029,38 @@ class BundledAggregationsTestCase(BaseRelationsTestCase): bundled_aggregations.get("latest_event"), ) - self._test_bundled_aggregations(RelationTypes.THREAD, assert_annotations, 9) + self._test_bundled_aggregations(RelationTypes.THREAD, assert_thread, 9) + + def test_thread_edit_latest_event(self) -> None: + """Test that editing the latest event in a thread works.""" + + # Create a thread and edit the last event. + channel = self._send_relation( + RelationTypes.THREAD, + "m.room.message", + content={"msgtype": "m.text", "body": "A threaded reply!"}, + ) + threaded_event_id = channel.json_body["event_id"] + + new_body = {"msgtype": "m.text", "body": "I've been edited!"} + channel = self._send_relation( + RelationTypes.REPLACE, + "m.room.message", + content={"msgtype": "m.text", "body": "foo", "m.new_content": new_body}, + parent_id=threaded_event_id, + ) + + # Fetch the thread root, to get the bundled aggregation for the thread. + relations_dict = self._get_bundled_aggregations() + + # We expect that the edit message appears in the thread summary in the + # unsigned relations section. + self.assertIn(RelationTypes.THREAD, relations_dict) + + thread_summary = relations_dict[RelationTypes.THREAD] + self.assertIn("latest_event", thread_summary) + latest_event_in_thread = thread_summary["latest_event"] + self.assertEqual(latest_event_in_thread["content"]["body"], "I've been edited!") def test_aggregation_get_event_for_annotation(self) -> None: """Test that annotations do not get bundled aggregations included @@ -1093,7 +1087,7 @@ class BundledAggregationsTestCase(BaseRelationsTestCase): channel = self._send_relation(RelationTypes.THREAD, "m.room.test") thread_id = channel.json_body["event_id"] - # Annotate the annotation. + # Annotate the thread. self._send_relation( RelationTypes.ANNOTATION, "m.reaction", "a", parent_id=thread_id ) From 8bac3e043541003187e9f16598ff54b0327b9a04 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Mon, 25 Apr 2022 13:32:35 +0100 Subject: [PATCH 59/60] disallow-untyped-defs in `docker` and `stubs` directories (#12528) --- changelog.d/12528.misc | 1 + docker/configure_workers_and_start.py | 24 ++++++++-------- docker/start.py | 40 +++++++++++++++------------ stubs/sortedcontainers/sorteddict.pyi | 2 +- stubs/sortedcontainers/sortedlist.pyi | 6 ++-- stubs/sortedcontainers/sortedset.pyi | 2 +- stubs/txredisapi.pyi | 15 ++++++---- 7 files changed, 52 insertions(+), 38 deletions(-) create mode 100644 changelog.d/12528.misc diff --git a/changelog.d/12528.misc b/changelog.d/12528.misc new file mode 100644 index 0000000000..f64b5d24b0 --- /dev/null +++ b/changelog.d/12528.misc @@ -0,0 +1 @@ +Add type hints so `docker` and `stubs` directories pass `mypy --disallow-untyped-defs`. diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py index 23cac18e8d..3bda6c300b 100755 --- a/docker/configure_workers_and_start.py +++ b/docker/configure_workers_and_start.py @@ -29,7 +29,7 @@ import os import subprocess import sys -from typing import Any, Dict, Mapping, Set +from typing import Any, Dict, List, Mapping, MutableMapping, NoReturn, Set import jinja2 import yaml @@ -201,7 +201,7 @@ upstream {upstream_worker_type} {{ # Utility functions -def log(txt: str): +def log(txt: str) -> None: """Log something to the stdout. Args: @@ -210,7 +210,7 @@ def log(txt: str): print(txt) -def error(txt: str): +def error(txt: str) -> NoReturn: """Log something and exit with an error code. Args: @@ -220,7 +220,7 @@ def error(txt: str): sys.exit(2) -def convert(src: str, dst: str, **template_vars): +def convert(src: str, dst: str, **template_vars: object) -> None: """Generate a file from a template Args: @@ -290,7 +290,7 @@ def add_sharding_to_shared_config( shared_config.setdefault("media_instance_running_background_jobs", worker_name) -def generate_base_homeserver_config(): +def generate_base_homeserver_config() -> None: """Starts Synapse and generates a basic homeserver config, which will later be modified for worker support. @@ -302,12 +302,14 @@ def generate_base_homeserver_config(): subprocess.check_output(["/usr/local/bin/python", "/start.py", "migrate_config"]) -def generate_worker_files(environ, config_path: str, data_dir: str): +def generate_worker_files( + environ: Mapping[str, str], config_path: str, data_dir: str +) -> None: """Read the desired list of workers from environment variables and generate shared homeserver, nginx and supervisord configs. Args: - environ: _Environ[str] + environ: os.environ instance. config_path: The location of the generated Synapse main worker config file. data_dir: The location of the synapse data directory. Where log and user-facing config files live. @@ -369,13 +371,13 @@ def generate_worker_files(environ, config_path: str, data_dir: str): nginx_locations = {} # Read the desired worker configuration from the environment - worker_types = environ.get("SYNAPSE_WORKER_TYPES") - if worker_types is None: + worker_types_env = environ.get("SYNAPSE_WORKER_TYPES") + if worker_types_env is None: # No workers, just the main process worker_types = [] else: # Split type names by comma - worker_types = worker_types.split(",") + worker_types = worker_types_env.split(",") # Create the worker configuration directory if it doesn't already exist os.makedirs("/conf/workers", exist_ok=True) @@ -547,7 +549,7 @@ def generate_worker_log_config( return log_config_filepath -def main(args, environ): +def main(args: List[str], environ: MutableMapping[str, str]) -> None: config_dir = environ.get("SYNAPSE_CONFIG_DIR", "/data") config_path = environ.get("SYNAPSE_CONFIG_PATH", config_dir + "/homeserver.yaml") data_dir = environ.get("SYNAPSE_DATA_DIR", "/data") diff --git a/docker/start.py b/docker/start.py index ac62bbc8ba..4ac8f03477 100755 --- a/docker/start.py +++ b/docker/start.py @@ -6,27 +6,28 @@ import os import platform import subprocess import sys +from typing import Any, Dict, List, Mapping, MutableMapping, NoReturn, Optional import jinja2 # Utility functions -def log(txt): +def log(txt: str) -> None: print(txt, file=sys.stderr) -def error(txt): +def error(txt: str) -> NoReturn: log(txt) sys.exit(2) -def convert(src, dst, environ): +def convert(src: str, dst: str, environ: Mapping[str, object]) -> None: """Generate a file from a template Args: - src (str): path to input file - dst (str): path to file to write - environ (dict): environment dictionary, for replacement mappings. + src: path to input file + dst: path to file to write + environ: environment dictionary, for replacement mappings. """ with open(src) as infile: template = infile.read() @@ -35,25 +36,30 @@ def convert(src, dst, environ): outfile.write(rendered) -def generate_config_from_template(config_dir, config_path, environ, ownership): +def generate_config_from_template( + config_dir: str, + config_path: str, + os_environ: Mapping[str, str], + ownership: Optional[str], +) -> None: """Generate a homeserver.yaml from environment variables Args: - config_dir (str): where to put generated config files - config_path (str): where to put the main config file - environ (dict): environment dictionary - ownership (str|None): ":" string which will be used to set + config_dir: where to put generated config files + config_path: where to put the main config file + os_environ: environment mapping + ownership: ":" string which will be used to set ownership of the generated configs. If None, ownership will not change. """ for v in ("SYNAPSE_SERVER_NAME", "SYNAPSE_REPORT_STATS"): - if v not in environ: + if v not in os_environ: error( "Environment variable '%s' is mandatory when generating a config file." % (v,) ) # populate some params from data files (if they exist, else create new ones) - environ = environ.copy() + environ: Dict[str, Any] = dict(os_environ) secrets = { "registration": "SYNAPSE_REGISTRATION_SHARED_SECRET", "macaroon": "SYNAPSE_MACAROON_SECRET_KEY", @@ -127,12 +133,12 @@ def generate_config_from_template(config_dir, config_path, environ, ownership): subprocess.check_output(args) -def run_generate_config(environ, ownership): +def run_generate_config(environ: Mapping[str, str], ownership: Optional[str]) -> None: """Run synapse with a --generate-config param to generate a template config file Args: - environ (dict): env var dict - ownership (str|None): "userid:groupid" arg for chmod. If None, ownership will not change. + environ: env vars from `os.enrivon`. + ownership: "userid:groupid" arg for chmod. If None, ownership will not change. Never returns. """ @@ -178,7 +184,7 @@ def run_generate_config(environ, ownership): os.execv(sys.executable, args) -def main(args, environ): +def main(args: List[str], environ: MutableMapping[str, str]) -> None: mode = args[1] if len(args) > 1 else "run" # if we were given an explicit user to switch to, do so diff --git a/stubs/sortedcontainers/sorteddict.pyi b/stubs/sortedcontainers/sorteddict.pyi index 344d55cce1..e18d617281 100644 --- a/stubs/sortedcontainers/sorteddict.pyi +++ b/stubs/sortedcontainers/sorteddict.pyi @@ -103,7 +103,7 @@ class SortedDict(Dict[_KT, _VT]): self, start: Optional[int] = ..., stop: Optional[int] = ..., - reverse=bool, + reverse: bool = ..., ) -> Iterator[_KT]: ... def bisect_left(self, value: _KT) -> int: ... def bisect_right(self, value: _KT) -> int: ... diff --git a/stubs/sortedcontainers/sortedlist.pyi b/stubs/sortedcontainers/sortedlist.pyi index f80a3a72ce..403897e391 100644 --- a/stubs/sortedcontainers/sortedlist.pyi +++ b/stubs/sortedcontainers/sortedlist.pyi @@ -81,7 +81,7 @@ class SortedList(MutableSequence[_T]): self, start: Optional[int] = ..., stop: Optional[int] = ..., - reverse=bool, + reverse: bool = ..., ) -> Iterator[_T]: ... def _islice( self, @@ -153,14 +153,14 @@ class SortedKeyList(SortedList[_T]): maximum: Optional[int] = ..., inclusive: Tuple[bool, bool] = ..., reverse: bool = ..., - ): ... + ) -> Iterator[_T]: ... def irange_key( self, min_key: Optional[Any] = ..., max_key: Optional[Any] = ..., inclusive: Tuple[bool, bool] = ..., reserve: bool = ..., - ): ... + ) -> Iterator[_T]: ... def bisect_left(self, value: _T) -> int: ... def bisect_right(self, value: _T) -> int: ... def bisect(self, value: _T) -> int: ... diff --git a/stubs/sortedcontainers/sortedset.pyi b/stubs/sortedcontainers/sortedset.pyi index f9c2908386..43c860f422 100644 --- a/stubs/sortedcontainers/sortedset.pyi +++ b/stubs/sortedcontainers/sortedset.pyi @@ -103,7 +103,7 @@ class SortedSet(MutableSet[_T], Sequence[_T]): self, start: Optional[int] = ..., stop: Optional[int] = ..., - reverse=bool, + reverse: bool = ..., ) -> Iterator[_T]: ... def irange( self, diff --git a/stubs/txredisapi.pyi b/stubs/txredisapi.pyi index 2d8ca018fb..695a2307c2 100644 --- a/stubs/txredisapi.pyi +++ b/stubs/txredisapi.pyi @@ -18,6 +18,8 @@ from typing import Any, List, Optional, Type, Union from twisted.internet import protocol from twisted.internet.defer import Deferred +from twisted.internet.interfaces import IAddress +from twisted.python.failure import Failure class RedisProtocol(protocol.Protocol): def publish(self, channel: str, message: bytes) -> "Deferred[None]": ... @@ -34,11 +36,14 @@ class RedisProtocol(protocol.Protocol): def get(self, key: str) -> "Deferred[Any]": ... class SubscriberProtocol(RedisProtocol): - def __init__(self, *args, **kwargs): ... + def __init__(self, *args: object, **kwargs: object): ... password: Optional[str] - def subscribe(self, channels: Union[str, List[str]]): ... - def connectionMade(self): ... - def connectionLost(self, reason): ... + def subscribe(self, channels: Union[str, List[str]]) -> "Deferred[None]": ... + def connectionMade(self) -> None: ... + # type-ignore: twisted.internet.protocol.Protocol provides a default argument for + # `reason`. txredisapi's LineReceiver Protocol doesn't. But that's fine: it's what's + # actually specified in twisted.internet.interfaces.IProtocol. + def connectionLost(self, reason: Failure) -> None: ... # type: ignore[override] def lazyConnection( host: str = ..., @@ -74,7 +79,7 @@ class RedisFactory(protocol.ReconnectingClientFactory): replyTimeout: Optional[int] = None, convertNumbers: Optional[int] = True, ): ... - def buildProtocol(self, addr) -> RedisProtocol: ... + def buildProtocol(self, addr: IAddress) -> RedisProtocol: ... class SubscriberFactory(RedisFactory): def __init__(self) -> None: ... From 813d728d09cb9e3b66ae49ee894814618db9774b Mon Sep 17 00:00:00 2001 From: Sami Olmari Date: Mon, 25 Apr 2022 15:39:15 +0300 Subject: [PATCH 60/60] Correct typo in user_admin_api.md device deletion JSON (#12533) Signed-off-by: Sami Olmari --- changelog.d/12533.doc | 1 + docs/admin_api/user_admin_api.md | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/12533.doc diff --git a/changelog.d/12533.doc b/changelog.d/12533.doc new file mode 100644 index 0000000000..2c15488111 --- /dev/null +++ b/changelog.d/12533.doc @@ -0,0 +1 @@ +Remove extraneous comma in User Admin API's device deletion section so that the example JSON is actually valid and works. Contributed by @olmari. \ No newline at end of file diff --git a/docs/admin_api/user_admin_api.md b/docs/admin_api/user_admin_api.md index 4076fcab65..c8794299e7 100644 --- a/docs/admin_api/user_admin_api.md +++ b/docs/admin_api/user_admin_api.md @@ -804,7 +804,7 @@ POST /_synapse/admin/v2/users//delete_devices "devices": [ "QBUAZIFURK", "AUIECTSRND" - ], + ] } ```