Compare commits

...

669 Commits

Author SHA1 Message Date
Will Lewis d58dd1d1ff Update README.rst
(cherry picked from commit be65a8ec01)
2023-12-13 14:55:57 +00:00
Will Lewis f65f316bc3 Update README.rst 2023-12-13 14:47:55 +00:00
Erik Johnston 70c020b532 Update text 2023-12-12 20:32:48 +00:00
Patrick Cloke e1f8440c89 Update the README pointing to the Element fork. 2023-12-12 20:28:30 +00:00
Erik Johnston 15733b0931 Update changelog 2023-12-12 15:51:28 +00:00
Erik Johnston 128aad4fe3 1.98.0 2023-12-12 15:10:16 +00:00
Action Bot ee37039031 Version picker added for v1.98 docs 2023-12-11 14:51:26 +00:00
David Robertson 44377f5ac0
Revert postgres logical replication deltaas
This reverts two commits:

    0bb8e418a4
    "Fix postgres schema after dropping old tables (#16730)"

and

    51e4e35653
    "Add a Postgres `REPLICA IDENTITY` to tables that do not have an implicit one. This should allow use of Postgres logical replication. (take  2, now with no added deadlocks!) (#16658)"

and also amends the changelog.
2023-12-05 16:10:48 +00:00
David Robertson c8a24c55a9
Amend changelog typo 2023-12-05 13:38:09 +00:00
David Robertson 386649325a
Fixup dependency bumps syntax in changelog 2023-12-05 13:16:59 +00:00
David Robertson 3c83d8f0af
1.98.0rc1 2023-12-05 13:14:36 +00:00
David Robertson 0a00c99823
Fix upgrading a room without `events` field in power levels (#16725) 2023-12-05 12:06:21 +00:00
Amanda H. L. de Andrade Katz e87499b3f4
Add how to validate configuration file with synapse.config script (#16714) 2023-12-05 11:42:56 +00:00
Will Hunt ea783550bb
Set response values to zero if None for /_synapse/admin/v1/federation/destinations (#16729) 2023-12-05 11:40:27 +00:00
David Robertson 0bb8e418a4
Fix postgres schema after dropping old tables (#16730) 2023-12-05 11:08:40 +00:00
reivilibre 51e4e35653
Add a Postgres `REPLICA IDENTITY` to tables that do not have an implicit one. This should allow use of Postgres logical replication. (take 2, now with no added deadlocks!) (#16658)
* Add `ALTER TABLE ... REPLICA IDENTITY ...` for individual tables

We can't combine them into one file as it makes it likely to hit a deadlock

if Synapse is running, as it only takes one other transaction to access two

tables in a different order to the schema delta.

* Add notes

* Newsfile

Signed-off-by: Olivier Wilkinson (reivilibre) <oliverw@matrix.org>

* Re-introduce REPLICA IDENTITY test

---------

Signed-off-by: Olivier Wilkinson (reivilibre) <oliverw@matrix.org>
2023-12-04 14:57:28 +00:00
villepeh 0aa4d3b6f7
Switch UNIX socket paths to /run, and add a UNIX socket example for HAProxy (#16700) 2023-12-04 12:38:46 +00:00
dependabot[bot] 15c46cf86a
Bump phonenumbers from 8.13.23 to 8.13.26 (#16722)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-12-04 12:32:06 +00:00
Mathieu Velten 9e7f80037d
Server notices: add an autojoin setting for the notices room (#16699)
Co-authored-by: Patrick Cloke <clokep@users.noreply.github.com>
2023-12-04 12:31:42 +00:00
dependabot[bot] 506f5c7553
Bump matrix-org/netlify-pr-preview from 2 to 3 (#16719)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-12-04 12:02:54 +00:00
Travis Ralston d6e194b2bc
Implement MSC4069: Inhibit profile propagation (#16636)
MSC: https://github.com/matrix-org/matrix-spec-proposals/pull/4069
2023-12-04 11:36:12 +00:00
dependabot[bot] 2686a05766
Bump idna from 3.4 to 3.6 (#16720)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-12-04 11:34:53 +00:00
dependabot[bot] c915b91840
Bump cryptography from 41.0.6 to 41.0.7 (#16721)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-12-04 11:30:15 +00:00
dependabot[bot] dd02c6340e
Bump sphinx-autodoc2 from 0.4.2 to 0.5.0 (#16723)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-12-04 11:17:42 +00:00
dependabot[bot] a5c14346fa
Bump types-jsonschema from 4.19.0.4 to 4.20.0.0 (#16724)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-12-04 11:16:45 +00:00
Andrew Yasinishyn 63d96bfc61
ModuleAPI SSO auth callbacks (#15207)
Signed-off-by: Andrii Yasynyshyn yasinishyn.a.n@gmail.com
2023-12-01 14:31:50 +00:00
Patrick Cloke 579c6be5f6
Drop unused tables & unneeded access token ID for events. (#16522) 2023-12-01 10:12:00 +00:00
Mo Balaa 3a092699e5
Upgrade poetry-core range to fix issue with .so file (#16702)
poetry-core 1.8.x includes a fix which properly moves the generate
synapse_rust.abi3.so file to the synapse directory when using an
editable install.

Without this change developers are left with a confusing experience
of the synapse.synapse_rust module not being found after installation.
2023-11-29 15:46:43 -05:00
Patrick Cloke dcf949cd87
Declare support for Matrix v1.7, v1.8, and v1.9. (#16707) 2023-11-29 15:02:09 -05:00
Patrick Cloke d6c3b7584f
Request & follow redirects for /media/v3/download (#16701)
Implement MSC3860 to follow redirects for federated media downloads.

Note that the Client-Server API doesn't support this (yet) since the media
repository in Synapse doesn't have a way of supporting redirects.
2023-11-29 19:03:42 +00:00
Erik Johnston a14678492e
Reduce DB load when forget on leave setting is disabled (#16668)
* Reduce DB load when forget on leave setting is disabled

* Newsfile
2023-11-29 18:21:30 +00:00
Erik Johnston 19dac97480
Add a workflow to try and automatically fixup a PR (#16704)
Co-authored-by: Patrick Cloke <clokep@users.noreply.github.com>
2023-11-29 14:07:32 +00:00
Erik Johnston df366966b4
Speed up pruning of `user_ips` table (#16667)
Silly query planner
2023-11-29 11:54:42 +00:00
dependabot[bot] 6f2be7794e
Bump cryptography from 41.0.5 to 41.0.6 (#16703) 2023-11-28 19:57:48 -05:00
Erik Johnston 825ac7e6a1 Merge branch 'master' into develop 2023-11-28 16:35:11 +00:00
Patrick Cloke 77882b6a7d
Document which versions of Synapse have compatible schema versions. (#16661) 2023-11-28 11:01:24 -05:00
Erik Johnston d75d6d65d1 1.97.0 2023-11-28 14:09:21 +00:00
Mathieu Velten b0ed14d815
Ignore `encryption_enabled_by_default_for_room_type` for notices room (#16677) 2023-11-28 13:15:26 +00:00
Patrick Cloke d199b84006
Remove old full schema dumps. (#16697)
These are not useful and make it difficult to search for
table definitions, etc.
2023-11-28 07:28:07 -05:00
David Robertson 8751f0ef32
Fix poetry version typo in contributors' guide (#16695) 2023-11-27 15:16:20 +00:00
dependabot[bot] b3e8d503c7
Bump prometheus-client from 0.18.0 to 0.19.0 (#16691)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-11-27 11:27:27 +00:00
dependabot[bot] 62e96a2929
Bump pyasn1 from 0.5.0 to 0.5.1 (#16689)
Bumps [pyasn1](https://github.com/pyasn1/pyasn1) from 0.5.0 to 0.5.1.
- [Release notes](https://github.com/pyasn1/pyasn1/releases)
- [Changelog](https://github.com/pyasn1/pyasn1/blob/main/CHANGES.rst)
- [Commits](https://github.com/pyasn1/pyasn1/compare/v0.5.0...v0.5.1)

---
updated-dependencies:
- dependency-name: pyasn1
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-11-27 11:15:52 +00:00
dependabot[bot] 3238ae3aa0
Bump types-setuptools from 68.2.0.0 to 68.2.0.2 (#16688)
Bumps [types-setuptools](https://github.com/python/typeshed) from 68.2.0.0 to 68.2.0.2.
- [Commits](https://github.com/python/typeshed/commits)

---
updated-dependencies:
- dependency-name: types-setuptools
  dependency-type: direct:development
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-11-27 10:49:15 +00:00
dependabot[bot] 73794dd8c4
Bump ruff from 0.1.4 to 0.1.6 (#16690)
Bumps [ruff](https://github.com/astral-sh/ruff) from 0.1.4 to 0.1.6.
- [Release notes](https://github.com/astral-sh/ruff/releases)
- [Changelog](https://github.com/astral-sh/ruff/blob/main/CHANGELOG.md)
- [Commits](https://github.com/astral-sh/ruff/compare/v0.1.4...v0.1.6)

---
updated-dependencies:
- dependency-name: ruff
  dependency-type: direct:development
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-11-27 10:48:53 +00:00
dependabot[bot] 9a6181fb4e
Bump jsonschema from 4.19.1 to 4.20.0 (#16692)
Bumps [jsonschema](https://github.com/python-jsonschema/jsonschema) from 4.19.1 to 4.20.0.
- [Release notes](https://github.com/python-jsonschema/jsonschema/releases)
- [Changelog](https://github.com/python-jsonschema/jsonschema/blob/main/CHANGELOG.rst)
- [Commits](https://github.com/python-jsonschema/jsonschema/compare/v4.19.1...v4.20.0)

---
updated-dependencies:
- dependency-name: jsonschema
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-11-27 10:48:41 +00:00
dependabot[bot] 1c63dfedfd
Bump serde from 1.0.192 to 1.0.193 (#16693)
Bumps [serde](https://github.com/serde-rs/serde) from 1.0.192 to 1.0.193.
- [Release notes](https://github.com/serde-rs/serde/releases)
- [Commits](https://github.com/serde-rs/serde/compare/v1.0.192...v1.0.193)

---
updated-dependencies:
- dependency-name: serde
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-11-27 10:48:27 +00:00
David Robertson 0619c2bbd2
Move media retention tests out of rest tests (#16684)
* Move media retention tests out of rest tests

AFAICS this doesn't make any HTTP requests and so it ought not to belong
in `tests.rest`.

* Changelog
2023-11-27 01:29:46 +00:00
David Robertson c3627d0f99
Correctly read to-device stream pos on SQLite (#16682) 2023-11-24 13:42:38 +00:00
David Robertson 32a59a6495
Keep track of `user_ips` and `monthly_active_users` when delegating auth (#16672)
* Describe `insert_client_ip`
* Pull out client_ips and MAU tracking to BaseAuth
* Define HAS_AUTHLIB once in tests

sick of copypasting

* Track ips and token usage when delegating auth
* Test that we track MAU and user_ips
* Don't track `__oidc_admin`
2023-11-23 12:35:37 +00:00
Charles Wright 1a5f9bb651
Enable refreshable tokens on the admin registration endpoint (#16642)
Signed-off-by: Charles Wright <cvwright@futo.org>
2023-11-22 15:01:09 +00:00
V02460 f2430b16d1
Bump pyo3 (0.20), pythonize (0.20), pyo3-log (0.9) (#16673)
Signed-off-by: Kai A. Hiller <V02460@gmail.com>
2023-11-22 14:55:43 +00:00
Mathieu Velten c432d8f18f
Admin API for server notice: consistently bypass rate limits (#16670)
* Admin API for server notice: disable rate limit for all calls

* Add changelog

* Update changelog.d/16670.bugfix
2023-11-22 13:47:29 +00:00
dependabot[bot] c8118ba8c9
Bump pydantic from 2.4.2 to 2.5.1 (#16663)
Bumps [pydantic](https://github.com/pydantic/pydantic) from 2.4.2 to 2.5.1.
- [Release notes](https://github.com/pydantic/pydantic/releases)
- [Changelog](https://github.com/pydantic/pydantic/blob/main/HISTORY.md)
- [Commits](https://github.com/pydantic/pydantic/compare/v2.4.2...v2.5.1)

---
updated-dependencies:
- dependency-name: pydantic
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-11-22 13:14:00 +00:00
Jason Little 460743da16
Filter out auth chain queries that don't exist (#16552) 2023-11-22 10:59:16 +00:00
David Robertson 8d5c1fe921
Merge branch 'release-v1.97' into develop 2023-11-21 14:07:08 +00:00
David Robertson 536f9c96d9
fix changelog typo 2023-11-21 13:27:32 +00:00
David Robertson bb86eb9814
1.97.0rc1 2023-11-21 12:38:46 +00:00
dependabot[bot] 7611df705e
Bump sentry-sdk from 1.32.0 to 1.35.0 (#16666)
Bumps [sentry-sdk](https://github.com/getsentry/sentry-python) from 1.32.0 to 1.35.0.
- [Release notes](https://github.com/getsentry/sentry-python/releases)
- [Changelog](https://github.com/getsentry/sentry-python/blob/master/CHANGELOG.md)
- [Commits](https://github.com/getsentry/sentry-python/compare/1.32.0...1.35.0)

---
updated-dependencies:
- dependency-name: sentry-sdk
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-11-20 12:14:06 +00:00
dependabot[bot] e934e7f7e7
Bump pyopenssl from 23.2.0 to 23.3.0 (#16662)
Bumps [pyopenssl](https://github.com/pyca/pyopenssl) from 23.2.0 to 23.3.0.
- [Changelog](https://github.com/pyca/pyopenssl/blob/main/CHANGELOG.rst)
- [Commits](https://github.com/pyca/pyopenssl/compare/23.2.0...23.3.0)

---
updated-dependencies:
- dependency-name: pyopenssl
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-11-20 12:11:15 +00:00
dependabot[bot] d792e0f2d9
Bump types-pillow from 10.1.0.0 to 10.1.0.2 (#16664)
Bumps [types-pillow](https://github.com/python/typeshed) from 10.1.0.0 to 10.1.0.2.
- [Commits](https://github.com/python/typeshed/commits)

---
updated-dependencies:
- dependency-name: types-pillow
  dependency-type: direct:development
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-11-20 12:08:25 +00:00
dependabot[bot] 89d9ab0a0a
Bump types-psycopg2 from 2.9.21.15 to 2.9.21.16 (#16665)
Bumps [types-psycopg2](https://github.com/python/typeshed) from 2.9.21.15 to 2.9.21.16.
- [Commits](https://github.com/python/typeshed/commits)

---
updated-dependencies:
- dependency-name: types-psycopg2
  dependency-type: direct:development
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-11-20 12:07:34 +00:00
Erik Johnston 6088303efb
Speed up how quickly we launch new tasks (#16660)
Now that we're reducing concurrency (#16656), this is more important.
2023-11-17 16:36:02 +00:00
Patrick Cloke d9dcfe2a35
Bump requests-toolbelt from 0.10.1 to 1.0.0. (#16659) 2023-11-17 10:23:07 -05:00
Erik Johnston 9c02ef21e0
Speed up purge room by adding index (#16657)
What it says on the tin
2023-11-17 14:15:44 +00:00
Erik Johnston 6fec2d035f
Also discard 'caches' and 'backfill' stream POSITIONS (#16655)
Follow on from #16640
2023-11-17 14:14:29 +00:00
Patrick Cloke bdb0cbc5ca Merge branch 'master' into develop 2023-11-17 08:43:47 -05:00
Michael Weimann 518e4de758
Update admin user API return types in docs. (#16654) 2023-11-17 13:38:25 +00:00
Erik Johnston 700c8a0de5
Reduce task concurrency (#16656) 2023-11-17 13:14:26 +00:00
Patrick Cloke c4f5522189 Tweaks from review. 2023-11-17 08:01:13 -05:00
Patrick Cloke 6a1352e564 Move the forking note to 1.96.1. 2023-11-17 07:52:54 -05:00
Patrick Cloke 76f990c244 1.96.1 2023-11-17 07:51:59 -05:00
Patrick Cloke 47c682101f
Fix building wheels in CI. (#16653)
pip was using a vendored setuptools that was incompatible with
Python 3.12. Upgrading cibuildwheels to a version with a newer
version of pip (and thus a newer version of setuptools) fixes
the issue.
2023-11-17 07:42:49 -05:00
Patrick Cloke 2de2258bd2 Add blogpost link to changelog. 2023-11-16 13:01:32 -05:00
Patrick Cloke ff0148a165 1.96.0 2023-11-16 12:58:00 -05:00
Erik Johnston 4d6b800385
Revert "Fix test not detecting tables with missing primary keys and missing replica identities, then add more replica identities. (#16647)" (#16652)
This reverts commit 830988ae72.
2023-11-16 16:57:26 +00:00
Erik Johnston ef5329a9f9
Revert "Add a Postgres `REPLICA IDENTITY` to tables that do not have an implicit one. This should allow use of Postgres logical replication. (#16456)" (#16651)
This reverts commit 69afe3f7a0.
2023-11-16 16:48:48 +00:00
Erik Johnston 3e8531d3ba
Speed up deleting device messages (#16643)
Keeping track of a lower bound of stream ID where we've deleted everything below makes the queries much faster. Otherwise, every time we scan for rows to delete we'd re-scan across all the rows that have previously deleted (until the next table VACUUM).
2023-11-16 15:19:35 +00:00
Erik Johnston 1b238e8837
Speed up persisting large number of outliers (#16649)
Recalculating the roots tuple every iteration could be very expensive, so instead let's do a topological sort.
2023-11-16 14:25:35 +00:00
Erik Johnston fef08cbee8
Fix sending out of order `POSITION` over replication (#16639)
If a worker reconnects to Redis we send out the current positions of all our streams. However, if we're also trying to send out a backlog of RDATA at the same time then we can end up sending a `POSITION` with the current token *before* we've sent all the RDATA before the current token.

This doesn't cause actual bugs as the receiving servers see the POSITION, fetch the relevant rows from the DB, and then ignore the old RDATA as they come in. However, this is inefficient so it'd be better if we didn't  send out-of-order positions
2023-11-16 13:05:09 +00:00
Erik Johnston 898655fd12
More efficiently handle no-op POSITION (#16640)
We may receive `POSITION` commands where we already know that worker has
advanced past that position, so there is no point in handling it.
2023-11-16 12:32:17 +00:00
reivilibre 830988ae72
Fix test not detecting tables with missing primary keys and missing replica identities, then add more replica identities. (#16647)
* Fix the CI query that did not detect all cases of missing primary keys

* Add more missing REPLICA IDENTITY entries

* Newsfile

Signed-off-by: Olivier Wilkinson (reivilibre) <oliverw@matrix.org>

---------

Signed-off-by: Olivier Wilkinson (reivilibre) <oliverw@matrix.org>
2023-11-16 12:26:27 +00:00
David Robertson 43d1aa75e8
Add an Admin API to temporarily grant the ability to update an existing cross-signing key without UIA (#16634) 2023-11-15 17:28:10 +00:00
Sumner Evans 999bd77d3a
Asynchronous Uploads (#15503)
Support asynchronous uploads as defined in MSC2246.
2023-11-15 09:19:24 -05:00
Patrick Cloke 80922dc46e
Add links to pre-1.0 changelog issue/PR references. (#16638) 2023-11-15 13:31:24 +00:00
Patrick Cloke f2f2c7c1f0
Use full GitHub links instead of bare issue numbers. (#16637) 2023-11-15 08:02:11 -05:00
Will Hunt 4dd18bdc2e
Improve documentation for `/_synapse/admin/v1/rooms/<room_id>/timestamp_to_event` (#16631) 2023-11-14 11:43:44 -05:00
Nick Mills-Barrett 0e36a57b60
Remove whole table locks on push rule add/delete (#16051)
The statements are already executed within a transaction thus a table
level lock is unnecessary.
2023-11-13 16:57:44 +00:00
reivilibre 69afe3f7a0
Add a Postgres `REPLICA IDENTITY` to tables that do not have an implicit one. This should allow use of Postgres logical replication. (#16456)
* Add Postgres replica identities to tables that don't have an implicit one

Fixes #16224

* Newsfile

Signed-off-by: Olivier Wilkinson (reivilibre) <oliverw@matrix.org>

* Move the delta to version 83 as we missed the boat for 82

* Add a test that all tables have a REPLICA IDENTITY

* Extend the test to include when indices are deleted

* isort

* black

* Fully qualify `oid` as it is a 'hidden attribute' in Postgres 11

* Update tests/storage/test_database.py

Co-authored-by: Patrick Cloke <clokep@users.noreply.github.com>

* Add missed tables

---------

Signed-off-by: Olivier Wilkinson (reivilibre) <oliverw@matrix.org>
Co-authored-by: Patrick Cloke <clokep@users.noreply.github.com>
2023-11-13 16:03:22 +00:00
David Robertson fb2554b11f
Fix outbound_federation_restricted_to docs & note when added (#16628) 2023-11-13 14:26:49 +00:00
dependabot[bot] 7455b9e27d
Bump serde from 1.0.190 to 1.0.192 (#16627)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-11-13 11:10:28 +00:00
dependabot[bot] 35fac66d20
Bump prometheus-client from 0.17.1 to 0.18.0 (#16626)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-11-13 11:09:30 +00:00
dependabot[bot] 69d1ee3feb
Bump treq from 22.2.0 to 23.11.0 (#16623)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-11-13 11:08:31 +00:00
dependabot[bot] f92af19fa5
Bump types-pyopenssl from 23.2.0.2 to 23.3.0.0 (#16625)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-11-13 11:06:10 +00:00
dependabot[bot] 22a513014d
Bump types-bleach from 6.1.0.0 to 6.1.0.1 (#16624)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-11-13 11:05:37 +00:00
dependabot[bot] ca7421b5fd
Bump towncrier from 23.6.0 to 23.11.0 (#16622)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-11-13 11:05:02 +00:00
Patrick Cloke 2c6a7dfcbf
Use attempt_to_set_autocommit everywhere. (#16615)
To avoid asserting the type of the database connection.
2023-11-09 16:19:42 -05:00
reivilibre dc7f068d9c
Fix a long-standing bug where Synapse would not unbind third-party identifiers for Application Service users when deactivated and would not emit a compliant response. (#16617)
* Don't skip unbinding 3PIDs and returning success status when deactivating AS user

Fixes #16608

* Newsfile

Signed-off-by: Olivier Wilkinson (reivilibre) <oliverw@matrix.org>

---------

Signed-off-by: Olivier Wilkinson (reivilibre) <oliverw@matrix.org>
2023-11-09 20:18:25 +00:00
Patrick Cloke bc4372ad81
Use dbname instead of database for Postgres config. (#16618) 2023-11-09 14:40:45 -05:00
Patrick Cloke 9f514dd0fb
Use _invalidate_cache_and_stream_bulk in more places. (#16616)
This takes advantage of the new bulk method in more places to
invalidate caches for many keys at once (and then to stream that
over replication).
2023-11-09 14:40:30 -05:00
Patrick Cloke ab3f1b3b53
Convert simple_select_one_txn and simple_select_one to return tuples. (#16612) 2023-11-09 11:13:31 -05:00
Patrick Cloke ff716b483b
Return attrs for more media repo APIs. (#16611) 2023-11-09 11:00:30 -05:00
David Robertson 91587d4cf9
Bulk-invalidate e2e cached queries after claiming keys (#16613)
Co-authored-by: Patrick Cloke <patrickc@matrix.org>
2023-11-09 15:57:09 +00:00
dependabot[bot] f6aa047aa2
Bump pyicu from 2.11 to 2.12 (#16603)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-11-08 09:28:51 -05:00
dependabot[bot] 2a336cd2fc
Bump serde_json from 1.0.107 to 1.0.108 (#16604)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-11-08 09:27:51 -05:00
Patrick Cloke 455ef04187
Avoid updating the same rows multiple times with simple_update_many_txn. (#16609)
simple_update_many_txn had a bug in it which would cause each
update to be applied twice.
2023-11-07 14:02:09 -05:00
Patrick Cloke 9738b1c497
Avoid executing no-op queries. (#16583)
If simple_{insert,upsert,update}_many_txn is called without any data
to modify then return instead of executing the query.

This matches the behavior of simple_{select,delete}_many_txn.
2023-11-07 14:00:25 -05:00
Patrick Cloke ec9ff389f4
More tests for the simple_* methods. (#16596)
Expand tests for the simple_* database methods, additionally
test against both PostgreSQL and SQLite variants.
2023-11-07 09:34:23 -05:00
Patrick Cloke 7e5d3b06fa
Collect information for PushRuleEvaluator in parallel. (#16590)
Fetch information needed for push rule evaluation in parallel.
Ideally this would use query pipelining, but this is not
available in psycopg2.

Due to the database thread pool this may result in little
to no parallelization.
2023-11-06 15:41:57 -05:00
Patrick Cloke 1dd3074629
Bump setuptools_rust to match pinned version. (#16605) 2023-11-06 09:13:53 -05:00
Patrick Cloke cc4fe68adf
Support reactor timing metric on more reactors. (#16532)
Previously only Twisted's EPollReactor was compatible with the
reactor timing metric, notably not working when asyncio was used.

After this change, the following configurations support the reactor
timing metric:

* poll, epoll, or select reactors
* asyncio reactor with a poll, epoll, select, /dev/poll, or kqueue event loop.
2023-11-06 08:31:22 -05:00
dependabot[bot] 1a9b22a3d1
Bump setuptools-rust from 1.8.0 to 1.8.1 (#16601)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-11-06 08:12:18 -05:00
dependabot[bot] 5cf2988694
Bump types-pyyaml from 6.0.12.11 to 6.0.12.12 (#16602)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-11-06 08:12:01 -05:00
dependabot[bot] a28339b867
Bump types-jsonschema from 4.19.0.3 to 4.19.0.4 (#16599)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-11-06 08:11:31 -05:00
dependabot[bot] 2f689a6326
Bump ruff from 0.0.292 to 0.1.4 (#16600)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-11-06 08:11:15 -05:00
Patrick Cloke 92828a7f95
Simplify event persistence code (#16584)
The event persistence code used to handle multiple rooms
at a time, but was simplified to only ever be called with a
single room at a time (different rooms are now handled in
parallel). The code is still generic to multiple rooms causing
a lot of work that is unnecessary (e.g. unnecessary loops, and
partitioning data by room).

This strips out the ability to handle multiple rooms at once, greatly
simplifying the code.
2023-11-03 07:30:31 -04:00
Patrick Cloke bf69b57422
Fix "'int' object is not iterable" error in set_device_id_for_pushers background update (#16594)
A regression from removing the cursor_to_dict call, adds back
the wrapping into a tuple.
2023-11-02 14:00:18 +00:00
Patrick Cloke 0afbef30cf
Use simple_select_many_txn in event persistance code. (#16585)
Just to standardize on the normal helpers, it might also have
a slight perf improvement on PostgreSQL which will now use
`ANY (?)` instead of `IN (?, ?, ...)`.
2023-11-02 09:41:00 -04:00
dependabot[bot] c812f43bd7
Bump twisted from 23.8.0 to 23.10.0 (#16588) 2023-11-01 10:23:13 +00:00
Patrick Cloke ed1b879576
Do not call getfullargspec on every call. (#16589)
getfullargspec is relatively expensive and the results will
not change between calls, so precalculate it outside the
wrapper.
2023-10-31 20:16:17 +00:00
Patrick Cloke cfb6d38c47
Remove remaining usage of cursor_to_dict. (#16564) 2023-10-31 13:13:28 -04:00
Erik Johnston c0ba319b22 Merge branch 'release-v1.96' into develop 2023-10-31 16:30:16 +00:00
Patrick Cloke 70b503f144 Fix import ordering issue introduced in 7a3a55ac98. 2023-10-31 10:32:35 -04:00
Erik Johnston c5b543938b Update changelog 2023-10-31 14:17:27 +00:00
Erik Johnston 4bb2b4aa9a 1.96.0rc1 2023-10-31 14:09:13 +00:00
Erik Johnston 1f033ad5e4 Merge remote-tracking branch 'origin/develop' into release-v1.96 2023-10-31 14:08:51 +00:00
Erik Johnston 60c5e8d79b Revert "1.96.0rc1"
This reverts commit 4724a6ded1.
2023-10-31 14:08:45 +00:00
Erik Johnston 0cc6509cfe Merge branch 'release-v1.95' into develop 2023-10-31 14:05:52 +00:00
Erik Johnston a11511954a 1.95.1 2023-10-31 14:02:32 +00:00
Patrick Cloke daec55e1fe Merge pull request from GHSA-mp92-3jfm-3575 2023-10-31 13:59:09 +00:00
Patrick Cloke 7a3a55ac98
Merge pull request from GHSA-mp92-3jfm-3575 2023-10-31 13:58:30 +00:00
Erik Johnston 4724a6ded1 1.96.0rc1 2023-10-31 13:47:08 +00:00
David Robertson de981ae567
Claim local one-time-keys in bulk (#16565)
Co-authored-by: Patrick Cloke <clokep@users.noreply.github.com>
2023-10-30 21:25:21 +00:00
Niranjan Kurhade 91aa52c911
Clients link fixed in README (#16569) 2023-10-30 16:05:34 +00:00
Erik Johnston 408c13801a
Add fast path for replication events stream fetch (#16580)
We can bail early if the from token is greater than or equal to the
current token.
2023-10-30 14:47:57 +00:00
David Robertson fdce83ee60
Claim fallback keys in bulk (#16570) 2023-10-30 14:34:37 +00:00
dependabot[bot] a3f6200d65
Bump setuptools-rust from 1.7.0 to 1.8.0 (#16574)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: David Robertson <davidr@element.io>
2023-10-30 13:40:17 +00:00
David Robertson 4e1a19d375
Run actions/setup-go after checking out complement (#16567) 2023-10-30 13:07:08 +00:00
Erik Johnston 8c63e93286
Fix HTTP repl response to use minimum token (#16578) 2023-10-30 12:27:14 +00:00
dependabot[bot] ba55835000
Bump cryptography from 41.0.4 to 41.0.5 (#16572)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-10-30 11:44:36 +00:00
dependabot[bot] 8f7cd4cd03
Bump serde from 1.0.189 to 1.0.190 (#16577)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-10-30 11:32:36 +00:00
dependabot[bot] 425cb9c23c
Bump phonenumbers from 8.13.22 to 8.13.23 (#16576)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-10-30 11:30:26 +00:00
dependabot[bot] 13f6467785
Bump black from 23.10.0 to 23.10.1 (#16575)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-10-30 11:29:52 +00:00
dependabot[bot] bcaaeab410
Bump types-psycopg2 from 2.9.21.14 to 2.9.21.15 (#16573)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-10-30 11:28:50 +00:00
David Robertson 747416e94c
Portdb: don't copy a table that gets rebuilt (#16563) 2023-10-27 20:14:02 +01:00
kegsay 11a8ae0632
complement: enable dirty runs (#16520)
* complement: enable dirty runs

* Add changelog

* Set a low connpool limit when running in Complement

Dirty runs can cause many containers to be running concurrently,
which seems to easily exhaust resources on the host. The increased
speedup from dirty runs also seems to use more db connections on
workers, which are misconfigured currently to have
`SUM(workers * cp_max) > max_connections`, causing
```
FATAL:  sorry, too many clients already
```
which results in tests failing.

* Try p=2 concurrency to restrict slowness of servers which causes partial state join tests to flake

* Debug logging

* Only run flakey tests

* Only adjust connection pool limits in worker mode

* Move cp vars to somewhere where they get executed in CI

* Move cp values back to where they actually work

* Debug logging

* Try p=1 to see if this makes worker mode happier

* Remove debug logging
2023-10-27 18:29:20 +01:00
Patrick Cloke 2bf9341406
Ensure local invited & knocking users leave before purge. (#16559)
This is mostly useful for federated rooms where some users
would get stuck in the invite or knock state when the room
was purged from their homeserver.
2023-10-27 12:50:50 -04:00
Erik Johnston 5413cefe32
Reduce amount of caches POSITIONS we send (#16561)
Follow on from / actually correctly does #16557
2023-10-27 16:07:11 +01:00
Erik Johnston 89dbbd68e1
Reduce spurious replication catchup (#16555) 2023-10-27 13:27:20 +00:00
Erik Johnston 928e964857
Fix cross-worker ratelimiting (#16558)
c.f. #16481
2023-10-27 12:52:40 +01:00
Erik Johnston 0680d76659
Reduce replication traffic due to reflected cache stream POSITION (#16557) 2023-10-27 12:51:08 +01:00
Erik Johnston c02406ac71
Add new module API for adding custom fields to events `unsigned` section (#16549) 2023-10-27 09:04:08 +00:00
Patrick Cloke 679c691f6f
Remove more usages of cursor_to_dict. (#16551)
Mostly to improve type safety.
2023-10-26 15:12:28 -04:00
Patrick Cloke 85e5f2dc25
Add a new module API to update user presence state. (#16544)
This adds a module API which allows a module to update a user's
presence state/status message. This is useful for controlling presence
from an external system.

To fully control presence from the module the presence.enabled config
parameter gains a new state of "untracked" which disables internal tracking
of presence changes via user actions, etc. Only updates from the module will
be persisted and sent down sync properly).
2023-10-26 15:11:24 -04:00
Patrick Cloke 9407d5ba78
Convert simple_select_list and simple_select_list_txn to return lists of tuples (#16505)
This should use fewer allocations and improves type hints.
2023-10-26 13:01:36 -04:00
David Robertson c14a7de6af
Pin the recommended poetry version in contributors' guide (#16550) 2023-10-25 16:31:15 +01:00
Erik Johnston ba47fea528
Allow multiple workers to write to receipts stream. (#16432)
Fixes #16417
2023-10-25 16:16:19 +01:00
Patrick Cloke e182dbb5b9
Fix tests on Twisted trunk. (#16528)
Twisted trunk makes a change to the `TLSMemoryBIOFactory` where
the underlying protocol is changed from `TLSMemoryBIOProtocol` to
`BufferingTLSTransport` to improve performance of TLS code (see
https://github.com/twisted/twisted/issues/11989).

In order to properly hook this code up in tests we need to pass the test
reactor's clock into `TLSMemoryBIOFactory` to avoid the global (trial)
reactor being used by default.

Twisted does something similar internally for tests:
157cd8e659/src/twisted/web/test/test_agent.py (L871-L874)
2023-10-25 07:39:45 -04:00
Richard Brežák 95076f77c1
Fix http/s proxy authentication with long username/passwords (#16504) 2023-10-24 13:45:21 +00:00
David Robertson 2f1065f81b
Revert "Add test case to detect dodgy b64 encoding"
This reverts commit 5fe76b9434.

I think I had this accidentally commited on my local develop branch, and
so it accidentally got merged into upstream develop.

This should re-land with corrections in #16504.
2023-10-24 14:34:47 +01:00
David Robertson 2f35424812
Merge branch 'master' into develop 2023-10-24 14:23:20 +01:00
David Robertson c0d2f7649e
Merge branch 'develop' of github.com:matrix-org/synapse into develop 2023-10-24 14:23:19 +01:00
David Robertson 6ec98810e3
Rework alias and public room list rules docs (#16541) 2023-10-24 13:26:41 +01:00
Jason Little ffbe9b7666
Remove duplicate call to wake a remote destination when using federation sending worker (#16515) 2023-10-24 08:09:59 -04:00
David Robertson 79f48b2b4f
1.95.0 2023-10-24 13:01:02 +01:00
Michael Sasser 3df70aa800
Replace all Prometheus datasource UIDs of the Grafana Dashboard with the variable `${DS_PROMETHEUS}` and remove `__inputs` (#16471) 2023-10-23 19:50:50 +01:00
David Robertson 5fe76b9434
Add test case to detect dodgy b64 encoding 2023-10-23 19:29:22 +01:00
Patrick Cloke 3ab861ab9e
Fix type hint errors from Twisted trunk (#16526) 2023-10-23 14:28:05 -04:00
Erik Johnston 8f35f8148e
Fix bug where a new writer advances their token too quickly (#16473)
* Fix bug where a new writer advances their token too quickly

When starting a new writer (for e.g. persisting events), the
`MultiWriterIdGenerator` doesn't have a minimum token for it as there
are no rows matching that new writer in the DB.

This results in the the first stream ID it acquired being announced as
persisted *before* it actually finishes persisting, if another writer
gets and persists a subsequent stream ID. This is due to the logic of
setting the minimum persisted position to the minimum known position of
across all writers, and the new writer starts off not being considered.

* Fix sending out POSITIONs when our token advances without update

Broke in #14820

* For replication HTTP requests, only wait for minimal position
2023-10-23 16:57:30 +01:00
Erik Johnston 3bc23cc45c
Fix bug that could cause a `/sync` to tightloop with sqlite after restart (#16540)
This could happen if the last rows in the account data stream were inserted into `account_data`. After a restart the max account ID would be calculated without looking at the `account_data` table, and so have an old ID.
2023-10-23 13:39:25 +00:00
Marcel 3bcb6a059f
Mention how to redirect the Jaeger traces to a specific Jaeger instance (#16531) 2023-10-23 11:55:36 +00:00
Denis Kasak 3a0aa6fe76
Force TLS certificate verification in registration script. (#16530)
If using the script remotely, there's no particularly convincing reason
to disable certificate verification, as this makes the connection
interceptible.

If on the other hand, the script is used locally (the most common use
case), you can simply target the HTTP listener and avoid TLS altogether.
This is what the script already attempts to do if passed a homeserver
configuration YAML file.
2023-10-23 07:38:51 -04:00
Patrick Cloke 12ca87f5ea
Remove the last reference to event_txn_id. (#16521)
This table was no longer used, except for a background process
which purged old entries in it.
2023-10-23 07:37:45 -04:00
David Robertson 478a6c65eb
Bump matrix-synapse-ldap3 from 0.2.2 to 0.3.0 (#16539) 2023-10-23 12:28:29 +01:00
dependabot[bot] f835ab8de5
Bump black from 23.9.1 to 23.10.0 (#16538)
Bumps [black](https://github.com/psf/black) from 23.9.1 to 23.10.0.
- [Release notes](https://github.com/psf/black/releases)
- [Changelog](https://github.com/psf/black/blob/main/CHANGES.md)
- [Commits](https://github.com/psf/black/compare/23.9.1...23.10.0)

---
updated-dependencies:
- dependency-name: black
  dependency-type: direct:development
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-10-23 10:25:14 +01:00
dependabot[bot] 786b614fb2
Bump types-requests from 2.31.0.2 to 2.31.0.10 (#16537)
Bumps [types-requests](https://github.com/python/typeshed) from 2.31.0.2 to 2.31.0.10.
- [Commits](https://github.com/python/typeshed/commits)

---
updated-dependencies:
- dependency-name: types-requests
  dependency-type: direct:development
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-10-23 10:25:02 +01:00
dependabot[bot] a8026209d2
Bump gitpython from 3.1.37 to 3.1.40 (#16534)
Bumps [gitpython](https://github.com/gitpython-developers/GitPython) from 3.1.37 to 3.1.40.
- [Release notes](https://github.com/gitpython-developers/GitPython/releases)
- [Changelog](https://github.com/gitpython-developers/GitPython/blob/main/CHANGES)
- [Commits](https://github.com/gitpython-developers/GitPython/compare/3.1.37...3.1.40)

---
updated-dependencies:
- dependency-name: gitpython
  dependency-type: direct:development
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-10-23 10:24:46 +01:00
dependabot[bot] 2d12163cb4
Bump types-pillow from 10.0.0.3 to 10.1.0.0 (#16536)
Bumps [types-pillow](https://github.com/python/typeshed) from 10.0.0.3 to 10.1.0.0.
- [Commits](https://github.com/python/typeshed/commits)

---
updated-dependencies:
- dependency-name: types-pillow
  dependency-type: direct:development
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-10-23 09:46:55 +01:00
dependabot[bot] 9171bf3b35
Bump pygithub from 1.59.1 to 2.1.1 (#16535)
Bumps [pygithub](https://github.com/pygithub/pygithub) from 1.59.1 to 2.1.1.
- [Release notes](https://github.com/pygithub/pygithub/releases)
- [Changelog](https://github.com/PyGithub/PyGithub/blob/main/doc/changes.rst)
- [Commits](https://github.com/pygithub/pygithub/compare/v1.59.1...v2.1.1)

---
updated-dependencies:
- dependency-name: pygithub
  dependency-type: direct:development
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-10-23 09:45:12 +01:00
Patrick Cloke d2eab22de7
Clarify presence router docs. (#16529) 2023-10-20 11:40:26 -04:00
David Robertson 345c61f632
Build Debian packages for Ubuntu 23.10 Mantic Minotaur (#16524) 2023-10-19 21:53:05 +01:00
Erik Johnston e9069c9f91
Mark sync as limited if there is a gap in the timeline (#16485)
This splits thinsg into two queries, but most of the time we won't have
new event backwards extremities so this shouldn't actually add an extra
RTT for the majority of cases.

Note this removes the check for events with no prev events, but that was
part of MSC2716 work that has since been removed.
2023-10-19 15:04:18 +01:00
Patrick Cloke 49c9745b45
Avoid sending massive replication updates when purging a room. (#16510) 2023-10-18 12:26:01 -04:00
Mathieu Velten bcff01b406
Improve performance of delete device messages query (#16492) 2023-10-18 16:42:01 +01:00
Patrick Cloke 8841db4d27
Run trial/integration tests if .ci is modified. (#16512) 2023-10-18 07:19:53 -04:00
dependabot[bot] 19033313e6
Bump urllib3 from 1.26.17 to 1.26.18 (#16516)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-10-18 11:58:16 +01:00
Patrick Cloke 68d9559fef
Test against Python 3.12 release (#16511) 2023-10-17 14:41:10 -04:00
Patrick Cloke c81908bcd9 Update the changelog. 2023-10-17 13:07:12 -04:00
Patrick Cloke 6e6d611f55 1.95.0rc1 2023-10-17 11:54:45 -04:00
Patrick Cloke 6ad1f9eac2
Convert DeviceLastConnectionInfo to attrs. (#16507)
To improve type safety & memory usage.
2023-10-17 12:47:42 +00:00
Patrick Cloke 77dfc1f939
Fix a bug where servers could be marked as up when they were failing (#16506)
After this change a server will only be reported as back online
if they were previously having requests fail.
2023-10-17 07:32:40 -04:00
reivilibre 7291c68eea
Update the release script to remind releaser to check for special release notes. (#16461)
* Add reminder to check special release notes board in release script

* Newsfile

Signed-off-by: Olivier Wilkinson (reivilibre) <oliverw@matrix.org>

* Update release.py

* Bah, black

---------

Signed-off-by: Olivier Wilkinson (reivilibre) <oliverw@matrix.org>
2023-10-16 15:22:50 +01:00
Patrick Cloke e3e0ae4ab1
Convert state delta processing from a dict to attrs. (#16469)
For improved type checking & memory usage.
2023-10-16 07:35:22 -04:00
dependabot[bot] 4fe73f8f2f
Bump pillow from 10.0.1 to 10.1.0 (#16498)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-10-16 12:17:30 +01:00
dependabot[bot] 7154724671
Bump serde from 1.0.188 to 1.0.189 (#16494)
Bumps [serde](https://github.com/serde-rs/serde) from 1.0.188 to 1.0.189.
- [Release notes](https://github.com/serde-rs/serde/releases)
- [Commits](https://github.com/serde-rs/serde/compare/v1.0.188...v1.0.189)

---
updated-dependencies:
- dependency-name: serde
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-10-16 10:31:05 +00:00
dependabot[bot] 37d9edcef2
Bump sentry-sdk from 1.31.0 to 1.32.0 (#16496)
Bumps [sentry-sdk](https://github.com/getsentry/sentry-python) from 1.31.0 to 1.32.0.
- [Release notes](https://github.com/getsentry/sentry-python/releases)
- [Changelog](https://github.com/getsentry/sentry-python/blob/master/CHANGELOG.md)
- [Commits](https://github.com/getsentry/sentry-python/compare/1.31.0...1.32.0)

---
updated-dependencies:
- dependency-name: sentry-sdk
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-10-16 10:23:33 +00:00
Mathieu Velten eee6474bce
Remove useless async job to delete device messages on sync (#16491) 2023-10-16 11:06:27 +01:00
dependabot[bot] 9be4db29f2
Bump jsonschema from 4.19.0 to 4.19.1 (#16500)
Bumps [jsonschema](https://github.com/python-jsonschema/jsonschema) from 4.19.0 to 4.19.1.
- [Release notes](https://github.com/python-jsonschema/jsonschema/releases)
- [Changelog](https://github.com/python-jsonschema/jsonschema/blob/main/CHANGELOG.rst)
- [Commits](https://github.com/python-jsonschema/jsonschema/compare/v4.19.0...v4.19.1)

---
updated-dependencies:
- dependency-name: jsonschema
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-10-16 11:05:28 +01:00
dependabot[bot] aaca9773e3
Bump types-jsonschema from 4.17.0.10 to 4.19.0.3 (#16499)
Bumps [types-jsonschema](https://github.com/python/typeshed) from 4.17.0.10 to 4.19.0.3.
- [Commits](https://github.com/python/typeshed/commits)

---
updated-dependencies:
- dependency-name: types-jsonschema
  dependency-type: direct:development
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-10-16 11:05:20 +01:00
dependabot[bot] b220f8224e
Bump packaging from 23.1 to 23.2 (#16497)
Bumps [packaging](https://github.com/pypa/packaging) from 23.1 to 23.2.
- [Release notes](https://github.com/pypa/packaging/releases)
- [Changelog](https://github.com/pypa/packaging/blob/main/CHANGELOG.rst)
- [Commits](https://github.com/pypa/packaging/compare/23.1...23.2)

---
updated-dependencies:
- dependency-name: packaging
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-10-16 11:04:23 +01:00
dependabot[bot] a832212d4f
Bump pyo3-log from 0.8.3 to 0.8.4 (#16495)
Bumps [pyo3-log](https://github.com/vorner/pyo3-log) from 0.8.3 to 0.8.4.
- [Changelog](https://github.com/vorner/pyo3-log/blob/main/CHANGELOG.md)
- [Commits](https://github.com/vorner/pyo3-log/compare/v0.8.3...v0.8.4)

---
updated-dependencies:
- dependency-name: pyo3-log
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-10-16 11:03:40 +01:00
Richard van der Hoff 109882230c
Clean up logging on event persister endpoints (#16488) 2023-10-14 17:57:27 +01:00
Laurence Gill 166ffc0f23
Fix typo in useful_sql_for_admins.md (#16477) 2023-10-12 16:18:32 +01:00
Patrick Cloke 4cc729d480
Revert "Drop unused tables & unneeded access token ID for events. (#16268)" (#16465)
This reverts commit cabd577460.

There are additional usages of these tables
which need to be removed first.
2023-10-12 08:56:10 -04:00
kegsay f710d5480b
Update complement.sh to match new public API shape (#16466)
* Update complement.sh to match new public API shape

Sister PR to https://github.com/matrix-org/complement/pull/666

Context: https://github.com/matrix-org/complement/issues/654#issuecomment-1746613495

* Changelog

* Pedantry

* Run complement plz
2023-10-12 11:33:14 +01:00
Patrick Cloke cc865fffc0
Convert user_get_threepids response to attrs. (#16468)
This improves type annotations by not having a dictionary of Any values.
2023-10-11 20:08:11 -04:00
Patrick Cloke a4904dcb04
Convert simple_select_many_batch, simple_select_many_txn to tuples. (#16444) 2023-10-11 13:24:56 -04:00
Patrick Cloke d6b7d49a61
Handle content types with parameters. (#16440) 2023-10-11 07:50:34 -04:00
Patrick Cloke f1e43018b7
Inline simple_search_list/simple_search_list_txn. (#16434)
This only has a single use and is over abstracted. Inline it so that
we can improve type hints.
2023-10-10 12:16:36 -04:00
Patrick Cloke b6cb610d50 Merge remote-tracking branch 'origin/master' into develop 2023-10-10 11:40:30 -04:00
Patrick Cloke 5f12090fd7 Add CVE number for advisory GHSA-5chr-wjw5-3gq4. 2023-10-10 11:38:56 -04:00
Olivier Wilkinson (reivilibre) 336b0b9e89 Merge branch 'master' into develop 2023-10-10 13:21:23 +01:00
Olivier Wilkinson (reivilibre) 4f87edc6e8 Add security advisory note to the changelog 2023-10-10 13:20:59 +01:00
Olivier Wilkinson (reivilibre) b52ad6e4fa Merge branch 'master' into develop 2023-10-10 12:09:29 +01:00
Olivier Wilkinson (reivilibre) 25c412b3c5 1.94.0 2023-10-10 10:57:54 +01:00
David Robertson 28fd28e92e
Add DB indices to speed up purging rooms (#16457) 2023-10-10 10:33:39 +01:00
reivilibre 8902b3031d
Disable statement timeout whilst purging rooms (#16455)
* Disable statement timeout whilst purging rooms

* Newsfile

Signed-off-by: Olivier Wilkinson (reivilibre) <oliverw@matrix.org>

* Note the introduction version

---------

Signed-off-by: Olivier Wilkinson (reivilibre) <oliverw@matrix.org>
2023-10-09 15:41:17 +01:00
Erik Johnston a6abee36bc
Don't block CI on sign-off (#16454)
As this doesn't work with the private sign off flow.
2023-10-09 12:22:54 +01:00
dependabot[bot] 0a67743d9e
Bump ruff from 0.0.290 to 0.0.292 (#16449)
* Bump ruff from 0.0.290 to 0.0.292

Bumps [ruff](https://github.com/astral-sh/ruff) from 0.0.290 to 0.0.292.
- [Release notes](https://github.com/astral-sh/ruff/releases)
- [Changelog](https://github.com/astral-sh/ruff/blob/main/BREAKING_CHANGES.md)
- [Commits](https://github.com/astral-sh/ruff/compare/v0.0.290...v0.0.292)

---
updated-dependencies:
- dependency-name: ruff
  dependency-type: direct:development
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>

* Fix up lint

---------

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Erik Johnston <erik@matrix.org>
2023-10-09 10:46:17 +01:00
dependabot[bot] 3727b84a51
Bump netaddr from 0.8.0 to 0.9.0 (#16453)
Bumps [netaddr](https://github.com/drkjam/netaddr) from 0.8.0 to 0.9.0.
- [Release notes](https://github.com/drkjam/netaddr/releases)
- [Changelog](https://github.com/netaddr/netaddr/blob/master/CHANGELOG)
- [Commits](https://github.com/drkjam/netaddr/compare/0.8.0...0.9.0)

---
updated-dependencies:
- dependency-name: netaddr
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-10-09 10:40:55 +01:00
dependabot[bot] 6fb0c43106
Bump psycopg2 from 2.9.8 to 2.9.9 (#16452)
Bumps [psycopg2](https://github.com/psycopg/psycopg2) from 2.9.8 to 2.9.9.
- [Changelog](https://github.com/psycopg/psycopg2/blob/master/NEWS)
- [Commits](https://github.com/psycopg/psycopg2/compare/2.9.8...2.9.9)

---
updated-dependencies:
- dependency-name: psycopg2
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-10-09 10:40:31 +01:00
dependabot[bot] 45738e2739
Bump bleach from 6.0.0 to 6.1.0 (#16451)
Bumps [bleach](https://github.com/mozilla/bleach) from 6.0.0 to 6.1.0.
- [Changelog](https://github.com/mozilla/bleach/blob/main/CHANGES)
- [Commits](https://github.com/mozilla/bleach/compare/v6.0.0...v6.1.0)

---
updated-dependencies:
- dependency-name: bleach
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-10-09 10:40:09 +01:00
dependabot[bot] 3d2f5332c0
Bump types-bleach from 6.0.0.4 to 6.1.0.0 (#16450)
Bumps [types-bleach](https://github.com/python/typeshed) from 6.0.0.4 to 6.1.0.0.
- [Commits](https://github.com/python/typeshed/commits)

---
updated-dependencies:
- dependency-name: types-bleach
  dependency-type: direct:development
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-10-09 10:40:00 +01:00
Christoph 32fd9bc673
Fix possible AttributeError when account-api is called over unix socket (#16404)
Fixes #16396
2023-10-09 10:16:07 +01:00
David Robertson 1f10c20806
Apply join rate limiter outside the lineariser (#16441) 2023-10-06 17:31:52 +00:00
Patrick Cloke 06bbf1029c
Convert simple_select_list_paginate_txn to return tuples. (#16433) 2023-10-06 11:41:57 -04:00
Patrick Cloke 7615e2bf48
Return ThumbnailInfo in more places (#16438)
Improves type hints by using concrete types instead of
dictionaries.
2023-10-06 10:12:43 -04:00
Patrick Cloke cabd577460
Drop unused tables & unneeded access token ID for events. (#16268)
Drop the event_txn_id table and the tables related to MSC2716,
which is no longer supported in Synapse.
2023-10-06 08:29:33 -04:00
Patrick Cloke fc31b495b3
Stop sending incorrect knock_state_events. (#16403)
Synapse was incorrectly implemented with a knock_state_events
property on some APIs (instead of knock_room_state). This was
correct in Synapse 1.70.0, but *both* fields were sent to also be
compatible with Synapse versions expecting the wrong field.

Enough time has passed that only the correct field needs to be
included/handled.
2023-10-06 07:27:35 -04:00
Patrick Cloke ae5b997cfa
Fix comments related to replication. (#16428) 2023-10-06 07:25:44 -04:00
Patrick Cloke 694802eecd
Add documentation on background updates. (#16420) 2023-10-06 07:23:20 -04:00
Patrick Cloke 26b960b08b
Register media servlets via regex. (#16419)
This converts the media servlet URLs in the same way as
(most) of the rest of Synapse. This will give more flexibility
in the versions each endpoint exists under.
2023-10-06 07:22:55 -04:00
V02460 5946074d69
Bump pyo3 from 0.17.1 to 0.19.2 (#16162)
Signed-off-by: Kai A. Hiller <V02460@gmail.com>
2023-10-06 11:27:59 +01:00
Patrick Cloke 3555790b27
Remove unused method. (#16435) 2023-10-05 17:42:44 -04:00
Patrick Cloke fa907025f4
Remove manys calls to cursor_to_dict (#16431)
This avoids calling cursor_to_dict and then immediately
unpacking the values in the dict for other users. By not
creating the intermediate dictionary we can avoid allocating
the dictionary and strings for the keys, which should generally
be more performant.

Additionally this improves type hints by avoid Dict[str, Any]
dictionaries coming out of the database layer.
2023-10-05 11:07:38 -04:00
Patrick Cloke 4e302b30b6
Add __slots__ to replication commands. (#16429)
To slightly reduce the amount of memory each command takes.
2023-10-05 07:38:55 -04:00
Erik Johnston 009b47badf
Factor out `MultiWriter` token from `RoomStreamToken` (#16427) 2023-10-05 10:46:28 +01:00
Patrick Cloke ab9c1e8f39
Add type hints to synmark. (#16421) 2023-10-04 13:53:04 -04:00
Erik Johnston 80ec81dcc5
Some refactors around receipts stream (#16426) 2023-10-04 16:28:40 +01:00
David Robertson a01ee24734
Update changelog 2023-10-03 13:21:45 +01:00
David Robertson 8b50a9d01d
1.94.0rc1 2023-10-03 11:50:57 +01:00
dependabot[bot] cce9484452
Bump urllib3 from 1.26.15 to 1.26.17 (#16422)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-10-03 10:57:11 +01:00
Erik Johnston 1e67191a79
Add note to 'federation_domain_whitelist' option (#16416) 2023-10-03 09:55:29 +01:00
Patrick Cloke 127b940dc0
Clean-up old release notes (#16418)
Fixes some broken formatting from the reStructuedText to Markdown
conversion and fixes some typos.
2023-10-02 11:05:29 -04:00
David Robertson 1026776380
mypy plugin to check `@cached` return types (#14911)
Co-authored-by: David Robertson <davidr@element.io>
Co-authored-by: Patrick Cloke <patrickc@matrix.org>
Co-authored-by: Erik Johnston <erik@matrix.org>

Assert that the return type of callables wrapped in @cached
and @cachedList are cachable (aka immutable).
2023-10-02 14:22:36 +00:00
MomentQYC 5725712d47
Remove Python version from `/_synapse/admin/v1/server_version` (#16380)
There's no reason to expose the full Python version over what is
frequently a public API.
2023-10-02 09:07:53 -04:00
dependabot[bot] d40a939ff6
Bump phonenumbers from 8.13.19 to 8.13.22 (#16413) 2023-10-02 10:35:57 +01:00
dependabot[bot] 891f42f8c8
Bump msgpack from 1.0.6 to 1.0.7 (#16412) 2023-10-02 10:35:45 +01:00
dependabot[bot] 0a59372d9d
Bump types-netaddr from 0.8.0.9 to 0.9.0.1 (#16411) 2023-10-02 10:35:30 +01:00
dependabot[bot] 36c8b66403
Bump pydantic from 2.3.0 to 2.4.2 (#16410) 2023-10-02 10:35:11 +01:00
dependabot[bot] 18b453488f
Bump psycopg2 from 2.9.7 to 2.9.8 (#16409) 2023-10-02 10:34:22 +01:00
dependabot[bot] 451c08d868
Bump regex from 1.9.5 to 1.9.6 (#16408) 2023-10-02 10:33:59 +01:00
Erik Johnston 20fb08ec80
Downgrade repl stream time out error to warning (#16401)
This is because if a worker reaches ~100% CPU then everything starts
lagging and we hit the log line a lot. When at error we invoke sentry
and that has a lot of overhead, which then puts even more pressure on
the worker.
2023-09-29 11:52:48 +00:00
Will Hunt 79eb6c0cdc
Support rendering some media downloads as inline (#15988)
Use an `inline` Content-Disposition header when the media is
"safe" to display inline (some known text, image, video, audio
formats).
2023-09-29 07:19:38 -04:00
Patrick Cloke fb664cf159
Remove warnings from the docs about using message retention. (#16382)
There are no known bugs in the message retention code, but
it is possible that there still exists race conditions. Additional
fixes will be made as reported.
2023-09-28 07:02:31 -04:00
Patrick Cloke cdb89dcefe
Improve state types. (#16395) 2023-09-28 07:01:46 -04:00
Patrick Cloke c690fd16c4
Use modern config for maturin. (#16394)
This allows maturin >= 0.15 to build the properly named
shared library object.

For now the old configuration is also kept to allow for
older maturin installs to be used.
2023-09-28 07:01:16 -04:00
Erik Johnston 11c6cc1115 Merge branch 'master' into develop 2023-09-26 17:00:34 +01:00
Patrick Cloke f84da3c32e
Add a cache around server ACL checking (#16360)
* Pre-compiles the server ACLs onto an object per room and
  invalidates them when new events come in.
* Converts the server ACL checking into Rust.
2023-09-26 11:57:50 -04:00
Patrick Cloke 17800a0e97
Implement MSC4028: push all encrypted events. (#16361)
This unstable push rule is implemented behind an experimental
configuration flag.
2023-09-26 11:52:19 -04:00
Erik Johnston 88ba67eb91 1.93.0 2023-09-26 15:56:54 +01:00
David Robertson 06f650f5f4
Skip export-data on non-code (e.g. docs) PRs (#16387) 2023-09-26 15:21:07 +01:00
Patrick Cloke ec1c709440
Add documentation about the user directory search algorithm (#16320) 2023-09-26 13:44:38 +00:00
Erik Johnston 47ffc7e548
Reduce calls to `send_presence_to_destinations` (#16385) 2023-09-26 11:49:44 +01:00
Patrick Cloke 2763c49eca
Improve comments in StateGroupBackgroundUpdateStore. (#16383) 2023-09-25 18:50:47 +00:00
reivilibre 9fd18e9b06
Add developer documentation concerning gradual schema migrations with column alterations. (#15691)
Co-authored-by: Eric Eastwood <erice@element.io>
2023-09-25 17:43:09 +00:00
reivilibre 525d9d6e11
Avoid running CI steps when the files they check have not been changed. (#14745) 2023-09-25 16:59:40 +00:00
dependabot[bot] b225acf3e6
Bump types-psycopg2 from 2.9.21.11 to 2.9.21.14 (#16381)
* Bump types-psycopg2 from 2.9.21.11 to 2.9.21.14

Bumps [types-psycopg2](https://github.com/python/typeshed) from 2.9.21.11 to 2.9.21.14.
- [Commits](https://github.com/python/typeshed/commits)

---
updated-dependencies:
- dependency-name: types-psycopg2
  dependency-type: direct:development
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>

* Relax the annotation of Cursor.description

See
https://github.com/matrix-org/synapse/pull/16343#issuecomment-1726083384
for rationale.

* Changelog

* Changelog

---------

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: David Robertson <davidr@element.io>
2023-09-25 17:48:42 +01:00
dependabot[bot] 139a24de9e
Bump actions/checkout from 3 to 4 (#16250)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-09-25 11:39:54 -04:00
Maxwell G 12611bfcdd
Add support for pydantic v2 via pydantic.v1 compat module (#16332)
While maintaining support with pydantic v1.
2023-09-25 15:19:08 +00:00
dependabot[bot] 6d7095913a
Bump cryptography from 41.0.3 to 41.0.4 (#16362)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-09-25 11:14:05 -04:00
David Robertson fad4c63d25
Get CI to check PRs have been signed-off (#16348) 2023-09-25 14:45:17 +00:00
dependabot[bot] 9ee54c1370
Bump sentry-sdk from 1.30.0 to 1.31.0 (#16378)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-09-25 11:51:04 +01:00
dependabot[bot] d61d6a3e68
Bump msgpack from 1.0.5 to 1.0.6 (#16377)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-09-25 11:42:46 +01:00
dependabot[bot] 9479998c76
Bump gitpython from 3.1.35 to 3.1.37 (#16376)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-09-25 11:35:24 +01:00
dependabot[bot] a40f7724ff
Bump docker/setup-buildx-action from 2 to 3 (#16375)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-09-25 11:34:16 +01:00
dependabot[bot] 3d9661baaa
Bump dawidd6/action-download-artifact from 2.27.0 to 2.28.0 (#16374)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-09-25 11:33:42 +01:00
Patrick Cloke c9a0e1673a
Add missing license header. (#16359) 2023-09-21 09:47:29 -04:00
Patrick Cloke 7ec0a141b4
Convert more cached return values to immutable types (#16356) 2023-09-20 07:48:55 -04:00
Patrick Cloke d7c89c5908
Return immutable objects for cachedList decorators (#16350) 2023-09-19 15:26:44 -04:00
Patrick Cloke 5a66ff2f5c Merge branch 'release-v1.93' into develop 2023-09-19 10:33:22 -04:00
Patrick Cloke 4345ca066d Update changelog. 2023-09-19 08:35:44 -04:00
Patrick Cloke 9fabde6ead
Fix-up deactivated notes in docs. (#16355) 2023-09-19 08:05:33 -04:00
Patrick Cloke ac293357d0 Tweak changelog. 2023-09-19 08:03:38 -04:00
Patrick Cloke 9caeb9be10 1.93.0rc1 2023-09-19 07:56:49 -04:00
Jan Christian Grünhage 3d60b07cde
Use string for federation_client_minimum_tls_version documentation examples (#16353) 2023-09-19 07:24:47 -04:00
dependabot[bot] 1f477d65f5
Bump serde_json from 1.0.106 to 1.0.107 (#16345)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-09-18 13:26:35 -04:00
dependabot[bot] 83f73d5d67
Bump furo from 2023.8.19 to 2023.9.10 (#16340)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-09-18 13:26:19 -04:00
Patrick Cloke 118036eeab
Test against PostgreSQL 16. (#16351) 2023-09-18 13:21:00 -04:00
dependabot[bot] eee2b6642d
Bump ruff from 0.0.286 to 0.0.290 (#16342)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Patrick Cloke <patrickc@matrix.org>
2023-09-18 11:30:43 -04:00
Mathieu Velten 1f36041c80 Merge branch 'master' into develop 2023-09-18 16:55:05 +02:00
Patrick Cloke 47d4bb6057
Stop patching EventBase.__eq__ in tests. (#16349)
It is clearer to directly test equality instead of doing indirect
assertions via patching __eq__.
2023-09-18 14:48:02 +00:00
Hanadi eef2b9e344
Filter locked users in the admin API (#16328)
Co-authored-by: Hanadi Tamimi <hanadi.tamimi@sdui.de>
2023-09-18 15:37:51 +01:00
Patrick Cloke c1e244c8f7
Make cached account data/tags/admin types immutable (#16325) 2023-09-18 09:55:04 -04:00
Mathieu Velten e36990c00e Update changelog 2023-09-18 15:35:57 +02:00
Mathieu Velten d8aed6fba7 1.92.3 2023-09-18 15:29:46 +02:00
Patrick Cloke 85bfd4735e
Return an immutable value from get_latest_event_ids_in_room. (#16326) 2023-09-18 09:29:05 -04:00
Mathieu Velten 053155a2af Mandate Pillow>=10.0.1 because of libwebp CVE (#16347) 2023-09-18 15:02:16 +02:00
Patrick Cloke 63d28a88c1
Additional validation of receipts (#16327)
Reject invalid receipts with a reasonable error message &
expands tests for receipts.
2023-09-18 13:02:12 +00:00
Mathieu Velten 4663d55502
Mandate Pillow>=10.0.1 because of libwebp CVE (#16347) 2023-09-18 15:01:23 +02:00
dependabot[bot] 53b7d9ccf2 Bump pillow from 10.0.0 to 10.0.1 (#16344)
Bumps [pillow](https://github.com/python-pillow/Pillow) from 10.0.0 to 10.0.1.
- [Release notes](https://github.com/python-pillow/Pillow/releases)
- [Changelog](https://github.com/python-pillow/Pillow/blob/main/CHANGES.rst)
- [Commits](https://github.com/python-pillow/Pillow/compare/10.0.0...10.0.1)

---
updated-dependencies:
- dependency-name: pillow
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-09-18 13:48:00 +02:00
dependabot[bot] 5ad1714d42
Bump pillow from 10.0.0 to 10.0.1 (#16344)
Bumps [pillow](https://github.com/python-pillow/Pillow) from 10.0.0 to 10.0.1.
- [Release notes](https://github.com/python-pillow/Pillow/releases)
- [Changelog](https://github.com/python-pillow/Pillow/blob/main/CHANGES.rst)
- [Commits](https://github.com/python-pillow/Pillow/compare/10.0.0...10.0.1)

---
updated-dependencies:
- dependency-name: pillow
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-09-18 13:46:02 +02:00
José Joaquín Atria 6946209e67
Set email charset as utf-8 rather than utf8 (#16329) 2023-09-18 07:32:01 -04:00
dependabot[bot] fedaebc440
Bump typing-extensions from 4.7.1 to 4.8.0 (#16341)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-09-18 10:21:32 +01:00
dependabot[bot] ab5f4f906d
Bump docker/login-action from 2 to 3 (#16339)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-09-18 10:21:14 +01:00
dependabot[bot] 4cdc2ed7dc
Bump docker/metadata-action from 4 to 5 (#16337)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-09-18 10:20:50 +01:00
dependabot[bot] 47f9837008
Bump docker/setup-qemu-action from 2 to 3 (#16338)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-09-18 10:19:05 +01:00
dependabot[bot] d2eacfe051
Bump docker/build-push-action from 4 to 5 (#16336)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-09-18 10:17:48 +01:00
Erik Johnston 98d39e590a Merge branch 'master' into develop 2023-09-15 14:39:20 +01:00
Mathieu Velten dd44ee00b6
Add automatic purge after all users forget a room (#15488)
Also add restore of purge/shutdown rooms after a synapse restart.

Co-authored-by:  Eric Eastwood <erice@matrix.org>
Co-authored-by: Erik Johnston <erikj@matrix.org>
2023-09-15 15:37:44 +02:00
Erik Johnston e7274f764b Fix Changelog 2023-09-15 13:34:44 +01:00
Erik Johnston e6be9a3ca4 1.92.2 2023-09-15 13:30:16 +01:00
Erik Johnston 3cf1a3aa17
Use bookwork as docker base image (#16324) 2023-09-15 13:14:10 +01:00
Jason Little 2a0f86f88f
Convert `_insert_graph_receipts_txn` to `simple_upsert` (#16299) 2023-09-15 09:16:45 +01:00
Erik Johnston edec0b93ca
Only use literal strings for process names (#16315) 2023-09-15 09:10:24 +01:00
Erik Johnston 329597022e
Some minor performance fixes for task schedular (#16313) 2023-09-14 16:20:47 +01:00
Erik Johnston e9e2904eb2
Speed up deleting to-device messages task (#16318) 2023-09-14 14:56:07 +01:00
6543 39dc5de399
docs: Link to the Alpine Linux community package for Synapse (#16304) 2023-09-14 14:13:48 +01:00
Erik Johnston 954921736b
Refactor `get_user_by_id` (#16316) 2023-09-14 12:46:30 +01:00
Erik Johnston 032cf84f52
Remove a reference cycle in background process (#16314) 2023-09-13 16:17:06 +01:00
Hanadi 7afb5e0410
Fix using dehydrated devices (MSC2697) & refresh tokens (#16288)
Refresh tokens were not correctly moved to the rehydrated
device (similar to how the access token is currently handled).
This resulted in invalid refresh tokens after rehydration.
2023-09-13 08:33:39 -04:00
Patrick Cloke d38d0dffc9
Use StrCollection in additional places. (#16301) 2023-09-13 07:57:19 -04:00
Erik Johnston e9addf6a01
Don't schedule an async task on every sync (#16312) 2023-09-13 11:59:44 +01:00
Erik Johnston be3c7b08a3
Fix deleting device inbox when using background worker (#16311)
Introduced in #16240

The action for the task was only defined on the "master" handler, rather than the base worker one.
2023-09-13 11:54:16 +01:00
Erik Johnston ab13fb08bf
Improve logging of replication (#16309) 2023-09-13 09:51:50 +00:00
Mathieu Velten 48387c56f1 Update changelog 2023-09-12 15:34:10 +02:00
Mathieu Velten 12bee2dcc2 Merge branch 'master' into develop 2023-09-12 15:22:26 +02:00
Mathieu Velten 1296e471c3 1.92.1 2023-09-12 13:20:48 +02:00
Patrick Cloke ba48c563c9
Bump mypy from 1.4.1 to 1.5.1. (#16300) 2023-09-12 07:16:09 -04:00
Patrick Cloke 16ef6f1e3c
Stop purging tables which are slated for removal. (#16273) 2023-09-12 07:12:31 -04:00
Mathieu Velten 622463636c Remove kinetic deb build, it's EOL 2023-09-12 12:16:44 +02:00
Erik Johnston 2b35626b6b
Refactor storing of server keys (#16261) 2023-09-12 11:08:04 +01:00
Mathieu Velten 4a5bf74372 Inverse changelog order 2023-09-12 12:06:47 +02:00
Mathieu Velten efe778a0b8 1.92.0 2023-09-12 11:59:35 +02:00
Patrick Cloke 9400dc0535
Add the List-Unsubscribe header for notification emails. (#16274)
Adds both the List-Unsubscribe (RFC2369) and List-Unsubscribe-Post (RFC8058)
headers to push notification emails, which together should:

* Show an "Unsubscribe" link in the MUA UI when viewing Synapse notification emails.
* Enable "one-click" unsubscribe (the user never leaves their MUA, which automatically
  makes a POST request to the specified endpoint).
2023-09-11 09:49:48 -04:00
Erik Johnston 151e4bbc45
Filter out down hosts when retrying fetching device lists (#16298) 2023-09-11 13:11:02 +01:00
dependabot[bot] b0e93b63d4
Bump types-pillow from 10.0.0.2 to 10.0.0.3 (#16293) 2023-09-11 09:54:37 +01:00
dependabot[bot] f93cd6abbb
Bump types-setuptools from 68.0.0.3 to 68.2.0.0 (#16292) 2023-09-11 09:54:28 +01:00
dependabot[bot] ee65d8f750
Bump mypy-zope from 1.0.0 to 1.0.1 (#16291) 2023-09-11 09:53:56 +01:00
dependabot[bot] aafcaf277e
Bump black from 23.7.0 to 23.9.1 (#16295) 2023-09-11 09:53:46 +01:00
dependabot[bot] e8ebc730ca
Bump serde_json from 1.0.105 to 1.0.106 (#16296) 2023-09-11 09:53:24 +01:00
David Robertson edd83f23b7
Improve type hints for attrs classes (#16276) 2023-09-08 19:29:38 +01:00
V02460 a0ed55ef12
Upgrade CI run of Python 3.12 from rc1 to rc2 (#16280) 2023-09-08 12:55:43 -04:00
Patrick Cloke aa483cb4c9
Update ruff config (#16283)
Enable additional checks & clean-up unneeded configuration.
2023-09-08 11:24:36 -04:00
David Robertson c1c6c95d72
Log values at DEBUG level with execute_values (#16281) 2023-09-08 14:50:13 +01:00
Patrick Cloke 5c8870cb28
Fix-up incorrect spellings in docs. (#16282) 2023-09-08 09:47:36 -04:00
Erik Johnston f43d994624
Fix bug with new task scheduler using lots of CPU. (#16278)
Using the new `TaskScheduler` meant that we'ed create lots of new
metrics (due to adding task ID to the desc of background process),
resulting in requests for metrics taking an increasing amount of CPU.
2023-09-08 14:43:01 +01:00
dependabot[bot] 9084429a6c
Bump gitpython from 3.1.34 to 3.1.35 (#16279)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-09-08 08:59:23 -04:00
Patrick Cloke 69b74d9330
Avoid temporary storage of sensitive information. (#16272)
During the UI auth process, avoid storing sensitive information
into the database.
2023-09-08 08:57:56 -04:00
V02460 583d5963e6
Raise setuptools_rust version cap to 1.7.0 (#16277) 2023-09-08 08:10:26 -04:00
Erik Johnston d23c394669
Reduce CPU overhead of change password endpoint (#16264) 2023-09-08 13:06:00 +01:00
Erik Johnston 1cd410a783
Recheck if remote device is cached before requesting it (#16252)
This fixes a bug where we could get stuck re-requesting the device over
replication again and again.
2023-09-07 12:45:43 +00:00
Patrick Cloke 7e98d382f9
Support releasing on macOS. (#16266) 2023-09-07 07:00:41 -04:00
Erik Johnston 8940d1b28e
Add `/notifications` endpoint to workers (#16265) 2023-09-07 09:26:07 +00:00
dependabot[bot] a83f75a37d
Bump gitpython from 3.1.32 to 3.1.34 (#16267)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-09-06 15:19:56 -04:00
Marcel 13e9cad537
Send the opentracing span information to appservices (#16227) 2023-09-06 15:19:17 -04:00
Aurélien Grimpard fe69e7f617
Handle "registration_enabled" parameter for CAS (#16262)
Similar to OIDC, CAS providers can now disable registration such
that only existing users are able to login via SSO.
2023-09-06 14:32:24 -04:00
Patrick Cloke 32fb264120 Merge remote-tracking branch 'origin/release-v1.92' into develop 2023-09-06 13:08:22 -04:00
Patrick Cloke fd50a9b47c Add back newsfile from #16258. 2023-09-06 13:06:33 -04:00
Patrick Cloke 55c20da4a3 Merge remote-tracking branch 'origin/release-v1.91' into release-v1.92 2023-09-06 11:25:28 -04:00
reivilibre 51303035f2
Apply missed suggestions from the review of #16090. (#16263)
* Suggestions from PR

* Newsfile

Signed-off-by: Olivier Wilkinson (reivilibre) <oliverw@matrix.org>

---------

Signed-off-by: Olivier Wilkinson (reivilibre) <oliverw@matrix.org>
2023-09-06 16:15:56 +01:00
Patrick Cloke 9de615b3aa Link to MSC in changelog. 2023-09-06 11:10:57 -04:00
Patrick Cloke c9282baf03 1.91.2 2023-09-06 11:01:56 -04:00
Quentin Gliech 1940d990a3
Revert MSC3861 introspection cache, admin impersonation and account lock (#16258) 2023-09-06 15:19:51 +01:00
reivilibre 35934b02a9
Add GCC and GNU Make to the Nix flake development environment so that `ruff` can be compiled. (#16090)
* Add gcc and GNU make to the Nix flake

* Newsfile

Signed-off-by: Olivier Wilkinson (reivilibre) <oliverw@matrix.org>

* unset LD_LIBRARY_PATH

---------

Signed-off-by: Olivier Wilkinson (reivilibre) <oliverw@matrix.org>
2023-09-06 14:35:02 +01:00
Andrew Morgan ffe4ea1302
Update rust in flake.nix: 1.70.0 -> 1.71.1 to address CVE-2023-38497 (#16260) 2023-09-06 14:34:01 +01:00
reivilibre e937e2111a
Add the ability to use `G` (GiB) and `T` (TiB) suffixes in configuration options that refer to numbers of bytes. (#16219)
* Add more suffixes to `parse_size`

* Newsfile

Signed-off-by: Olivier Wilkinson (reivilibre) <oliverw@matrix.org>

---------

Signed-off-by: Olivier Wilkinson (reivilibre) <oliverw@matrix.org>
2023-09-06 14:01:10 +01:00
reivilibre 698f6fa250
Allow modules to delete rooms. (#15997)
* Allow user_id to be optional for room deletion

* Add module API method to delete a room

* Newsfile

Signed-off-by: Olivier Wilkinson (reivilibre) <oliverw@matrix.org>

* Don't worry about the case block=True && requester_user_id is None

---------

Signed-off-by: Olivier Wilkinson (reivilibre) <oliverw@matrix.org>
2023-09-06 11:50:07 +01:00
Mathieu Velten 4f1840a88a
Delete device messages asynchronously and in staged batches (#16240) 2023-09-06 09:30:53 +02:00
Will Hunt 1e571cd664
Fix appservices being unable to handle to_device messages for multiple users (#16251) 2023-09-05 15:46:57 -04:00
Travis Ralston b1d71c687a
Add MSC4040 `matrix-fed` service lookups (#16137) 2023-09-05 15:45:39 -04:00
Erik Johnston c9cec2daed
Fix bug where we kept re-requesting a remote server's key repeatedly. (#16257)
* Correctly handle multiple rows per server/key

* Newsfile
2023-09-05 20:27:41 +01:00
Patrick Cloke a2b8814d64
Fix incorrect docstring for Ratelimiter. (#16255) 2023-09-05 12:11:05 -04:00
David Robertson 02bc5906ec - Add configuration setting for CAS protocol version. Contributed by Aurélien Grimpard. ([\#15816](https://github.com/matrix-org/synapse/issues/15816))
- Suppress notifications from message edits per [MSC3958](https://github.com/matrix-org/matrix-spec-proposals/pull/3958). ([\#16113](https://github.com/matrix-org/synapse/issues/16113))
 - Return a `Retry-After` with `M_LIMIT_EXCEEDED` error responses. ([\#16136](https://github.com/matrix-org/synapse/issues/16136))
 - Add `last_seen_ts` to the [admin users API](https://matrix-org.github.io/synapse/latest/admin_api/user_admin_api.html). ([\#16218](https://github.com/matrix-org/synapse/issues/16218))
 - Improve resource usage when sending data to a large number of remote hosts that are marked as "down". ([\#16223](https://github.com/matrix-org/synapse/issues/16223))
 
 - Fix IPv6-related bugs on SMTP settings, adding groundwork to fix similar issues. Contributed by @evilham and @telmich (ungleich.ch). ([\#16155](https://github.com/matrix-org/synapse/issues/16155))
 - Fix a spec compliance issue where requests to the `/publicRooms` federation API would specify `include_all_networks` as a string. ([\#16185](https://github.com/matrix-org/synapse/issues/16185))
 - Fix inaccurate error message while attempting to ban or unban a user with the same or higher PL by spliting the conditional statements. Contributed by @leviosacz. ([\#16205](https://github.com/matrix-org/synapse/issues/16205))
 - Fix a rare bug that broke looping calls, which could lead to e.g. linearly increasing memory usage. Introduced in v1.90.0. ([\#16210](https://github.com/matrix-org/synapse/issues/16210))
 - Fix a long-standing bug where uploading images would fail if we could not generate thumbnails for them. ([\#16211](https://github.com/matrix-org/synapse/issues/16211))
 - Fix a long-standing bug where we did not correctly back off from servers that had "gone" if they returned 4xx series error codes. ([\#16221](https://github.com/matrix-org/synapse/issues/16221))
 
 - Update links to the [matrix.org blog](https://matrix.org/blog/). ([\#16008](https://github.com/matrix-org/synapse/issues/16008))
 - Document which [admin APIs](https://matrix-org.github.io/synapse/latest/usage/administration/admin_api/index.html) are disabled when experimental [MSC3861](https://github.com/matrix-org/matrix-spec-proposals/pull/3861) support is enabled. ([\#16168](https://github.com/matrix-org/synapse/issues/16168))
 - Document [`exclude_rooms_from_sync`](https://matrix-org.github.io/synapse/v1.92/usage/configuration/config_documentation.html#exclude_rooms_from_sync) configuration option. ([\#16178](https://github.com/matrix-org/synapse/issues/16178))
 
 - Prepare unit tests for Python 3.12. ([\#16099](https://github.com/matrix-org/synapse/issues/16099))
 - Fix nightly CI jobs. ([\#16121](https://github.com/matrix-org/synapse/issues/16121), [\#16213](https://github.com/matrix-org/synapse/issues/16213))
 - Describe which rate limiter was hit in logs. ([\#16135](https://github.com/matrix-org/synapse/issues/16135))
 - Simplify presence code when using workers. ([\#16170](https://github.com/matrix-org/synapse/issues/16170))
 - Track per-device information in the presence code. ([\#16171](https://github.com/matrix-org/synapse/issues/16171), [\#16172](https://github.com/matrix-org/synapse/issues/16172))
 - Stop using the `event_txn_id` table. ([\#16175](https://github.com/matrix-org/synapse/issues/16175))
 - Use `AsyncMock` instead of custom code. ([\#16179](https://github.com/matrix-org/synapse/issues/16179), [\#16180](https://github.com/matrix-org/synapse/issues/16180))
 - Improve error reporting of invalid data passed to `/_matrix/key/v2/query`. ([\#16183](https://github.com/matrix-org/synapse/issues/16183))
 - Task scheduler: add replication notify for new task to launch ASAP. ([\#16184](https://github.com/matrix-org/synapse/issues/16184))
 - Improve type hints. ([\#16186](https://github.com/matrix-org/synapse/issues/16186), [\#16188](https://github.com/matrix-org/synapse/issues/16188), [\#16201](https://github.com/matrix-org/synapse/issues/16201))
 - Bump black version to 23.7.0. ([\#16187](https://github.com/matrix-org/synapse/issues/16187))
 - Log the details of background update failures. ([\#16212](https://github.com/matrix-org/synapse/issues/16212))
 - Cache device resync requests over replication. ([\#16241](https://github.com/matrix-org/synapse/issues/16241))
 
 * Bump anyhow from 1.0.72 to 1.0.75. ([\#16141](https://github.com/matrix-org/synapse/issues/16141))
 * Bump furo from 2023.7.26 to 2023.8.19. ([\#16238](https://github.com/matrix-org/synapse/issues/16238))
 * Bump phonenumbers from 8.13.18 to 8.13.19. ([\#16237](https://github.com/matrix-org/synapse/issues/16237))
 * Bump psycopg2 from 2.9.6 to 2.9.7. ([\#16196](https://github.com/matrix-org/synapse/issues/16196))
 * Bump regex from 1.9.3 to 1.9.4. ([\#16195](https://github.com/matrix-org/synapse/issues/16195))
 * Bump ruff from 0.0.277 to 0.0.286. ([\#16198](https://github.com/matrix-org/synapse/issues/16198))
 * Bump sentry-sdk from 1.29.2 to 1.30.0. ([\#16236](https://github.com/matrix-org/synapse/issues/16236))
 * Bump serde from 1.0.184 to 1.0.188. ([\#16194](https://github.com/matrix-org/synapse/issues/16194))
 * Bump serde_json from 1.0.104 to 1.0.105. ([\#16140](https://github.com/matrix-org/synapse/issues/16140))
 * Bump types-psycopg2 from 2.9.21.10 to 2.9.21.11. ([\#16200](https://github.com/matrix-org/synapse/issues/16200))
 * Bump types-pyyaml from 6.0.12.10 to 6.0.12.11. ([\#16199](https://github.com/matrix-org/synapse/issues/16199))
 -----BEGIN PGP SIGNATURE-----
 
 iQIzBAABCAAdFiEE1508oLYUKainYFJakD7OEIo53t0FAmT3RFIACgkQkD7OEIo5
 3t1gGA/9HQwV3vNgZkQj9Y0TVEj/ECoASSOvaTWbsXmoyctEqUOF7yoin+e6rZdB
 b5hrCMN8dGGny/Nqb7hF2/Zcz+zHOhNOshEE7VDnt2dLZY3MaaF7SOFVnfM6KDz5
 UyKkSbvhO2wgZIu8YebxW9C7a3EUnI8ocnb2PsyNNCPYtOVdZ7h+JeV9T2OS78rG
 dw/hFNBG8jQYwbbpaFtdCjDCaz67/GB9LnWcXwbE6kJr5lJ903uH0NPLdrdITsrH
 wvvw1vQHFZtzuAjfnKA/dz42H1h/s3ogxBMQxS0rVdxvFDIJiLUpWyDdPxjixzRS
 w96Pw4+hTlposAZT1qWRf0n263M5BZpoEQuZ/C4q088FrLubuaummW2IfMWx8ono
 y556CLRg+yjv2MmGeYYabcSxj1OPc/FV3w73tbxrVjKP7x2fnfGqTJeDhIqXh2wj
 DphMFBVHmvUEJv6TRfPV/V/ocoN3Tg1vXCPiY3JGcU9ihtHAjL0F9Af03nMKg594
 ZQc/xbroEFln11Ip+q9LYxP8q0ptKZph/Fo10WaeYSLII2WO0dC+mP+QvDfQle3N
 ftXJm+Utk/cg73gSYNqex4EmHkGAB0np8YjpA41mdVNcJlXGQ3WQn0FgqX7JZ1jQ
 lc0YtGm8jqp045tU2WAcacOg8/udTL+UkbDSdkhzq3KRXbB5bE8=
 =K4NS
 -----END PGP SIGNATURE-----
gpgsig -----BEGIN PGP SIGNATURE-----
 
 iQIzBAABCAAdFiEE1508oLYUKainYFJakD7OEIo53t0FAmT3T8IACgkQkD7OEIo5
 3t3hyhAAoTswFfnZxfee5fk458OIe05KdpQ/oYSv+5iKeqe3SGrPPgPD58gGi6+c
 vUccRAB7frZIdzjGnTuN98WMn2ll7LPvn1tHzRIQM78qoIwAyJrpe40BL5HQR6Iy
 3weNug9ae2NGOqNaO816aDFOHvkE0lNDpU9fE4iZcyxs5b7kyD8CTmNv1DrRRtPV
 DOcCqtRgSMZXC7aVI4c8f7q4WdzfjWSLrieeDaHwVBFPAm1jcg3WiWauFpCYScHg
 sPq0D0zeRdkaNNw8+B4B6WOePL1lm31RRsjmxqf3+bgfyzEB8DzJw8JEv02BnkTi
 yNzJKAouBahjgXEcGXmfm5APBbu1eW4hkHeZPWkM1aHHNnQ3MdRPGHt0Nb+cmsxO
 AEu6GiFY2w14y6DwATXGpOpzlKHmYfdsncjpC17f35aqt94LnSXH39tt1idYABbI
 uC4sfStXFN+bURYUbsIDTGXbe1/yjaTNGR0Cq8GvqWDWQGN2fy2z9dhgZzvr7SEL
 3TRHtUOcuCHVBv/FxCl3jjWNFgDveemSVDqvBmtbvNjGaIP2jSpJD9tJTRWk+m3n
 7c0T8UiryfAsgkme3cwjfnvIHaPRevLNqImNlQnJS42xjQaplx67aP4HZ1P3JiBU
 lSC+JhKQ7TOxajFlmrB3LUkFaRpYt7i3ZCFvRZACWkDPxh285r8=
 =WqeW
 -----END PGP SIGNATURE-----

Merge tag 'v1.92.0rc1' into develop

- Add configuration setting for CAS protocol version. Contributed by Aurélien Grimpard. ([\#15816](https://github.com/matrix-org/synapse/issues/15816))
- Suppress notifications from message edits per [MSC3958](https://github.com/matrix-org/matrix-spec-proposals/pull/3958). ([\#16113](https://github.com/matrix-org/synapse/issues/16113))
- Return a `Retry-After` with `M_LIMIT_EXCEEDED` error responses. ([\#16136](https://github.com/matrix-org/synapse/issues/16136))
- Add `last_seen_ts` to the [admin users API](https://matrix-org.github.io/synapse/latest/admin_api/user_admin_api.html). ([\#16218](https://github.com/matrix-org/synapse/issues/16218))
- Improve resource usage when sending data to a large number of remote hosts that are marked as "down". ([\#16223](https://github.com/matrix-org/synapse/issues/16223))

- Fix IPv6-related bugs on SMTP settings, adding groundwork to fix similar issues. Contributed by @evilham and @telmich (ungleich.ch). ([\#16155](https://github.com/matrix-org/synapse/issues/16155))
- Fix a spec compliance issue where requests to the `/publicRooms` federation API would specify `include_all_networks` as a string. ([\#16185](https://github.com/matrix-org/synapse/issues/16185))
- Fix inaccurate error message while attempting to ban or unban a user with the same or higher PL by spliting the conditional statements. Contributed by @leviosacz. ([\#16205](https://github.com/matrix-org/synapse/issues/16205))
- Fix a rare bug that broke looping calls, which could lead to e.g. linearly increasing memory usage. Introduced in v1.90.0. ([\#16210](https://github.com/matrix-org/synapse/issues/16210))
- Fix a long-standing bug where uploading images would fail if we could not generate thumbnails for them. ([\#16211](https://github.com/matrix-org/synapse/issues/16211))
- Fix a long-standing bug where we did not correctly back off from servers that had "gone" if they returned 4xx series error codes. ([\#16221](https://github.com/matrix-org/synapse/issues/16221))

- Update links to the [matrix.org blog](https://matrix.org/blog/). ([\#16008](https://github.com/matrix-org/synapse/issues/16008))
- Document which [admin APIs](https://matrix-org.github.io/synapse/latest/usage/administration/admin_api/index.html) are disabled when experimental [MSC3861](https://github.com/matrix-org/matrix-spec-proposals/pull/3861) support is enabled. ([\#16168](https://github.com/matrix-org/synapse/issues/16168))
- Document [`exclude_rooms_from_sync`](https://matrix-org.github.io/synapse/v1.92/usage/configuration/config_documentation.html#exclude_rooms_from_sync) configuration option. ([\#16178](https://github.com/matrix-org/synapse/issues/16178))

- Prepare unit tests for Python 3.12. ([\#16099](https://github.com/matrix-org/synapse/issues/16099))
- Fix nightly CI jobs. ([\#16121](https://github.com/matrix-org/synapse/issues/16121), [\#16213](https://github.com/matrix-org/synapse/issues/16213))
- Describe which rate limiter was hit in logs. ([\#16135](https://github.com/matrix-org/synapse/issues/16135))
- Simplify presence code when using workers. ([\#16170](https://github.com/matrix-org/synapse/issues/16170))
- Track per-device information in the presence code. ([\#16171](https://github.com/matrix-org/synapse/issues/16171), [\#16172](https://github.com/matrix-org/synapse/issues/16172))
- Stop using the `event_txn_id` table. ([\#16175](https://github.com/matrix-org/synapse/issues/16175))
- Use `AsyncMock` instead of custom code. ([\#16179](https://github.com/matrix-org/synapse/issues/16179), [\#16180](https://github.com/matrix-org/synapse/issues/16180))
- Improve error reporting of invalid data passed to `/_matrix/key/v2/query`. ([\#16183](https://github.com/matrix-org/synapse/issues/16183))
- Task scheduler: add replication notify for new task to launch ASAP. ([\#16184](https://github.com/matrix-org/synapse/issues/16184))
- Improve type hints. ([\#16186](https://github.com/matrix-org/synapse/issues/16186), [\#16188](https://github.com/matrix-org/synapse/issues/16188), [\#16201](https://github.com/matrix-org/synapse/issues/16201))
- Bump black version to 23.7.0. ([\#16187](https://github.com/matrix-org/synapse/issues/16187))
- Log the details of background update failures. ([\#16212](https://github.com/matrix-org/synapse/issues/16212))
- Cache device resync requests over replication. ([\#16241](https://github.com/matrix-org/synapse/issues/16241))

* Bump anyhow from 1.0.72 to 1.0.75. ([\#16141](https://github.com/matrix-org/synapse/issues/16141))
* Bump furo from 2023.7.26 to 2023.8.19. ([\#16238](https://github.com/matrix-org/synapse/issues/16238))
* Bump phonenumbers from 8.13.18 to 8.13.19. ([\#16237](https://github.com/matrix-org/synapse/issues/16237))
* Bump psycopg2 from 2.9.6 to 2.9.7. ([\#16196](https://github.com/matrix-org/synapse/issues/16196))
* Bump regex from 1.9.3 to 1.9.4. ([\#16195](https://github.com/matrix-org/synapse/issues/16195))
* Bump ruff from 0.0.277 to 0.0.286. ([\#16198](https://github.com/matrix-org/synapse/issues/16198))
* Bump sentry-sdk from 1.29.2 to 1.30.0. ([\#16236](https://github.com/matrix-org/synapse/issues/16236))
* Bump serde from 1.0.184 to 1.0.188. ([\#16194](https://github.com/matrix-org/synapse/issues/16194))
* Bump serde_json from 1.0.104 to 1.0.105. ([\#16140](https://github.com/matrix-org/synapse/issues/16140))
* Bump types-psycopg2 from 2.9.21.10 to 2.9.21.11. ([\#16200](https://github.com/matrix-org/synapse/issues/16200))
* Bump types-pyyaml from 6.0.12.10 to 6.0.12.11. ([\#16199](https://github.com/matrix-org/synapse/issues/16199))
2023-09-05 16:56:43 +01:00
David Robertson 79aa26936f
Amend changelog for MSC4041 2023-09-05 16:45:20 +01:00
Patrick Cloke 8b5013dcbc
Time out busy presence status & test multi-device busy (#16174)
Add a (long) timeout to when a "busy" device is considered not online.
This does *not* match MSC3026, but is a reasonable thing for an
implementation to do.

Expands tests for the (unstable) busy presence with multiple devices.
2023-09-05 10:39:38 -04:00
Patrick Cloke ea75346f6a
Track presence state per-device and combine to a user state. (#16066)
Tracks presence on an individual per-device basis and combine
the per-device state into a per-user state. This should help in
situations where a user has multiple devices with conflicting status
(e.g. one is syncing with unavailable and one is syncing with online).

The tie-breaking is done by priority:

    BUSY > ONLINE > UNAVAILABLE > OFFLINE
2023-09-05 09:58:51 -04:00
dependabot[bot] 36ae8611fe
Bump regex from 1.9.4 to 1.9.5 (#16233)
Bumps [regex](https://github.com/rust-lang/regex) from 1.9.4 to 1.9.5.
- [Release notes](https://github.com/rust-lang/regex/releases)
- [Changelog](https://github.com/rust-lang/regex/blob/master/CHANGELOG.md)
- [Commits](https://github.com/rust-lang/regex/compare/1.9.4...1.9.5)

---
updated-dependencies:
- dependency-name: regex
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-09-05 13:14:00 +00:00
Erik Johnston dfcfa9f0ed
Bump minimum supported Rust version to 1.61.0 (#16248) 2023-09-05 13:12:50 +01:00
dependabot[bot] 757010905e
Bump twisted from 22.10.0 to 23.8.0 (#16235)
* Bump twisted from 22.10.0 to 23.8.0

Bumps [twisted](https://github.com/twisted/twisted) from 22.10.0 to 23.8.0.
- [Release notes](https://github.com/twisted/twisted/releases)
- [Changelog](https://github.com/twisted/twisted/blob/trunk/NEWS.rst)
- [Commits](https://github.com/twisted/twisted/compare/twisted-22.10.0...twisted-23.8.0)

---
updated-dependencies:
- dependency-name: twisted
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>

* Fix types

* Fix lint

* Newsfile

---------

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Erik Johnston <erik@matrix.org>
2023-09-05 11:14:14 +00:00
David Robertson d77154be01
Update changelog 2023-09-05 11:28:49 +01:00
David Robertson 0425dd28f4
1.92.0rc1 2023-09-05 11:21:54 +01:00
Erik Johnston d35bed8369
Don't wake up destination transaction queue if they're not due for retry. (#16223) 2023-09-04 17:14:09 +01:00
Mathieu Velten dcb2778341
Add last_seen_ts to the admin users API (#16218) 2023-09-04 18:13:28 +02:00
David Robertson 721346631e
Merge branch 'master' into develop 2023-09-04 15:04:27 +01:00
Erik Johnston f84baecb6f
Don't reset retry timers on "valid" error codes (#16221) 2023-09-04 14:04:43 +01:00
David Robertson 1cd0715a0f
1.91.1 2023-09-04 14:04:16 +01:00
dependabot[bot] 748c38921c
Bump furo from 2023.7.26 to 2023.8.19 (#16238)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-09-04 13:25:39 +01:00
dependabot[bot] 4382d57640
Bump phonenumbers from 8.13.18 to 8.13.19 (#16237)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-09-04 12:04:50 +01:00
dependabot[bot] 8065eea6c7
Bump sentry-sdk from 1.29.2 to 1.30.0 (#16236)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-09-04 12:03:27 +01:00
David Robertson e9eb26e3af
Cache device resync requests over replication (#16241) 2023-09-04 11:57:59 +01:00
David Robertson dcd3698e1f
Tentatively update changelog
Will need to confirm this though
2023-09-01 16:09:23 +01:00
Erik Johnston b85c3485b1
Fix typo where we ended up with multiple `WorkerLocksHandler` (#16220)
I don't think has caused any actual issues.

Introduced in #15891
2023-09-01 16:03:31 +01:00
Erik Johnston 93f2fdd8d1
Fix typo where we ended up with multiple `WorkerLocksHandler` (#16220)
I don't think has caused any actual issues.

Introduced in #15891
2023-09-01 12:52:57 +00:00
David Robertson 6525fd65ee
Log the details of background update failures (#16212) 2023-09-01 12:41:56 +01:00
David Robertson ed5e8a77ca
Ignore redundant casts in latest deps CI job (#16213) 2023-08-30 22:55:47 +01:00
Erik Johnston 3de82bb2af
Gracefully handle failing to thumbnail images (#16211) 2023-08-30 15:18:34 +01:00
Erik Johnston a2e0d4cd60
Fix rare bug that broke looping calls (#16210)
* Fix rare bug that broke looping calls

We can't interact with the reactor from the main thread via looping
call.

Introduced in v1.90.0 / #15791.

* Newsfile
2023-08-30 14:18:42 +01:00
David Robertson 05d824526a
Merge branch 'master' into develop 2023-08-30 12:41:42 +01:00
David Robertson 8c56e18e47
1.91.0 2023-08-30 11:21:00 +01:00
Patrick Cloke ebd8374fb5
Stop writing to the event_txn_id table (#16175) 2023-08-30 11:10:56 +01:00
David Robertson 62a1a9be52
Describe which rate limiter was hit in logs (#16135) 2023-08-30 00:39:39 +01:00
Patrick Cloke e9235d92f2
Track currently syncing users by device for presence (#16172)
Refactoring to use both the user ID & the device ID when tracking
the currently syncing users in the presence handler.

This is done both locally and over replication. Note that the device
ID is discarded but will be used in a future change.
2023-08-29 11:44:07 -04:00
Patrick Cloke 9ec3da06da
Bump mypy-zope & mypy. (#16188) 2023-08-29 10:38:56 -04:00
dependabot[bot] 001fc7bd19
Bump ruff from 0.0.277 to 0.0.286 (#16198)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-08-29 09:41:43 -04:00
Evilham 63b51ef3fb
Support IPv6-only SMTP servers (#16155)
Use Twisted HostnameEndpoint to connect to SMTP servers (instead
of connectTCP/connectSSL) which properly supports IPv6-only servers.
2023-08-29 09:33:58 -04:00
Patrick Cloke 2d72367367
Update black & fix the targeted Python versions. (#16187)
Black should target Python 3.8 to 3.11.
2023-08-29 08:34:53 -04:00
Chen Zhang 692ee2af19
Fix inaccurate error message while trying to ban or unban a user with the same or higher PL (#16205) 2023-08-29 10:37:09 +01:00
Patrick Cloke 40901af5e0
Pass the device ID around in the presence handler (#16171)
Refactoring to pass the device ID (in addition to the user ID) through
the presence handler (specifically the `user_syncing`, `set_state`,
and `bump_presence_active_time` methods and their replication
versions).
2023-08-28 13:08:49 -04:00
Patrick Cloke 1bf143699c
Combine logic about not overriding BUSY presence. (#16170)
Simplify some of the presence code by reducing duplicated code between
worker & non-worker modes.

The main change is to push some of the logic from `user_syncing` into
`set_state`. This is done by passing whether the user is setting the presence
via a `/sync` with a new `is_sync` flag to `set_state`. If this is `true` some
additional logic is performed:

* Don't override `busy` presence.
* Update the `last_user_sync_ts`.
* Never update the status message.
2023-08-28 11:03:23 -04:00
Mathieu Velten 501da8ecd8
Task scheduler: add replication notify for new task to launch ASAP (#16184) 2023-08-28 14:03:51 +00:00
dependabot[bot] 224c2bbcfa
Bump serde from 1.0.184 to 1.0.188 (#16194)
Bumps [serde](https://github.com/serde-rs/serde) from 1.0.184 to 1.0.188.
- [Release notes](https://github.com/serde-rs/serde/releases)
- [Commits](https://github.com/serde-rs/serde/compare/v1.0.184...v1.0.188)

---
updated-dependencies:
- dependency-name: serde
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-08-28 15:38:45 +02:00
Patrick Cloke 4379d3ef63
Bump setuptools-rust from 1.6.0 to 1.7.0. (#16201) 2023-08-28 09:04:15 -04:00
dependabot[bot] 1511a55539
Bump regex from 1.9.3 to 1.9.4 (#16195)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-08-28 08:01:23 -04:00
dependabot[bot] c0bbad8a96
Bump psycopg2 from 2.9.6 to 2.9.7 (#16196)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-08-28 07:59:27 -04:00
dependabot[bot] 743860e6a6
Bump types-pyyaml from 6.0.12.10 to 6.0.12.11 (#16199)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-08-28 07:54:13 -04:00
dependabot[bot] e54c1d4ed3
Bump types-psycopg2 from 2.9.21.10 to 2.9.21.11 (#16200)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-08-28 07:53:57 -04:00
V02460 84f441f88f
Prepare unit tests for Python 3.12 (#16099) 2023-08-25 15:05:10 -04:00
Patrick Cloke ed6de4b2d4
service-identity, incremental, and setuptools-rust now have type hints. (#16186) 2023-08-25 14:10:47 -04:00
Patrick Cloke 82699428e3
Validate input to POST /key/v2/query endpoint. (#16183)
To avoid 500 internal server errors with garbage input.
2023-08-25 14:10:31 -04:00
Patrick Cloke fcf7a5759e
Send proper JSON POST data to /publicRooms (#16185)
The include_all_networks was previously sent in the JSON body as
string "true" and "false" instead of boolean true and false.
2023-08-25 12:11:40 -04:00
Patrick Cloke a8a46b1336
Replace simple_async_mock with AsyncMock (#16180)
Python 3.8 has a native AsyncMock, use it instead of a custom
implementation.
2023-08-25 09:27:21 -04:00
Will Hunt 5c9402b9fd
Add warnings about MSC3861 on certain APIs. (#16168) 2023-08-25 07:25:34 -04:00
Patrick Cloke daf11e26ef
Replace make_awaitable with AsyncMock (#16179)
Python 3.8 provides a native AsyncMock, we can replace the
homegrown version we have.
2023-08-24 19:38:46 -04:00
Shay 5856a8ba42
Document `exclude_rooms_fom_sync` configuration option (#16178) 2023-08-24 13:57:53 -07:00
Aurélien Grimpard aeeca2a62e
Add configuration setting for CAS protocol version (#15816) 2023-08-24 16:11:23 -04:00
dependabot[bot] efdb87c898
Bump anyhow from 1.0.72 to 1.0.75 (#16141)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-08-24 15:50:41 -04:00
Amirreza Aflakparast 5427cc20b9
Update URLs to matrix.org blog categories. (#16008) 2023-08-24 11:06:06 -04:00
David Robertson e691243e19
Fix typechecking with twisted trunk (#16121) 2023-08-24 14:53:07 +00:00
Will Hunt 0538e3e2db
Add `Retry-After` to M_LIMIT_EXCEEDED error responses (#16136)
Implements MSC4041 behind an experimental configuration flag.
2023-08-24 10:40:26 -04:00
dependabot[bot] e3333bacff
Bump serde_json from 1.0.104 to 1.0.105 (#16140)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-08-24 08:49:37 -04:00
H. Shay 851cbdcb57 Merge branch 'release-v1.91' into develop 2023-08-23 11:44:01 -07:00
Patrick Cloke 33fa82a34c
Stabilize support for MSC3958 (suppress notifications from edits). (#16113) 2023-08-23 13:22:34 -04:00
H. Shay 23f88f9c59 1.91.0rc1 2023-08-23 09:47:29 -07:00
H. Shay 020ff1afe3 fix changelog 2023-08-23 09:36:34 -07:00
H. Shay 7064b4bcf3 fix changelog 2023-08-23 09:25:50 -07:00
Erik Johnston 18279631e9
Fix rare deadlock when using read/write locks (#16169) 2023-08-23 16:24:30 +01:00
Andrew Morgan 85118420a2
Switch `devenv` dependency in the nix development environment to the latest release (instead of the development branch) (#16063) 2023-08-23 16:16:14 +01:00
Neil Johnson ec662bbe41
Filter out unwanted user_agents from udv. (#16124) 2023-08-23 14:00:34 +01:00
Erik Johnston 4adaba9acf
Fix rare deadlock when using read/write locks (#16133) 2023-08-23 13:45:25 +01:00
Erik Johnston 7cd79ce051
Reduce DB contention on worker locks (#16160) 2023-08-23 13:45:19 +01:00
Erik Johnston 86ecd341ec
Always update `retry_last_ts` (#16164) 2023-08-23 13:04:46 +01:00
Mathieu Velten 873971a8b9
Task scheduler: mark task as active if we are scheduling ASAP (#16165) 2023-08-23 13:37:51 +02:00
Patrick Cloke da162cbe4e
Add tests for restoring the presence state after a restart. (#16151) 2023-08-23 07:31:00 -04:00
DeepBlueV7.X 19a1cda084
Properly update retry_last_ts when hitting the maximum retry interval (#16156)
* Properly update retry_last_ts when hitting the maximum retry interval

This was broken in 1.87 when the maximum retry interval got changed from
almost infinite to a week (and made configurable).

fixes #16101

Signed-off-by: Nicolas Werner <nicolas.werner@hotmail.de>

* Add changelog

* Change fix + add test

* Add comment

---------

Signed-off-by: Nicolas Werner <nicolas.werner@hotmail.de>
Co-authored-by: Mathieu Velten <mathieuv@matrix.org>
2023-08-23 09:35:23 +01:00
Erik Johnston dffe095642
Only lock when we're backfilling (#16159) 2023-08-23 09:23:41 +01:00
Erik Johnston 3b3fed7229
Increase perf of read/write locks (#16149)
We do this by marking the tables as `UNLOGGED` in PostgreSQL.
2023-08-23 09:23:22 +01:00
Patrick Cloke 3f17178728
Clean-up presence tests (#16158)
Reduce duplicated code & remove unused variables.
2023-08-22 11:43:44 -04:00
Erik Johnston 803f63df1c
Fix perf of `wait_for_stream_positions` (#16148) 2023-08-22 15:11:22 +00:00
Mathieu Velten 0ba17777be
Disable `m.3pid_changes` capability when MSC3861 is enabled. (#16134) 2023-08-22 14:47:59 +00:00
Shay 69048f7b48
Add an admin endpoint to allow authorizing server to signal token revocations (#16125) 2023-08-22 14:15:34 +00:00
dependabot[bot] 8aa5479986
Bump serde from 1.0.183 to 1.0.184 (#16139)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-08-22 09:48:06 -04:00
Patrick Cloke b657e89005
Fix user directory test for deactivated support user. (#16157)
Support users should not be added to the user directory after
being deactivated.
2023-08-22 09:08:24 -04:00
Theodore Ni bc72d803d5
Raise poetry-core version cap to 1.7.0 (#16152) 2023-08-22 13:51:35 +01:00
Patrick Cloke 6d7c63fcc6
Properly call setup_background_tasks in unit tests. (#16150)
This should only be called on HomeServer objects which are configured
to run background tasks, which is automatically (and properly) done via
the call to setup().
2023-08-22 07:46:32 -04:00
Hugh Nimmo-Smith 7dbac123f9
Disallow user_consent where experimental MSC3861 is enabled (#16127) 2023-08-22 07:42:08 -04:00
Maximilian Bosch d6ae4041a4
Add `client_secret_path` as alternative for `client_secret` for OIDC config (#16030) 2023-08-21 19:32:17 +00:00
Mathieu Velten 358896e1b8
Implements a task scheduler for resumable potentially long running tasks (#15891) 2023-08-21 14:17:13 +02:00
dependabot[bot] 79c349dfb8
Bump ijson from 3.2.1 to 3.2.3 (#16143)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-08-21 10:45:18 +01:00
dependabot[bot] 1e5a0e07a7
Bump pygithub from 1.59.0 to 1.59.1 (#16144)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-08-21 10:44:14 +01:00
dependabot[bot] 35d260d065
Bump click from 8.1.6 to 8.1.7 (#16145)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-08-21 10:41:44 +01:00
dependabot[bot] 07c0875aa5
Bump types-pyopenssl from 23.2.0.1 to 23.2.0.2 (#16146)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-08-21 10:41:00 +01:00
dependabot[bot] 406ff3eb62
Bump sentry-sdk from 1.28.1 to 1.29.2 (#16142)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-08-21 10:39:07 +01:00
Erik Johnston bd558a6dc3
Speed up state res in rare case we don't have all events (#16116)
If we don't have all the auth events in a room then not all state events will have a chain cover index. Even so, we can still use the chain cover index on the events that do have it, rather than bailing and using the slower functions.

This situation should not arise for newly persisted rooms, as we check we have the full auth chain for each event, but can happen for existing rooms.

c.f. #15245
2023-08-18 15:32:06 +01:00
Mathieu Velten 2d15e39684
MSC3861: allow impersonation by an admin using a query param (#16132) 2023-08-18 15:46:46 +02:00
Alexander Fechler 54317d34b7
Allow filtering for admins in the list accounts admin API (#16114) 2023-08-18 12:26:38 +01:00
Erik Johnston 6130afb862
Add response time metrics for introspection requests (#16131)
See #16119
2023-08-18 12:16:00 +01:00
Erik Johnston 0aba4a4eaa
Add cache to `get_server_keys_json_for_remote` (#16123) 2023-08-18 11:05:01 +01:00
Shay 54a51ff6c1
Cache token introspection response from OIDC provider (#16117) 2023-08-17 10:53:10 -07:00
Erik Johnston eb0dbab15b
Fix database performance of read/write worker locks (#16061)
We were seeing serialization errors when taking out multiple read locks.

The transactions were retried, so isn't causing any failures.

Introduced in #15782.
2023-08-17 14:07:57 +01:00
Shay 0377cb4fab
Override global statement timeout when creating indexes in Postgres (#16085) 2023-08-17 10:30:02 +01:00
Matthew Ma 8a4fb7a6ba
Disable caching in /sync corner case (#16080)
Fixes #15502
2023-08-17 10:22:50 +01:00
Mathieu Velten 8c3bcea2da
Rename pagination&purge locks and add comments explaining them (#16112) 2023-08-16 16:19:54 +02:00
axel simon 4513b36a75
Add link explaining ELK stack to structured_logging.md (#16091) 2023-08-16 14:08:35 +01:00
David Robertson 47c629bb27
Attempt to fix twisted trunk (#16115) 2023-08-15 16:07:13 +00:00
Patrick Cloke ad3f43be9a
Run pyupgrade for python 3.7 & 3.8. (#16110) 2023-08-15 08:11:20 -04:00
Olivier Wilkinson (reivilibre) 4347473946 Merge branch 'master' into develop 2023-08-15 12:14:32 +01:00
Olivier Wilkinson (reivilibre) 29638220ab 1.90.0 2023-08-15 11:17:54 +01:00
dependabot[bot] 837f28ce74
Bump log from 0.4.19 to 0.4.20 (#16109)
Bumps [log](https://github.com/rust-lang/log) from 0.4.19 to 0.4.20.
- [Release notes](https://github.com/rust-lang/log/releases)
- [Changelog](https://github.com/rust-lang/log/blob/master/CHANGELOG.md)
- [Commits](https://github.com/rust-lang/log/compare/0.4.19...0.4.20)

---
updated-dependencies:
- dependency-name: log
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-08-15 10:58:32 +01:00
dependabot[bot] 4ce32ade5a
Bump txredisapi from 1.4.9 to 1.4.10 (#16107)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-08-14 11:57:39 -04:00
dependabot[bot] 6fc411c7bf
Bump gitpython from 3.1.31 to 3.1.32 (#16103)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-08-14 08:43:08 -04:00
dependabot[bot] e21ff0f048
Bump types-bleach from 6.0.0.3 to 6.0.0.4 (#16106)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-08-14 08:04:24 -04:00
dependabot[bot] b80ff1602e
Bump types-pillow from 10.0.0.1 to 10.0.0.2 (#16105)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-08-14 08:03:18 -04:00
dependabot[bot] d834a80a12
Bump isort from 5.11.5 to 5.12.0 (#16108)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-08-14 07:30:02 -04:00
Gabriel Rodríguez 9ff84bccbb
Allow customizing IdP name and icon for SAML and CAS (#16094) 2023-08-11 20:15:17 +00:00
Nick Mills-Barrett 614efc488b
Add linearizer on user ID to push rule PUT/DELETE requests (#16052)
See: #16053

Signed off by Nick @ Beeper (@Fizzadar)
2023-08-11 11:37:09 +00:00
reivilibre 7f4b413690
Fix the type annotation on `run_db_interaction` in the Module API. (#16089)
* Fix the method signature of `run_db_interaction` on the module API

* Newsfile

Signed-off-by: Olivier Wilkinson (reivilibre) <oliverw@matrix.org>

---------

Signed-off-by: Olivier Wilkinson (reivilibre) <oliverw@matrix.org>
2023-08-10 18:28:31 +01:00
Patrick Cloke efd4d06d76
Clean-up presence code (#16092)
Misc. clean-ups to:

* Use keyword arguments.
* Return early (reducing indentation) of some functions.
* Removing duplicated / unused code.
* Use wrap_as_background_process.
2023-08-10 07:39:46 -04:00
Mathieu Velten dac97642e4
Implements admin API to lock an user (MSC3939) (#15870) 2023-08-10 09:10:55 +00:00
Shay 0328b56468
Support MSC3814: Dehydrated Devices Part 2 (#16010) 2023-08-08 12:04:46 -07:00
Erik Johnston 4581809846 Fixup changelog 2023-08-08 15:38:45 +01:00
Erik Johnston 3dfe5c0270 Fixup changelog 2023-08-08 15:33:38 +01:00
Erik Johnston 8e09b8aecb 1.90.0rc1 2023-08-08 15:29:44 +01:00
reivilibre a476d5048b
Allow modules to schedule delayed background calls. (#15993)
* Add a module API function to provide `call_later`

* Newsfile

Signed-off-by: Olivier Wilkinson (reivilibre) <oliverw@matrix.org>

* Add comments

* Update version number

---------

Signed-off-by: Olivier Wilkinson (reivilibre) <oliverw@matrix.org>
2023-08-08 11:53:49 +01:00
reivilibre f3dc6dc19f
Remove old rows from the `cache_invalidation_stream_by_instance` table automatically. (This table is not used when Synapse is configured to use SQLite.) (#15868)
* Add a cache invalidation clean-up task

* Run the cache invalidation stream clean-up on the background worker

* Tune down

* call_later is in millis!

* Newsfile

Signed-off-by: Olivier Wilkinson (reivilibre) <oliverw@matrix.org>

* fixup! Add a cache invalidation clean-up task

* Update synapse/storage/databases/main/cache.py

Co-authored-by: Eric Eastwood <erice@element.io>

* Update synapse/storage/databases/main/cache.py

Co-authored-by: Eric Eastwood <erice@element.io>

* MILLISEC -> MS

* Expand on comment

* Move and tweak comment about Postgres

* Use `wrap_as_background_process`

---------

Signed-off-by: Olivier Wilkinson (reivilibre) <oliverw@matrix.org>
Co-authored-by: Eric Eastwood <erice@element.io>
2023-08-08 11:10:07 +01:00
Shay 8af3f33d84
Fix endpoint improperly declaring support for MSC3814 (#16068) 2023-08-07 10:52:15 -07:00
Shay 81a6f8c9ae
Drop backwards compat hack for event serialization (#16069) 2023-08-07 10:37:08 -07:00
David Robertson 9d3713d6d5
Add notes describing Synapse's streams (#16015)
Co-authored-by: Patrick Cloke <clokep@users.noreply.github.com>
2023-08-07 18:36:04 +01:00
dependabot[bot] b57630c507
Bump jsonschema from 4.18.3 to 4.19.0 (#16081)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-08-07 14:18:09 +00:00
dependabot[bot] 340f08c6f7
Bump serde from 1.0.179 to 1.0.183 (#16074)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-08-07 08:27:55 -04:00
dependabot[bot] 8da3c2185b
Bump regex from 1.9.1 to 1.9.3 (#16073)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-08-07 08:23:20 -04:00
dependabot[bot] eca592b121
Bump types-opentracing from 2.4.10.5 to 2.4.10.6 (#16078)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-08-07 12:19:45 +00:00
dependabot[bot] 34b5db1fbc
Bump furo from 2023.5.20 to 2023.7.26 (#16077)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-08-07 08:15:00 -04:00
dependabot[bot] ec8499206e
Bump types-setuptools from 68.0.0.0 to 68.0.0.3 (#16079)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-08-07 10:59:04 +00:00
dependabot[bot] 4f6da0dba0
Bump phonenumbers from 8.13.14 to 8.13.18 (#16076)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-08-07 10:52:04 +00:00
Shay 84ae2e3f6f
Fix deletion for Dehydrated Devices (#16046) 2023-08-04 10:49:54 -07:00
Patrick Cloke d98a43d922
Stabilize support for MSC3970: updated transaction semantics (scope to `device_id`) (#15629)
For now this maintains compatible with old Synapses by falling back
to using transaction semantics on a per-access token. A future version
of Synapse will drop support for this.
2023-08-04 07:47:18 -04:00
Shay 0a5f4f7665
Move support for application service query parameter authorization behind a configuration option (#16017) 2023-08-03 11:43:51 -07:00
Mathieu Velten f0a860908b
Allow config of the backoff algorithm for the federation client. (#15754)
Adds three new configuration variables:

* destination_min_retry_interval is identical to before (10mn).
* destination_retry_multiplier is now 2 instead of 5, the maximum value will
  be reached slower.
* destination_max_retry_interval is one day instead of (essentially) infinity.

Capping this will cause destinations to continue to be retried sometimes instead
of being lost forever. The previous value was 2 ^ 62 milliseconds.
2023-08-03 14:36:55 -04:00
reivilibre 9c462f18a4
Allow modules to check whether the current worker is configured to run background tasks. (#15991) 2023-08-03 08:42:19 -04:00
Patrick Cloke 4f5bccbbba
Add forward-compatibility for the redacts property (MSC2174). (#16013)
The location of the redacts field changes in room version 11. Ensure
it is copied to the *new* location for *old* room versions for
forwards-compatibility with clients.

Note that copying it to the *old* location for the *new* room version
was previously handled.
2023-08-02 15:35:54 +00:00
Patrick Cloke 01a45869f0
Update MSC3958 support to interact with intentional mentions. (#15992)
* Updates the rule ID.
* Use `event_property_is` instead of `event_match`.

This updates the implementation of MSC3958 to match the latest
text from the MSC.
2023-08-02 08:41:32 -04:00
dependabot[bot] ca5d5de79b
Bump cryptography from 41.0.2 to 41.0.3 (#16048)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-08-02 09:46:32 +00:00
Andrew Morgan a51b0862a1
Update `flake.lock` to fix running the nix developer environment on MacOS (#16019) 2023-08-02 07:47:16 +01:00
Patrick Cloke 8fe1fd906a
Update certifi to 2023.7.22 and pygments to 2.15.1. (#16044) 2023-08-01 15:55:58 +00:00
Patrick Cloke 90ad836ed8
Properly setup the additional sequences in the portdb script. (#16043)
The un_partial_stated_event_stream_sequence and
application_services_txn_id_seq were never properly configured
in the portdb script, resulting in an error on start-up.
2023-08-01 10:36:33 -04:00
Mohit Rathee 5eb3fd785b
Trim whitespace when setting display names (#16031) 2023-08-01 09:14:02 -04:00
Jason Little 7cbb2a00d1
Add metrics tracking for eviction to ResponseCache (#16028)
Track whether the ResponseCache is evicting due to invalidation
or due to time.
2023-08-01 08:10:49 -04:00
David Robertson a4102d2a5f
Merge branch 'master' into develop 2023-08-01 12:01:34 +01:00
David Robertson 190c990a76
1.89.0 2023-08-01 11:09:30 +01:00
Patrick Cloke b7695ac388
Combine duplicated code for calculating an event ID from a txn ID (#16023)
Refactoring related to stabilization of MSC3970, refactor to combine
code which has the same logic.
2023-07-31 08:44:45 -04:00
dependabot[bot] 1fb5a7ad5d
Bump serde from 1.0.175 to 1.0.179 (#16033)
Bumps [serde](https://github.com/serde-rs/serde) from 1.0.175 to 1.0.179.
- [Release notes](https://github.com/serde-rs/serde/releases)
- [Commits](https://github.com/serde-rs/serde/compare/v1.0.175...v1.0.179)

---
updated-dependencies:
- dependency-name: serde
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-07-31 14:08:35 +02:00
dependabot[bot] fa2c116bef
Bump immutabledict from 2.2.4 to 3.0.0 (#16034)
Bumps [immutabledict](https://github.com/corenting/immutabledict) from 2.2.4 to 3.0.0.
- [Release notes](https://github.com/corenting/immutabledict/releases)
- [Changelog](https://github.com/corenting/immutabledict/blob/master/CHANGELOG.md)
- [Commits](https://github.com/corenting/immutabledict/compare/v2.2.4...v3.0.0)

---
updated-dependencies:
- dependency-name: immutabledict
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-07-31 13:27:17 +02:00
Nils e02f4b7de2
Do not expose Admin API in caddy reverse proxy example (#16027)
Signed-off-by: Nils ANDRÉ-CHANG <nils@nilsand.re>
2023-07-31 13:25:06 +02:00
dependabot[bot] 21407c6709
Bump service-identity from 21.1.0 to 23.1.0 (#16038)
Bumps [service-identity](https://github.com/pyca/service-identity) from 21.1.0 to 23.1.0.
- [Release notes](https://github.com/pyca/service-identity/releases)
- [Changelog](https://github.com/pyca/service-identity/blob/main/CHANGELOG.md)
- [Commits](https://github.com/pyca/service-identity/compare/21.1.0...23.1.0)

---
updated-dependencies:
- dependency-name: service-identity
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-07-31 13:24:32 +02:00
Erik Johnston ae55cc1e6b
Add ability to wait for locks and add locks to purge history / room deletion (#15791)
c.f. #13476
2023-07-31 10:58:03 +01:00
dependabot[bot] 0c6142c4a1
Bump types-commonmark from 0.9.2.3 to 0.9.2.4 (#16037)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-07-31 10:47:25 +01:00
dependabot[bot] fee0195b27
Bump serde_json from 1.0.103 to 1.0.104 (#16032)
Bumps [serde_json](https://github.com/serde-rs/json) from 1.0.103 to 1.0.104.
- [Release notes](https://github.com/serde-rs/json/releases)
- [Commits](https://github.com/serde-rs/json/compare/v1.0.103...v1.0.104)

---
updated-dependencies:
- dependency-name: serde_json
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-07-31 10:23:00 +02:00
dependabot[bot] 76b2218599
Bump types-jsonschema from 4.17.0.8 to 4.17.0.10 (#16036)
Bumps [types-jsonschema](https://github.com/python/typeshed) from 4.17.0.8 to 4.17.0.10.
- [Commits](https://github.com/python/typeshed/commits)

---
updated-dependencies:
- dependency-name: types-jsonschema
  dependency-type: direct:development
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-07-31 10:21:48 +02:00
dependabot[bot] ea4ece3fcc
Bump types-netaddr from 0.8.0.8 to 0.8.0.9 (#16035)
Bumps [types-netaddr](https://github.com/python/typeshed) from 0.8.0.8 to 0.8.0.9.
- [Commits](https://github.com/python/typeshed/commits)

---
updated-dependencies:
- dependency-name: types-netaddr
  dependency-type: direct:development
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-07-31 10:21:34 +02:00
Shay 68b2611783
Clarify comment on key uploads over replication (#16016) 2023-07-27 15:08:46 -07:00
Mathieu Velten a719b703d9
Fix 404 on /profile when the display name is empty but not the avatar (#16012) 2023-07-27 15:45:05 +02:00
Mathieu Velten a461f1f846
Update PyYAML to 6.0.1 (#16011) 2023-07-27 14:51:26 +02:00
David Robertson f9f3e89354
Attempt to fix labelling in docker workflow (#16009) 2023-07-27 13:47:48 +01:00
Shay f98f4f2e16
Remove support for legacy application service paths (#15964) 2023-07-26 12:59:47 -07:00
Anshul Madnawat 58f8305114
Inline SQL queries using boolean parameters (#15525)
SQLite now supports TRUE and FALSE constants, simplify some
queries by inlining those instead of passing them as arguments.
2023-07-26 18:45:47 +00:00
Mo Balaa 96529c4236
Add synapse version as Docker container label (#15972)
Co-authored-by: Mo Balaa <balaa@fractalnetworks.co>
2023-07-26 16:16:12 +00:00
Mathieu Velten 6dc019d9dd Merge branch 'release-v1.89' into develop 2023-07-26 17:07:42 +02:00
dependabot[bot] 8d2a5586f7
Bump serde from 1.0.171 to 1.0.175 (#15982)
Bumps [serde](https://github.com/serde-rs/serde) from 1.0.171 to 1.0.175.
- [Release notes](https://github.com/serde-rs/serde/releases)
- [Commits](https://github.com/serde-rs/serde/compare/v1.0.171...v1.0.175)

---
updated-dependencies:
- dependency-name: serde
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-07-26 15:16:39 +01:00
Mathieu Velten 76e392b0fa Edit changelog 2023-07-26 16:13:39 +02:00
Mathieu Velten d4ea465496 Remove changelog file 2023-07-26 14:54:37 +02:00
Mathieu Velten 8ebfd577e2
Bump DB version to 79 since synapse v1.88 was already there (#15998) 2023-07-26 14:51:44 +02:00
Mathieu Velten dbee081d14 1.89.0rc1 2023-07-25 14:32:47 +02:00
dependabot[bot] 99b7b801c3
Bump pygithub from 1.58.2 to 1.59.0 (#15834)
Bumps [pygithub](https://github.com/pygithub/pygithub) from 1.58.2 to 1.59.0.
- [Release notes](https://github.com/pygithub/pygithub/releases)
- [Changelog](https://github.com/PyGithub/PyGithub/blob/main/doc/changes.rst)
- [Commits](https://github.com/pygithub/pygithub/compare/v1.58.2...v1.59.0)

---
updated-dependencies:
- dependency-name: pygithub
  dependency-type: direct:development
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-07-25 14:19:46 +02:00
Shay 641ff9ef7e
Support MSC3814: Dehydrated Devices (#15929)
Signed-off-by: Nicolas Werner <n.werner@famedly.com>
Co-authored-by: Nicolas Werner <n.werner@famedly.com>
Co-authored-by: Nicolas Werner <89468146+nico-famedly@users.noreply.github.com>
Co-authored-by: Hubert Chathi <hubert@uhoreg.ca>
2023-07-24 08:23:19 -07:00
SnipeX_ 05f8dada8b
Fix broken Arch Linux package link (#15981) 2023-07-24 09:06:10 -04:00
Erik Johnston 654902a758
Resync stale devices in background (#15975)
This is so we don't block responding to federation transaction while we
try and fetch the device lists.
2023-07-24 13:43:43 +01:00
dependabot[bot] 4a711bf379
Bump click from 8.1.3 to 8.1.6 (#15984)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-07-24 10:17:02 +01:00
dependabot[bot] fc566cdf0a
Bump sentry-sdk from 1.26.0 to 1.28.1 (#15985)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-07-24 10:16:03 +01:00
dependabot[bot] 3b6208b835
Bump pillow from 9.4.0 to 10.0.0 (#15986)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-07-24 10:12:02 +01:00
dependabot[bot] 3b8348b06e
Bump types-requests from 2.31.0.1 to 2.31.0.2 (#15983)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-07-24 10:03:05 +01:00
Shay 5c7364fea5
Properly handle redactions of creation events (#15973) 2023-07-23 16:32:01 -07:00
Shay f08d05dd2c
Actually stop reading from column `user_id` of tables `profiles` (#15955) 2023-07-23 16:30:54 -07:00
Shay e1fa42249c
Build packages for Debian Trixie (#15961) 2023-07-23 16:30:05 -07:00
Erik Johnston fc1e534e41
Speed up updating state in large rooms (#15971)
This should speed up updating state in rooms with lots of state.
2023-07-20 15:51:28 +01:00
Will Lewis 835174180b
Fixed grafana deploy annotations in the dashboard config, so it shows for those not managing matrix.org (#15957)
Removed the 'matrix.org' hardcorded instance setting

Originally introduced in #15674

Co-authored-by: wrjlewis <will.lewis@askattest.com>
2023-07-20 12:33:06 +00:00
Erik Johnston fd44053b84
Don't log exceptions for every non-200 response (#15969)
Introduced in #15913
2023-07-20 11:07:58 +01:00
Erik Johnston ad52db3b5c
Reduce the amount of state we pull out (#15968) 2023-07-20 10:46:37 +01:00
Erik Johnston 67f9e5293e
Ensure a long state res does not starve CPU (#15960)
We do this by yielding the reactor in hot loops.
2023-07-19 17:00:33 +00:00
Erik Johnston 19796e20aa
Fix bad merge of #15933 (#15958)
This was because we reverted the bump of the schema version, so we were not applying the new deltas.
2023-07-19 12:17:08 +00:00
Erik Johnston 40a3583ba1
Fix race in triggers for read/write locks. (#15933) 2023-07-19 12:06:38 +01:00
Shay cb6e2c6cc7
Fix background schema updates failing over a large upgrade gap (#15887) 2023-07-18 16:59:27 -07:00
Olivier Wilkinson (reivilibre) 8e8431bc6e Merge branch 'master' into develop 2023-07-18 16:45:39 +01:00
Olivier Wilkinson (reivilibre) 69699a9bd1 1.88.0 2023-07-18 14:06:00 +01:00
Patrick Cloke 6d81aec09f
Support room version 11 (#15912)
And fix a bug in the implementation of the updated redaction
format (MSC2174) where the top-level redacts field was not
properly added for backwards-compatibility.
2023-07-18 08:44:59 -04:00
Shay e625c3dca0
Revert "Stop writing to column `user_id` of tables `profiles` and `user_filters`. (#15953)
* Revert "Stop writing to column `user_id` of tables `profiles` and `user_filters` (#15787)"

This reverts commit f25b0f8808.

* newsfragement
2023-07-18 11:44:09 +01:00
Jason Little 199c270947
Add a locality to a few presence metrics (#15952) 2023-07-18 10:36:40 +01:00
Eric Eastwood 1c802de626
Re-introduce the outbound federation proxy (#15913)
Allow configuring the set of workers to proxy outbound federation traffic through (`outbound_federation_restricted_to`).

This is useful when you have a worker setup with `federation_sender` instances responsible for sending outbound federation requests and want to make sure *all* outbound federation traffic goes through those instances. Before this change, the generic workers would still contact federation themselves for things like profile lookups, backfill, etc. This PR allows you to set more strict access controls/firewall for all workers and only allow the `federation_sender`'s to contact the outside world.
2023-07-18 09:49:21 +01:00
dependabot[bot] c692283751
Bump anyhow from 1.0.71 to 1.0.72 (#15949)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-07-17 13:20:34 +01:00
dependabot[bot] 43ee5d5bac
Bump pyo3-log from 0.8.2 to 0.8.3 (#15951)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-07-17 10:46:26 +01:00
dependabot[bot] 1768dd3c27
Bump serde_json from 1.0.100 to 1.0.103 (#15950)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-07-17 10:45:46 +01:00
dependabot[bot] 0d522b58a6
Bump jsonschema from 4.17.3 to 4.18.3 (#15948)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-07-17 10:39:51 +01:00
dependabot[bot] b0e66721a5
Bump typing-extensions from 4.5.0 to 4.7.1 (#15947)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-07-17 10:33:47 +01:00
dependabot[bot] 6396527015
Bump pydantic from 1.10.10 to 1.10.11 (#15946)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-07-17 10:30:46 +01:00
dependabot[bot] d2f46ae370
Bump prometheus-client from 0.17.0 to 0.17.1 (#15945)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-07-17 10:30:10 +01:00
Andrew Morgan 85e0541db1
Pin the rust version in `flake.nix`, and bump to 1.70.0 to fix installing `ruff` (#15940) 2023-07-17 09:36:12 +01:00
dependabot[bot] cba2df20b5
Bump cryptography from 41.0.1 to 41.0.2 (#15943)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-07-15 21:37:59 +01:00
Will Hunt 8d3656b994
Document that you cannot login as yourself on /_synapse/admin/v1/users/<user_id>/login (#15938) 2023-07-14 08:32:13 -04:00
Patrick Cloke 20ae617d14
Stop accepting 'user' parameter for application service registration. (#15928)
This is unspecced, but has existed for a very long time.
2023-07-13 07:23:56 -04:00
dependabot[bot] 2cacd0849a
Bump types-pillow from 9.5.0.4 to 10.0.0.1 (#15932)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-07-13 11:21:28 +01:00
Patrick Cloke 204b66c203
Remove unneeded __init__. (#15926)
Remove an __init__ which only calls super() without changing the
input arguments.
2023-07-12 14:30:05 +00:00
Patrick Cloke 5bdf01fccd
Fix running with an empty experimental features section. (#15925) 2023-07-12 12:39:25 +00:00
Erik Johnston 36c6b92bfc
Fix push for invites received over federation (#15820) 2023-07-12 11:02:11 +00:00
Mathieu Velten 8eb7bb975e
Mark get_user_in_directory private since only used in tests (#15884) 2023-07-12 11:09:13 +02:00
Eric Eastwood 3bdb9b07fd
Make it more obvious which Python version runs on a given Linux distribution (#15909)
Make it more obvious which Python version runs on a given Linux distribution so when we end up dropping support for a given Python version, we can more easily find the reference to the Python version and remove any references for the distribution. We don't want to be running tests or building packages on a distribution that no longer has a supported Python version.

This way, we can avoid another situation like when we dropped support for Python 3.7 but forgot to drop the Debian Buster references everywhere (https://github.com/matrix-org/synapse/pull/15893)
2023-07-11 17:15:06 -05:00
Eric Eastwood 0371a354cf
Better clarify how to run a worker instance (pass both configs) (#15921)
Previously, if you just followed the instructions per the docs, you just ran into an error:

```sh
$ poetry run synapse_worker --config-path homeserver_generic_worker1.yaml

Missing mandatory `server_name` config option.
```
2023-07-11 17:13:54 -05:00
Eric Eastwood ae391db777
Better warning in logs when we fail to fetch an alias (#15922)
**Before:**
```
Error retrieving alias
```

**After:**
```
Error retrieving alias #foo:bar -> 401 Unauthorized
```

*Spawning from creating the [manual testing strategy for the outbound federation proxy](https://github.com/matrix-org/synapse/pull/15773).*
2023-07-11 17:12:41 -05:00
Eric Eastwood d7fc87d973
Bump Unix sockets intro version (#15924)
https://github.com/matrix-org/synapse/pull/15708 didn't quite make the cut for `1.88.0` this morning.
2023-07-11 15:32:50 -05:00
Jason Little 224ef0b669
Unix Sockets for HTTP Replication (#15708)
Unix socket support for `federation` and `client` Listeners has existed now for a little while(since [1.81.0](https://github.com/matrix-org/synapse/pull/15353)), but there was one last hold out before it could be complete: HTTP Replication communication. This should finish it up. The Listeners would have always worked, but would have had no way to be talked to/at.

---------

Co-authored-by: Eric Eastwood <madlittlemods@gmail.com>
Co-authored-by: Olivier Wilkinson (reivilibre) <oliverw@matrix.org>
Co-authored-by: Eric Eastwood <erice@element.io>
2023-07-11 13:08:06 -05:00
Patrick Cloke a4243183f0
Add + as an allowed character for Matrix IDs (MSC4009) (#15911) 2023-07-11 12:21:00 -04:00
David Robertson 92014fbf72
Don't build wheels for Python 3.7 (#15917)
* Don't build wheels for CPython or PyPy 3.7

* Update pyproject.toml comments

* Manually update the changelog
2023-07-11 15:16:19 +01:00
David Robertson 4ccfa16081
Call out upgrade notes in README 2023-07-11 10:34:09 +01:00
David Robertson 7c7bd9898b
1.88.0rc1 2023-07-11 10:28:11 +01:00
Michael Telatynski b516d91999
Add `Server` to Access-Control-Expose-Headers header (#15908) 2023-07-11 09:18:50 +01:00
Eric Eastwood 2328e90fbb
Make the media `/upload` tracing less ambiguous (#15888)
A lot of the functions have the same name in this space like `store_file`,
and we also do it multiple times for different reasons (main media repo,
other storage providers, thumbnails, etc) so it's good to differentiate
them so your head doesn't explode.

Follow-up to https://github.com/matrix-org/synapse/pull/15850

Tracing instrumentation to media `/upload` code paths to investigate https://github.com/matrix-org/synapse/issues/15841
2023-07-10 17:23:11 -05:00
Shay 5e82b07d2c
Drop debian buster (#15893) 2023-07-10 10:39:36 -07:00
Eric Eastwood c9bf644fa0
Revert "Federation outbound proxy" (#15910)
Revert "Federation outbound proxy (#15773)"

This reverts commit b07b14b494.
2023-07-10 11:10:20 -05:00
Eric Eastwood a704a35dd7 Revert "Placeholder changelog"
This reverts commit 6e731e86bf.
2023-07-10 10:26:04 -05:00
Erik Johnston e55a9b3e41
Fix downgrading to previous version of Synapse (#15907)
We do this by marking the constraint as deferrable.
2023-07-10 16:24:42 +01:00
Erik Johnston 6774f265b4
Fix building rust with nightly (#15906)
Also fix up a warning.
2023-07-10 16:24:04 +01:00
Eric Eastwood 6e731e86bf Placeholder changelog 2023-07-10 10:23:30 -05:00
dependabot[bot] c971698bff
Bump regex from 1.8.4 to 1.9.1 (#15902)
Bumps [regex](https://github.com/rust-lang/regex) from 1.8.4 to 1.9.1.
- [Release notes](https://github.com/rust-lang/regex/releases)
- [Changelog](https://github.com/rust-lang/regex/blob/master/CHANGELOG.md)
- [Commits](https://github.com/rust-lang/regex/compare/1.8.4...1.9.1)

---
updated-dependencies:
- dependency-name: regex
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-07-10 15:55:39 +01:00
dependabot[bot] 7477f43fd8
Bump serde_json from 1.0.99 to 1.0.100 (#15901)
Bumps [serde_json](https://github.com/serde-rs/json) from 1.0.99 to 1.0.100.
- [Release notes](https://github.com/serde-rs/json/releases)
- [Commits](https://github.com/serde-rs/json/compare/v1.0.99...v1.0.100)

---
updated-dependencies:
- dependency-name: serde_json
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-07-10 15:34:26 +02:00
dependabot[bot] 3710fea19d
Bump ruff from 0.0.275 to 0.0.277 (#15900)
Bumps [ruff](https://github.com/astral-sh/ruff) from 0.0.275 to 0.0.277.
- [Release notes](https://github.com/astral-sh/ruff/releases)
- [Changelog](https://github.com/astral-sh/ruff/blob/main/BREAKING_CHANGES.md)
- [Commits](https://github.com/astral-sh/ruff/compare/v0.0.275...v0.0.277)

---
updated-dependencies:
- dependency-name: ruff
  dependency-type: direct:development
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-07-10 10:16:03 +00:00
dependabot[bot] df8c8a4f45
Bump lxml from 4.9.2 to 4.9.3 (#15897)
Bumps [lxml](https://github.com/lxml/lxml) from 4.9.2 to 4.9.3.
- [Release notes](https://github.com/lxml/lxml/releases)
- [Changelog](https://github.com/lxml/lxml/blob/master/CHANGES.txt)
- [Commits](https://github.com/lxml/lxml/compare/lxml-4.9.2...lxml-4.9.3)

---
updated-dependencies:
- dependency-name: lxml
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-07-10 10:24:11 +01:00
Shay 8a529e4fb6
Stop running sytest on buster/python3.7 (#15892) 2023-07-07 12:04:55 -07:00
Shay f25b0f8808
Stop writing to column `user_id` of tables `profiles` and `user_filters` (#15787) 2023-07-07 09:23:27 -07:00
Dirk Klimpel 677272caed
Remove `worker_replication_*` settings from worker doc (#15872)
Co-authored-by: Mathieu Velten <mathieuv@matrix.org>
2023-07-07 08:09:41 +00:00
Jason Little 2481b7dfa4
Remove `worker_replication_*` deprecated settings, with helpful errors on startup (#15860)
Co-authored-by: reivilibre <oliverw@matrix.org>
2023-07-07 07:45:25 +00:00
sarthak shah f19dd39dfc
Update link to the clients webpage, fix #15825 (#15874) 2023-07-06 17:28:09 +02:00
Eric Eastwood b07b14b494
Federation outbound proxy (#15773)
Allow configuring the set of workers to proxy outbound federation traffic through (`outbound_federation_restricted_to`).

This is useful when you have a worker setup with `federation_sender` instances responsible for sending outbound federation requests and want to make sure *all* outbound federation traffic goes through those instances. Before this change, the generic workers would still contact federation themselves for things like profile lookups, backfill, etc. This PR allows you to set more strict access controls/firewall for all workers and only allow the `federation_sender`'s to contact the outside world.

The original code is from @erikjohnston's branches which I've gotten in-shape to merge.
2023-07-05 18:53:55 -05:00
Eric Eastwood 561d06b481
Remove support for Python 3.7 (#15851)
Fix https://github.com/matrix-org/synapse/issues/15836
2023-07-05 18:45:42 -05:00
Erik Johnston 39d131b016
Add basic read/write lock (#15782) 2023-07-05 17:25:00 +01:00
Eric Eastwood ce857c05d5
Add tracing to media `/upload` endpoint (#15850)
Add tracing instrumentation to media `/upload` code paths to investigate https://github.com/matrix-org/synapse/issues/15841
2023-07-05 10:22:21 -05:00
Sumner Evans cc780b3f77
docs/admin_api: fix header level on 'Users' page (#15852)
Signed-off-by: Sumner Evans <sumner@beeper.com>
2023-07-05 16:15:56 +02:00
Jason Little 4cf9f92f39
Fix could not serialize access due to concurrent `DELETE` from presence_stream (#15826)
* Change update_presence to have a isolation level of READ_COMMITTED

* changelog
2023-07-05 11:44:02 +01:00
Erik Johnston 95a96b21eb
Add foreign key constraint to `event_forward_extremities`. (#15751) 2023-07-05 09:43:19 +00:00
an0nfunc c303eca8cc
use Image.LANCZOS instead of Image.ANTIALIAS for thumbnail resize (#15876)
Image.ANTIALIAS is not defined in current pillow releases. Since ANTIALIAS was just using LANCZOS anyways, this is just a cosmetic change, but makes synapse work with most recent pillow releases.

Signed-off-by: Giovanni Harting <539@idlegandalf.com>
2023-07-05 10:52:12 +02:00
Michael Weimann c8e81898b6
Add not_user_type param to the list accounts admin API (#15844)
Signed-off-by: Michael Weimann <michaelw@element.io>
2023-07-04 15:03:20 -07:00
Olivier Wilkinson (reivilibre) 861752b3aa Merge branch 'master' into develop 2023-07-04 17:40:37 +01:00
Olivier Wilkinson (reivilibre) 1294d10c70 Add notes about Python 3.7 EOL 2023-07-04 16:34:41 +01:00
Olivier Wilkinson (reivilibre) 718d7dfef2 Move warning up to the top 2023-07-04 16:26:50 +01:00
Olivier Wilkinson (reivilibre) 664ba14080 1.87.0 2023-07-04 16:25:33 +01:00
Paarth Shah 649848627c Pin `pydantic` to <2.0.0 (#15862)
Signed-off-by: Paarth Shah <mail@shahpaarth.com>
2023-07-04 16:22:33 +01:00
Paarth Shah 670d590f8a
Pin `pydantic` to <2.0.0 (#15862)
Signed-off-by: Paarth Shah <mail@shahpaarth.com>
2023-07-04 09:33:24 +02:00
pacien 07d7cbfe69
devices: use combined ANY clause for faster cleanup (#15861)
Old device entries for the same user were being removed in individual
SQL commands, making the batch take way longer than necessary.

This combines the commands into a single one with a IN/ANY clause.

Example of log entry before the change, regularly observed with
"log_min_duration_statement = 10000" in PostgreSQL's config:

    LOG:  duration: 42538.282 ms  statement:
    DELETE FROM device_lists_stream
    WHERE user_id = '@someone' AND device_id = 'someid1'
    AND stream_id < 123456789
    ;
    DELETE FROM device_lists_stream
    WHERE user_id = '@someone' AND device_id = 'someid2'
    AND stream_id < 123456789
    ;
    [repeated for each device ID of that user, potentially a lot...]

With the patch applied on my instance for the past couple of days, I
no longer notice overly long statements of that particular kind.

Signed-off-by: pacien <pacien.trangirard@pacien.net>
2023-07-03 16:39:38 +02:00
reivilibre cd8b73aa97
Fix the `devenv up` configuration which was ignoring the config overrides. (#15854)
* Fix use of config override directory in `devenv up`

`--config-directory` is for the generate config script; `-c` is for usage

* Add homeserver config override directory to gitignore

* Newsfile

Signed-off-by: Olivier Wilkinson (reivilibre) <oliverw@matrix.org>

---------

Signed-off-by: Olivier Wilkinson (reivilibre) <oliverw@matrix.org>
2023-07-03 11:39:52 +01:00
reivilibre 53aa26eddc
Add a timeout that aborts any Postgres statement taking more than 1 hour. (#15853)
* Add a timeout to Postgres statements

* Newsfile

Signed-off-by: Olivier Wilkinson (reivilibre) <oliverw@matrix.org>

---------

Signed-off-by: Olivier Wilkinson (reivilibre) <oliverw@matrix.org>
2023-07-03 11:38:57 +01:00
dependabot[bot] a587de96b8
Bump sentry-sdk from 1.25.1 to 1.26.0 (#15867)
Bumps [sentry-sdk](https://github.com/getsentry/sentry-python) from 1.25.1 to 1.26.0.
- [Release notes](https://github.com/getsentry/sentry-python/releases)
- [Changelog](https://github.com/getsentry/sentry-python/blob/master/CHANGELOG.md)
- [Commits](https://github.com/getsentry/sentry-python/compare/1.25.1...1.26.0)

---
updated-dependencies:
- dependency-name: sentry-sdk
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-07-03 12:34:57 +02:00
dependabot[bot] 411ba44790
Bump types-pyopenssl from 23.2.0.0 to 23.2.0.1 (#15866)
Bumps [types-pyopenssl](https://github.com/python/typeshed) from 23.2.0.0 to 23.2.0.1.
- [Commits](https://github.com/python/typeshed/commits)

---
updated-dependencies:
- dependency-name: types-pyopenssl
  dependency-type: direct:development
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-07-03 12:34:20 +02:00
dependabot[bot] aea94ca8cd
Bump importlib-metadata from 6.6.0 to 6.7.0 (#15865)
Bumps [importlib-metadata](https://github.com/python/importlib_metadata) from 6.6.0 to 6.7.0.
- [Release notes](https://github.com/python/importlib_metadata/releases)
- [Changelog](https://github.com/python/importlib_metadata/blob/main/NEWS.rst)
- [Commits](https://github.com/python/importlib_metadata/compare/v6.6.0...v6.7.0)

---
updated-dependencies:
- dependency-name: importlib-metadata
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-07-03 12:33:47 +02:00
dependabot[bot] 9345361c6b
Bump authlib from 1.2.0 to 1.2.1 (#15864)
Bumps [authlib](https://github.com/lepture/authlib) from 1.2.0 to 1.2.1.
- [Release notes](https://github.com/lepture/authlib/releases)
- [Changelog](https://github.com/lepture/authlib/blob/master/docs/changelog.rst)
- [Commits](https://github.com/lepture/authlib/compare/v1.2.0...v1.2.1)

---
updated-dependencies:
- dependency-name: authlib
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-07-03 12:33:27 +02:00
Eric Eastwood 13fc89148c
Split out 2022 changes from the changelog (#15846)
Split out 2022 changes from the changelog so the rendered version in GitHub doesn't timeout as much.
2023-06-28 15:10:33 -05:00
Eric Eastwood 10ed3e233e Note last release with Python 3.7 support
See https://github.com/matrix-org/synapse/issues/15836
2023-06-27 10:34:11 -05:00
Eric Eastwood 472c2c72f6 Prepare changelog for v1.87.0rc1 2023-06-27 10:29:20 -05:00
Shay 78cfa55dad
Fix sqlite `user_filters` upgrade (#15817) 2023-06-27 09:41:42 +01:00
dependabot[bot] 14c1bfd534
Bump serde_json from 1.0.97 to 1.0.99 (#15832)
Bumps [serde_json](https://github.com/serde-rs/json) from 1.0.97 to 1.0.99.
- [Release notes](https://github.com/serde-rs/json/releases)
- [Commits](https://github.com/serde-rs/json/compare/v1.0.97...v1.0.99)

---
updated-dependencies:
- dependency-name: serde_json
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-06-26 15:36:33 +01:00
dependabot[bot] 70dc44f667
Bump towncrier from 22.12.0 to 23.6.0 (#15831)
Bumps [towncrier](https://github.com/twisted/towncrier) from 22.12.0 to 23.6.0.
- [Release notes](https://github.com/twisted/towncrier/releases)
- [Changelog](https://github.com/twisted/towncrier/blob/trunk/NEWS.rst)
- [Commits](https://github.com/twisted/towncrier/compare/22.12.0...23.6.0)

---
updated-dependencies:
- dependency-name: towncrier
  dependency-type: direct:development
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-06-26 15:36:07 +01:00
Erik Johnston 25c55a9d22
Add login spam checker API (#15838) 2023-06-26 14:12:20 +00:00
dependabot[bot] 52d8131e87
Bump types-opentracing from 2.4.10.4 to 2.4.10.5 (#15830)
Bumps [types-opentracing](https://github.com/python/typeshed) from 2.4.10.4 to 2.4.10.5.
- [Commits](https://github.com/python/typeshed/commits)

---
updated-dependencies:
- dependency-name: types-opentracing
  dependency-type: direct:development
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-06-26 09:26:01 +01:00
dependabot[bot] 53ea381ec3
Bump ruff from 0.0.272 to 0.0.275 (#15833)
Bumps [ruff](https://github.com/astral-sh/ruff) from 0.0.272 to 0.0.275.
- [Release notes](https://github.com/astral-sh/ruff/releases)
- [Changelog](https://github.com/astral-sh/ruff/blob/main/BREAKING_CHANGES.md)
- [Commits](https://github.com/astral-sh/ruff/compare/v0.0.272...v0.0.275)

---
updated-dependencies:
- dependency-name: ruff
  dependency-type: direct:development
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-06-26 09:14:20 +01:00
dependabot[bot] 6e65ca0b36
Bump types-setuptools from 67.8.0.0 to 68.0.0.0 (#15835)
Bumps [types-setuptools](https://github.com/python/typeshed) from 67.8.0.0 to 68.0.0.0.
- [Commits](https://github.com/python/typeshed/commits)

---
updated-dependencies:
- dependency-name: types-setuptools
  dependency-type: direct:development
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-06-26 09:09:57 +01:00
dependabot[bot] d535473520
Bump cryptography from 40.0.2 to 41.0.1 (#15800)
Bumps [cryptography](https://github.com/pyca/cryptography) from 40.0.2 to 41.0.1.
- [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst)
- [Commits](https://github.com/pyca/cryptography/compare/40.0.2...41.0.1)

---
updated-dependencies:
- dependency-name: cryptography
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-06-22 16:32:53 +01:00
Nicolas Werner e0c39d6bb5
Fix forgotten rooms missing in initial sync (#15815)
If you leave a room and forget it, then rejoin it, the room would be
missing from the next initial sync.

fixes #13262

Signed-off-by: Nicolas Werner <n.werner@famedly.com>
2023-06-21 14:56:31 +01:00
Erik Johnston 289ce3b8d9
Fix harmless exception in port DB script (#15814)
The port DB script would try and run database background tasks, which
could fail if the data they acted on was in the process of being ported.
These exceptions were non fatal.

Fixes #15789
2023-06-21 13:20:46 +00:00
Erik Johnston 6c749c5124
Fix typo in faster join docs (#15812)
Fixes #15756
2023-06-21 11:34:32 +01:00
Mathieu Velten 496f73103d
Allow for the configuration of max request retries and min/max retry delays in the matrix federation client (#15783) 2023-06-21 10:41:11 +02:00
Erik Johnston 1fcefd8f3e Merge branch 'master' into develop 2023-06-20 18:56:18 +01:00
Mathieu Velten 7d3da399dd 1.86.0 2023-06-20 17:22:50 +02:00
Shay 6a5cf1a759
Fix Sytest environmental variable evaluation in CI (#15804) 2023-06-20 07:55:46 -07:00
ew-at-vier 2301a09d7a
Fix admin api documentation typo (#15805)
* Fix admin api documentation typo

Signed-off-by: Eric Wolf <eric.wolf@vier.ai>
2023-06-20 10:45:26 +00:00
Eric Eastwood 887fa4b66b
Switch from `matrix://` to `matrix-federation://` scheme for internal Synapse routing of outbound federation traffic (#15806)
`matrix://` is a registered specced scheme nowadays and doesn't make sense for
our internal to Synapse use case anymore. ([discussion]
(https://github.com/matrix-org/synapse/pull/15773#discussion_r1227598679))
2023-06-20 10:05:31 +01:00
dependabot[bot] 4ba528d9c3
Bump ijson from 3.2.0.post0 to 3.2.1 (#15802)
Bumps [ijson](https://github.com/ICRAR/ijson) from 3.2.0.post0 to 3.2.1.
- [Changelog](https://github.com/ICRAR/ijson/blob/master/CHANGELOG.md)
- [Commits](https://github.com/ICRAR/ijson/compare/v3.2.0.post0...v3.2.1)

---
updated-dependencies:
- dependency-name: ijson
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-06-19 10:30:17 +01:00
dependabot[bot] 5f9d5190aa
Bump attrs from 22.2.0 to 23.1.0 (#15801)
Bumps [attrs](https://github.com/python-attrs/attrs) from 22.2.0 to 23.1.0.
- [Release notes](https://github.com/python-attrs/attrs/releases)
- [Changelog](https://github.com/python-attrs/attrs/blob/main/CHANGELOG.md)
- [Commits](https://github.com/python-attrs/attrs/compare/22.2.0...23.1.0)

---
updated-dependencies:
- dependency-name: attrs
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-06-19 10:30:03 +01:00
dependabot[bot] 207cbe519d
Bump phonenumbers from 8.13.13 to 8.13.14 (#15798)
Bumps [phonenumbers](https://github.com/daviddrysdale/python-phonenumbers) from 8.13.13 to 8.13.14.
- [Commits](https://github.com/daviddrysdale/python-phonenumbers/compare/v8.13.13...v8.13.14)

---
updated-dependencies:
- dependency-name: phonenumbers
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-06-19 10:29:10 +01:00
dependabot[bot] d3cd9881c0
Bump ruff from 0.0.265 to 0.0.272 (#15799)
Bumps [ruff](https://github.com/charliermarsh/ruff) from 0.0.265 to 0.0.272.
- [Release notes](https://github.com/charliermarsh/ruff/releases)
- [Changelog](https://github.com/astral-sh/ruff/blob/main/BREAKING_CHANGES.md)
- [Commits](https://github.com/charliermarsh/ruff/compare/v0.0.265...v0.0.272)

---
updated-dependencies:
- dependency-name: ruff
  dependency-type: direct:development
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-06-19 10:28:57 +01:00
dependabot[bot] 10c509425f
Bump serde_json from 1.0.96 to 1.0.97 (#15797)
Bumps [serde_json](https://github.com/serde-rs/json) from 1.0.96 to 1.0.97.
- [Release notes](https://github.com/serde-rs/json/releases)
- [Commits](https://github.com/serde-rs/json/compare/v1.0.96...v1.0.97)

---
updated-dependencies:
- dependency-name: serde_json
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-06-19 10:28:43 +01:00
Eric Eastwood 0f02f0b4da
Remove experimental MSC2716 implementation to incrementally import history into existing rooms (#15748)
Context for why we're removing the implementation:

 - https://github.com/matrix-org/matrix-spec-proposals/pull/2716#issuecomment-1487441010
 - https://github.com/matrix-org/matrix-spec-proposals/pull/2716#issuecomment-1504262734

Anyone wanting to continue MSC2716, should also address these leftover tasks: https://github.com/matrix-org/synapse/issues/10737

Closes https://github.com/matrix-org/synapse/issues/10737 in the fact that it is not longer necessary to track those things.
2023-06-16 14:12:24 -05:00
Andrew Morgan 2ac6c3bbb5
Don't always lock "user_ips" table when performing non-native upsert (#15788) 2023-06-16 15:25:44 +01:00
Mathieu Velten 0618bf94cd
push rules: fix internal conversion from _type to value (#15781)
Also fix wrong rule names for `is_user_mention` and `is_room_mention`.
2023-06-16 14:17:02 +02:00
Mathieu Velten f63d4a3a65
Regularly try to wake up dests instead of waiting for next PDU/EDU (#15743) 2023-06-16 10:15:12 +00:00
Josh Qou d939120421
Fix unsafe hotserving behaviour for non-multimedia uploads. (#15680)
* Fix unsafe hotserving behaviour for non-multimedia uploads.

* invert disposition assert

* test_media_storage.py: run lint

* test_base.py: /inline/attachment/s

* Only return attachment for disposition type, update tests

* Update synapse/media/_base.py

Co-authored-by: Patrick Cloke <clokep@users.noreply.github.com>

* Update changelog.d/15680.bugfix

Co-authored-by: Patrick Cloke <clokep@users.noreply.github.com>

* add attribution

* Update changelog.

---------

Co-authored-by: Patrick Cloke <clokep@users.noreply.github.com>
2023-06-15 14:23:27 +01:00
Tulir Asokan 1404f68a03
Fix joining rooms through aliases where the alias server isn't a real homeserver (#15776) 2023-06-14 15:42:33 +01:00
Mathieu Velten 87e5df9a6e Merge branch 'release-v1.86' into develop 2023-06-14 14:54:19 +02:00
Jason Little 21fea6b749
Prefill events after invalidate not before when persisting events (#15758)
Fixes #15757
2023-06-14 09:42:18 +01:00
Eric Eastwood 8ddb2de553
Document `looping_call()` functionality that will wait for the given function to finish before scheduling another (#15772)
Thanks to @erikjohnston for clarifying, https://github.com/matrix-org/synapse/pull/15743#discussion_r1226544457

We don't have to worry about calls stacking up if the given function takes longer than the scheduled time.
2023-06-13 16:34:54 -05:00
Shay 553f2f53e7
Replace `EventContext` fields `prev_group` and `delta_ids` with field `state_group_deltas` (#15233) 2023-06-13 13:22:06 -07:00
Mathieu Velten 59ec4a0dc1
Fix MSC3983 support: only one OTK per device was returned through federation (#15770) 2023-06-13 19:51:47 +02:00
Eric Eastwood 0757d59ec4
Avoid backfill when we already have messages to return (#15737)
We now only block the client to backfill when we see a large gap in the events (more than 2 events missing in a row according to `depth`), more than 3 single-event holes, or not enough messages to fill the response. Otherwise, we return the messages directly to the client and backfill in the background for eventual consistency sake. 

Fix https://github.com/matrix-org/synapse/issues/15696
2023-06-13 12:31:08 -05:00
Patrick Cloke df945e0d7c
Fix MSC3983 support: Use the unstable /keys/claim federation endpoint if multiple keys are requested (#15755) 2023-06-13 18:07:55 +02:00
593 changed files with 33747 additions and 19916 deletions

View File

@ -29,11 +29,12 @@ IS_PR = os.environ["GITHUB_REF"].startswith("refs/pull/")
# First calculate the various trial jobs.
#
# For each type of test we only run on Py3.7 on PRs
# For PRs, we only run each type of test with the oldest Python version supported (which
# is Python 3.8 right now)
trial_sqlite_tests = [
{
"python-version": "3.7",
"python-version": "3.8",
"database": "sqlite",
"extras": "all",
}
@ -46,13 +47,12 @@ if not IS_PR:
"database": "sqlite",
"extras": "all",
}
for version in ("3.8", "3.9", "3.10", "3.11")
for version in ("3.9", "3.10", "3.11", "3.12")
)
trial_postgres_tests = [
{
"python-version": "3.7",
"python-version": "3.8",
"database": "postgres",
"postgres-version": "11",
"extras": "all",
@ -62,16 +62,16 @@ trial_postgres_tests = [
if not IS_PR:
trial_postgres_tests.append(
{
"python-version": "3.11",
"python-version": "3.12",
"database": "postgres",
"postgres-version": "15",
"postgres-version": "16",
"extras": "all",
}
)
trial_no_extra_tests = [
{
"python-version": "3.7",
"python-version": "3.8",
"database": "sqlite",
"extras": "",
}
@ -133,11 +133,6 @@ if not IS_PR:
"sytest-tag": "testing",
"postgres": "postgres",
},
{
"sytest-tag": "buster",
"postgres": "multi-postgres",
"workers": "workers",
},
]
)

View File

@ -8,21 +8,21 @@
# If ignoring a pull request that was not squash merged, only the merge
# commit needs to be put here. Child commits will be resolved from it.
# Run black (#3679).
# Run black (https://github.com/matrix-org/synapse/pull/3679).
8b3d9b6b199abb87246f982d5db356f1966db925
# Black reformatting (#5482).
# Black reformatting (https://github.com/matrix-org/synapse/pull/5482).
32e7c9e7f20b57dd081023ac42d6931a8da9b3a3
# Target Python 3.5 with black (#8664).
# Target Python 3.5 with black (https://github.com/matrix-org/synapse/pull/8664).
aff1eb7c671b0a3813407321d2702ec46c71fa56
# Update black to 20.8b1 (#9381).
# Update black to 20.8b1 (https://github.com/matrix-org/synapse/pull/9381).
0a00b7ff14890987f09112a2ae696c61001e6cf1
# Convert tests/rest/admin/test_room.py to unix file endings (#7953).
# Convert tests/rest/admin/test_room.py to unix file endings (https://github.com/matrix-org/synapse/pull/7953).
c4268e3da64f1abb5b31deaeb5769adb6510c0a7
# Update black to 23.1.0 (#15103)
# Update black to 23.1.0 (https://github.com/matrix-org/synapse/pull/15103)
9bb2eac71962970d02842bca441f4bcdbbf93a11

View File

@ -18,25 +18,35 @@ jobs:
steps:
- name: Set up QEMU
id: qemu
uses: docker/setup-qemu-action@v2
uses: docker/setup-qemu-action@v3
with:
platforms: arm64
- name: Set up Docker Buildx
id: buildx
uses: docker/setup-buildx-action@v2
uses: docker/setup-buildx-action@v3
- name: Inspect builder
run: docker buildx inspect
- name: Checkout repository
uses: actions/checkout@v4
- name: Extract version from pyproject.toml
# Note: explicitly requesting bash will mean bash is invoked with `-eo pipefail`, see
# https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsshell
shell: bash
run: |
echo "SYNAPSE_VERSION=$(grep "^version" pyproject.toml | sed -E 's/version\s*=\s*["]([^"]*)["]/\1/')" >> $GITHUB_ENV
- name: Log in to DockerHub
uses: docker/login-action@v2
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Log in to GHCR
uses: docker/login-action@v2
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
@ -58,10 +68,12 @@ jobs:
type=pep440,pattern={{raw}}
- name: Build and push all platforms
uses: docker/build-push-action@v4
uses: docker/build-push-action@v5
with:
push: true
labels: "gitsha1=${{ github.sha }}"
labels: |
gitsha1=${{ github.sha }}
org.opencontainers.image.version=${{ env.SYNAPSE_VERSION }}
tags: "${{ steps.set-tag.outputs.tags }}"
file: "docker/Dockerfile"
platforms: linux/amd64,linux/arm64

View File

@ -14,7 +14,7 @@ jobs:
# There's a 'download artifact' action, but it hasn't been updated for the workflow_run action
# (https://github.com/actions/download-artifact/issues/60) so instead we get this mess:
- name: 📥 Download artifact
uses: dawidd6/action-download-artifact@246dbf436b23d7c49e21a7ab8204ca9ecd1fe615 # v2.27.0
uses: dawidd6/action-download-artifact@268677152d06ba59fcec7a7f0b5d961b6ccd7e1e # v2.28.0
with:
workflow: docs-pr.yaml
run_id: ${{ github.event.workflow_run.id }}
@ -22,7 +22,7 @@ jobs:
path: book
- name: 📤 Deploy to Netlify
uses: matrix-org/netlify-pr-preview@v2
uses: matrix-org/netlify-pr-preview@v3
with:
path: book
owner: ${{ github.event.workflow_run.head_repository.owner.login }}

View File

@ -6,19 +6,30 @@ on:
- docs/**
- book.toml
- .github/workflows/docs-pr.yaml
- scripts-dev/schema_versions.py
jobs:
pages:
name: GitHub Pages
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
with:
# Fetch all history so that the schema_versions script works.
fetch-depth: 0
- name: Setup mdbook
uses: peaceiris/actions-mdbook@adeb05db28a0c0004681db83893d56c0388ea9ea # v1.2.0
with:
mdbook-version: '0.4.17'
- name: Setup python
uses: actions/setup-python@v4
with:
python-version: "3.x"
- run: "pip install 'packaging>=20.0' 'GitPython>=3.1.20'"
- name: Build the documentation
# mdbook will only create an index.html if we're including docs/README.md in SUMMARY.md.
# However, we're using docs/README.md for other purposes and need to pick a new page
@ -39,7 +50,7 @@ jobs:
name: Check links in documentation
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- name: Setup mdbook
uses: peaceiris/actions-mdbook@adeb05db28a0c0004681db83893d56c0388ea9ea # v1.2.0

View File

@ -50,13 +50,23 @@ jobs:
needs:
- pre
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
with:
# Fetch all history so that the schema_versions script works.
fetch-depth: 0
- name: Setup mdbook
uses: peaceiris/actions-mdbook@adeb05db28a0c0004681db83893d56c0388ea9ea # v1.2.0
with:
mdbook-version: '0.4.17'
- name: Setup python
uses: actions/setup-python@v4
with:
python-version: "3.x"
- run: "pip install 'packaging>=20.0' 'GitPython>=3.1.20'"
- name: Build the documentation
# mdbook will only create an index.html if we're including docs/README.md in SUMMARY.md.
# However, we're using docs/README.md for other purposes and need to pick a new page
@ -80,7 +90,7 @@ jobs:
needs:
- pre
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- name: "Set up Sphinx"
uses: matrix-org/setup-python-poetry@v1

52
.github/workflows/fix_lint.yaml vendored Normal file
View File

@ -0,0 +1,52 @@
# A helper workflow to automatically fixup any linting errors on a PR. Must be
# triggered manually.
name: Attempt to automatically fix linting errors
on:
workflow_dispatch:
jobs:
fixup:
name: Fix up
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Install Rust
uses: dtolnay/rust-toolchain@master
with:
# We use nightly so that `fmt` correctly groups together imports, and
# clippy correctly fixes up the benchmarks.
toolchain: nightly-2022-12-01
components: rustfmt
- uses: Swatinem/rust-cache@v2
- name: Setup Poetry
uses: matrix-org/setup-python-poetry@v1
with:
install-project: "false"
- name: Import order (isort)
continue-on-error: true
run: poetry run isort .
- name: Code style (black)
continue-on-error: true
run: poetry run black .
- name: Semantic checks (ruff)
continue-on-error: true
run: poetry run ruff --fix .
- run: cargo clippy --all-features --fix -- -D warnings
continue-on-error: true
- run: cargo fmt
continue-on-error: true
- uses: stefanzweifel/git-auto-commit-action@v5
with:
commit_message: "Attempt to fix linting"

View File

@ -39,7 +39,7 @@ jobs:
if: needs.check_repo.outputs.should_run_workflow == 'true'
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- name: Install Rust
uses: dtolnay/rust-toolchain@stable
- uses: Swatinem/rust-cache@v2
@ -57,8 +57,8 @@ jobs:
# `pip install matrix-synapse[all]` as closely as possible.
- run: poetry update --no-dev
- run: poetry run pip list > after.txt && (diff -u before.txt after.txt || true)
- name: Remove warn_unused_ignores from mypy config
run: sed '/warn_unused_ignores = True/d' -i mypy.ini
- name: Remove unhelpful options from mypy config
run: sed -e '/warn_unused_ignores = True/d' -e '/warn_redundant_casts = True/d' -i mypy.ini
- run: poetry run mypy
trial:
needs: check_repo
@ -72,7 +72,7 @@ jobs:
postgres-version: "14"
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- name: Install Rust
uses: dtolnay/rust-toolchain@stable
@ -145,7 +145,7 @@ jobs:
BLACKLIST: ${{ matrix.workers && 'synapse-blacklist-with-workers' }}
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- name: Install Rust
uses: dtolnay/rust-toolchain@stable
@ -192,16 +192,19 @@ jobs:
database: Postgres
steps:
- name: Run actions/checkout@v3 for synapse
uses: actions/checkout@v3
- name: Run actions/checkout@v4 for synapse
uses: actions/checkout@v4
with:
path: synapse
- uses: actions/setup-go@v4
- name: Prepare Complement's Prerequisites
run: synapse/.ci/scripts/setup_complement_prerequisites.sh
- uses: actions/setup-go@v4
with:
cache-dependency-path: complement/go.sum
go-version-file: complement/go.mod
- run: |
set -o pipefail
TEST_ONLY_IGNORE_POETRY_LOCKFILE=1 POSTGRES=${{ (matrix.database == 'Postgres') && 1 || '' }} WORKERS=${{ (matrix.arrangement == 'workers') && 1 || '' }} COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -json 2>&1 | synapse/.ci/scripts/gotestfmt
@ -222,7 +225,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- uses: JasonEtco/create-an-issue@e27dddc79c92bc6e4562f268fffa5ed752639abd # v2.9.1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

View File

@ -16,7 +16,7 @@ jobs:
name: "Check locked dependencies have sdists"
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- uses: actions/setup-python@v4
with:
python-version: '3.x'

View File

@ -33,29 +33,29 @@ jobs:
packages: write
steps:
- name: Checkout specific branch (debug build)
uses: actions/checkout@v3
uses: actions/checkout@v4
if: github.event_name == 'workflow_dispatch'
with:
ref: ${{ inputs.branch }}
- name: Checkout clean copy of develop (scheduled build)
uses: actions/checkout@v3
uses: actions/checkout@v4
if: github.event_name == 'schedule'
with:
ref: develop
- name: Checkout clean copy of master (on-push)
uses: actions/checkout@v3
uses: actions/checkout@v4
if: github.event_name == 'push'
with:
ref: master
- name: Login to registry
uses: docker/login-action@v2
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Work out labels for complement image
id: meta
uses: docker/metadata-action@v4
uses: docker/metadata-action@v5
with:
images: ghcr.io/${{ github.repository }}/complement-synapse
tags: |

View File

@ -27,7 +27,7 @@ jobs:
name: "Calculate list of debian distros"
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- uses: actions/setup-python@v4
with:
python-version: '3.x'
@ -55,13 +55,13 @@ jobs:
steps:
- name: Checkout
uses: actions/checkout@v3
uses: actions/checkout@v4
with:
path: src
- name: Set up Docker Buildx
id: buildx
uses: docker/setup-buildx-action@v2
uses: docker/setup-buildx-action@v3
with:
install: true
@ -121,7 +121,7 @@ jobs:
arch: aarch64
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- uses: actions/setup-python@v4
with:
@ -130,11 +130,11 @@ jobs:
python-version: "3.x"
- name: Install cibuildwheel
run: python -m pip install cibuildwheel==2.9.0
run: python -m pip install cibuildwheel==2.16.2
- name: Set up QEMU to emulate aarch64
if: matrix.arch == 'aarch64'
uses: docker/setup-qemu-action@v2
uses: docker/setup-qemu-action@v3
with:
platforms: arm64
@ -144,7 +144,7 @@ jobs:
- name: Only build a single wheel on PR
if: startsWith(github.ref, 'refs/pull/')
run: echo "CIBW_BUILD="cp37-manylinux_${{ matrix.arch }}"" >> $GITHUB_ENV
run: echo "CIBW_BUILD="cp38-manylinux_${{ matrix.arch }}"" >> $GITHUB_ENV
- name: Build wheels
run: python -m cibuildwheel --output-dir wheelhouse
@ -167,7 +167,7 @@ jobs:
if: ${{ !startsWith(github.ref, 'refs/pull/') }}
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- uses: actions/setup-python@v4
with:
python-version: '3.10'

View File

@ -12,12 +12,19 @@ concurrency:
cancel-in-progress: true
jobs:
check-signoff:
if: "github.event_name == 'pull_request'"
uses: "matrix-org/backend-meta/.github/workflows/sign-off.yml@v2"
# Job to detect what has changed so we don't run e.g. Rust checks on PRs that
# don't modify Rust code.
changes:
runs-on: ubuntu-latest
outputs:
rust: ${{ !startsWith(github.ref, 'refs/pull/') || steps.filter.outputs.rust }}
trial: ${{ !startsWith(github.ref, 'refs/pull/') || steps.filter.outputs.trial }}
integration: ${{ !startsWith(github.ref, 'refs/pull/') || steps.filter.outputs.integration }}
linting: ${{ !startsWith(github.ref, 'refs/pull/') || steps.filter.outputs.linting }}
steps:
- uses: dorny/paths-filter@v2
id: filter
@ -29,13 +36,56 @@ jobs:
- 'rust/**'
- 'Cargo.toml'
- 'Cargo.lock'
- '.rustfmt.toml'
- '.github/workflows/tests.yml'
trial:
- 'synapse/**'
- 'tests/**'
- 'rust/**'
- '.ci/scripts/calculate_jobs.py'
- 'Cargo.toml'
- 'Cargo.lock'
- 'pyproject.toml'
- 'poetry.lock'
- '.github/workflows/tests.yml'
integration:
- 'synapse/**'
- 'rust/**'
- 'docker/**'
- 'Cargo.toml'
- 'Cargo.lock'
- 'pyproject.toml'
- 'poetry.lock'
- 'docker/**'
- '.ci/**'
- 'scripts-dev/complement.sh'
- '.github/workflows/tests.yml'
linting:
- 'synapse/**'
- 'docker/**'
- 'tests/**'
- 'scripts-dev/**'
- 'contrib/**'
- 'synmark/**'
- 'stubs/**'
- '.ci/**'
- 'mypy.ini'
- 'pyproject.toml'
- 'poetry.lock'
- '.github/workflows/tests.yml'
check-sampleconfig:
runs-on: ubuntu-latest
needs: changes
if: ${{ needs.changes.outputs.linting == 'true' }}
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- name: Install Rust
uses: dtolnay/rust-toolchain@1.60.0
uses: dtolnay/rust-toolchain@1.61.0
- uses: Swatinem/rust-cache@v2
- uses: matrix-org/setup-python-poetry@v1
with:
@ -47,8 +97,11 @@ jobs:
check-schema-delta:
runs-on: ubuntu-latest
needs: changes
if: ${{ needs.changes.outputs.linting == 'true' }}
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- uses: actions/setup-python@v4
with:
python-version: "3.x"
@ -58,7 +111,7 @@ jobs:
check-lockfile:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- uses: actions/setup-python@v4
with:
python-version: "3.x"
@ -66,9 +119,12 @@ jobs:
lint:
runs-on: ubuntu-latest
needs: changes
if: ${{ needs.changes.outputs.linting == 'true' }}
steps:
- name: Checkout repository
uses: actions/checkout@v3
uses: actions/checkout@v4
- name: Setup Poetry
uses: matrix-org/setup-python-poetry@v1
@ -88,12 +144,15 @@ jobs:
lint-mypy:
runs-on: ubuntu-latest
name: Typechecking
needs: changes
if: ${{ needs.changes.outputs.linting == 'true' }}
steps:
- name: Checkout repository
uses: actions/checkout@v3
uses: actions/checkout@v4
- name: Install Rust
uses: dtolnay/rust-toolchain@1.60.0
uses: dtolnay/rust-toolchain@1.61.0
- uses: Swatinem/rust-cache@v2
- name: Setup Poetry
@ -123,7 +182,7 @@ jobs:
lint-crlf:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- name: Check line endings
run: scripts-dev/check_line_terminators.sh
@ -131,7 +190,7 @@ jobs:
if: ${{ (github.base_ref == 'develop' || contains(github.base_ref, 'release-')) && github.actor != 'dependabot[bot]' }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
with:
ref: ${{ github.event.pull_request.head.sha }}
fetch-depth: 0
@ -145,12 +204,15 @@ jobs:
lint-pydantic:
runs-on: ubuntu-latest
needs: changes
if: ${{ needs.changes.outputs.linting == 'true' }}
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
with:
ref: ${{ github.event.pull_request.head.sha }}
- name: Install Rust
uses: dtolnay/rust-toolchain@1.60.0
uses: dtolnay/rust-toolchain@1.61.0
- uses: Swatinem/rust-cache@v2
- uses: matrix-org/setup-python-poetry@v1
with:
@ -164,10 +226,10 @@ jobs:
if: ${{ needs.changes.outputs.rust == 'true' }}
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- name: Install Rust
uses: dtolnay/rust-toolchain@1.60.0
uses: dtolnay/rust-toolchain@1.61.0
with:
components: clippy
- uses: Swatinem/rust-cache@v2
@ -182,7 +244,7 @@ jobs:
if: ${{ needs.changes.outputs.rust == 'true' }}
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- name: Install Rust
uses: dtolnay/rust-toolchain@master
@ -199,7 +261,7 @@ jobs:
if: ${{ needs.changes.outputs.rust == 'true' }}
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- name: Install Rust
uses: dtolnay/rust-toolchain@master
@ -234,7 +296,7 @@ jobs:
needs: linting-done
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- uses: actions/setup-python@v4
with:
python-version: "3.x"
@ -245,15 +307,17 @@ jobs:
sytest_test_matrix: ${{ steps.get-matrix.outputs.sytest_test_matrix }}
trial:
if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail
needs: calculate-test-jobs
if: ${{ !cancelled() && !failure() && needs.changes.outputs.trial == 'true' }} # Allow previous steps to be skipped, but not fail
needs:
- calculate-test-jobs
- changes
runs-on: ubuntu-latest
strategy:
matrix:
job: ${{ fromJson(needs.calculate-test-jobs.outputs.trial_test_matrix) }}
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- run: sudo apt-get -qq install xmlsec1
- name: Set up PostgreSQL ${{ matrix.job.postgres-version }}
if: ${{ matrix.job.postgres-version }}
@ -268,7 +332,7 @@ jobs:
postgres:${{ matrix.job.postgres-version }}
- name: Install Rust
uses: dtolnay/rust-toolchain@1.60.0
uses: dtolnay/rust-toolchain@1.61.0
- uses: Swatinem/rust-cache@v2
- uses: matrix-org/setup-python-poetry@v1
@ -301,14 +365,16 @@ jobs:
trial-olddeps:
# Note: sqlite only; no postgres
if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail
needs: linting-done
if: ${{ !cancelled() && !failure() && needs.changes.outputs.trial == 'true' }} # Allow previous steps to be skipped, but not fail
needs:
- linting-done
- changes
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- name: Install Rust
uses: dtolnay/rust-toolchain@1.60.0
uses: dtolnay/rust-toolchain@1.61.0
- uses: Swatinem/rust-cache@v2
# There aren't wheels for some of the older deps, so we need to install
@ -320,7 +386,7 @@ jobs:
- uses: actions/setup-python@v4
with:
python-version: '3.7'
python-version: '3.8'
- name: Prepare old deps
if: steps.cache-poetry-old-deps.outputs.cache-hit != 'true'
@ -357,16 +423,18 @@ jobs:
trial-pypy:
# Very slow; only run if the branch name includes 'pypy'
# Note: sqlite only; no postgres. Completely untested since poetry move.
if: ${{ contains(github.ref, 'pypy') && !failure() && !cancelled() }}
needs: linting-done
if: ${{ contains(github.ref, 'pypy') && !failure() && !cancelled() && needs.changes.outputs.trial == 'true' }}
needs:
- linting-done
- changes
runs-on: ubuntu-latest
strategy:
matrix:
python-version: ["pypy-3.7"]
python-version: ["pypy-3.8"]
extras: ["all"]
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
# Install libs necessary for PyPy to build binary wheels for dependencies
- run: sudo apt-get -qq install xmlsec1 libxml2-dev libxslt-dev
- uses: matrix-org/setup-python-poetry@v1
@ -389,8 +457,10 @@ jobs:
|| true
sytest:
if: ${{ !failure() && !cancelled() }}
needs: calculate-test-jobs
if: ${{ !failure() && !cancelled() && needs.changes.outputs.integration == 'true' }}
needs:
- calculate-test-jobs
- changes
runs-on: ubuntu-latest
container:
image: matrixdotorg/sytest-synapse:${{ matrix.job.sytest-tag }}
@ -399,8 +469,8 @@ jobs:
env:
SYTEST_BRANCH: ${{ github.head_ref }}
POSTGRES: ${{ matrix.job.postgres && 1}}
MULTI_POSTGRES: ${{ (matrix.job.postgres == 'multi-postgres') && 1}}
ASYNCIO_REACTOR: ${{ (matrix.job.reactor == 'asyncio') && 1 }}
MULTI_POSTGRES: ${{ (matrix.job.postgres == 'multi-postgres') || '' }}
ASYNCIO_REACTOR: ${{ (matrix.job.reactor == 'asyncio') || '' }}
WORKERS: ${{ matrix.job.workers && 1 }}
BLACKLIST: ${{ matrix.job.workers && 'synapse-blacklist-with-workers' }}
TOP: ${{ github.workspace }}
@ -411,12 +481,12 @@ jobs:
job: ${{ fromJson(needs.calculate-test-jobs.outputs.sytest_test_matrix) }}
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- name: Prepare test blacklist
run: cat sytest-blacklist .ci/worker-blacklist > synapse-blacklist-with-workers
- name: Install Rust
uses: dtolnay/rust-toolchain@1.60.0
uses: dtolnay/rust-toolchain@1.61.0
- uses: Swatinem/rust-cache@v2
- name: Run SyTest
@ -435,8 +505,8 @@ jobs:
/logs/**/*.log*
export-data:
if: ${{ !failure() && !cancelled() }} # Allow previous steps to be skipped, but not fail
needs: [linting-done, portdb]
if: ${{ !failure() && !cancelled() && needs.changes.outputs.integration == 'true'}} # Allow previous steps to be skipped, but not fail
needs: [linting-done, portdb, changes]
runs-on: ubuntu-latest
env:
TOP: ${{ github.workspace }}
@ -456,7 +526,7 @@ jobs:
--health-retries 5
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- run: sudo apt-get -qq install xmlsec1 postgresql-client
- uses: matrix-org/setup-python-poetry@v1
with:
@ -471,13 +541,15 @@ jobs:
portdb:
if: ${{ !failure() && !cancelled() }} # Allow previous steps to be skipped, but not fail
needs: linting-done
if: ${{ !failure() && !cancelled() && needs.changes.outputs.integration == 'true'}} # Allow previous steps to be skipped, but not fail
needs:
- linting-done
- changes
runs-on: ubuntu-latest
strategy:
matrix:
include:
- python-version: "3.7"
- python-version: "3.8"
postgres-version: "11"
- python-version: "3.11"
@ -498,7 +570,7 @@ jobs:
--health-retries 5
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- name: Add PostgreSQL apt repository
# We need a version of pg_dump that can handle the version of
# PostgreSQL being tested against. The Ubuntu package repository lags
@ -532,8 +604,10 @@ jobs:
schema_diff
complement:
if: "${{ !failure() && !cancelled() }}"
needs: linting-done
if: "${{ !failure() && !cancelled() && needs.changes.outputs.integration == 'true' }}"
needs:
- linting-done
- changes
runs-on: ubuntu-latest
strategy:
@ -550,23 +624,27 @@ jobs:
database: Postgres
steps:
- name: Run actions/checkout@v3 for synapse
uses: actions/checkout@v3
- name: Run actions/checkout@v4 for synapse
uses: actions/checkout@v4
with:
path: synapse
- name: Install Rust
uses: dtolnay/rust-toolchain@1.60.0
uses: dtolnay/rust-toolchain@1.61.0
- uses: Swatinem/rust-cache@v2
- uses: actions/setup-go@v4
- name: Prepare Complement's Prerequisites
run: synapse/.ci/scripts/setup_complement_prerequisites.sh
- uses: actions/setup-go@v4
with:
cache-dependency-path: complement/go.sum
go-version-file: complement/go.mod
# use p=1 concurrency as GHA boxes are underpowered and don't like running tons of synapses at once.
- run: |
set -o pipefail
COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -json 2>&1 | synapse/.ci/scripts/gotestfmt
COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -p 1 -json 2>&1 | synapse/.ci/scripts/gotestfmt
shell: bash
env:
POSTGRES: ${{ (matrix.database == 'Postgres') && 1 || '' }}
@ -581,10 +659,10 @@ jobs:
- changes
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- name: Install Rust
uses: dtolnay/rust-toolchain@1.60.0
uses: dtolnay/rust-toolchain@1.61.0
- uses: Swatinem/rust-cache@v2
- run: cargo test
@ -599,7 +677,7 @@ jobs:
- changes
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- name: Install Rust
uses: dtolnay/rust-toolchain@master
@ -627,9 +705,16 @@ jobs:
with:
needs: ${{ toJSON(needs) }}
# The newsfile lint may be skipped on non PR builds
# Cargo test is skipped if there is no changes on Rust code
# Various bits are skipped if there was no applicable changes.
# The newsfile and signoff lint may be skipped on non PR builds.
skippable: |
trial
trial-olddeps
sytest
portdb
export-data
complement
check-signoff
lint-newsfile
cargo-test
cargo-bench

View File

@ -5,6 +5,9 @@ on:
- cron: 0 8 * * *
workflow_dispatch:
# NB: inputs are only present when this workflow is dispatched manually.
# (The default below is the default field value in the form to trigger
# a manual dispatch). Otherwise the inputs will evaluate to null.
inputs:
twisted_ref:
description: Commit, branch or tag to checkout from upstream Twisted.
@ -37,7 +40,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- name: Install Rust
uses: dtolnay/rust-toolchain@stable
@ -49,10 +52,10 @@ jobs:
extras: "all"
- run: |
poetry remove twisted
poetry add --extras tls git+https://github.com/twisted/twisted.git#${{ inputs.twisted_ref }}
poetry add --extras tls git+https://github.com/twisted/twisted.git#${{ inputs.twisted_ref || 'trunk' }}
poetry install --no-interaction --extras "all test"
- name: Remove warn_unused_ignores from mypy config
run: sed '/warn_unused_ignores = True/d' -i mypy.ini
- name: Remove unhelpful options from mypy config
run: sed -e '/warn_unused_ignores = True/d' -e '/warn_redundant_casts = True/d' -i mypy.ini
- run: poetry run mypy
trial:
@ -61,7 +64,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- run: sudo apt-get -qq install xmlsec1
- name: Install Rust
@ -96,12 +99,16 @@ jobs:
if: needs.check_repo.outputs.should_run_workflow == 'true'
runs-on: ubuntu-latest
container:
image: matrixdotorg/sytest-synapse:buster
# We're using ubuntu:focal because it uses Python 3.8 which is our minimum supported Python version.
# This job is a canary to warn us about unreleased twisted changes that would cause problems for us if
# they were to be released immediately. For simplicity's sake (and to save CI runners) we use the oldest
# version, assuming that any incompatibilities on newer versions would also be present on the oldest.
image: matrixdotorg/sytest-synapse:focal
volumes:
- ${{ github.workspace }}:/src
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- name: Install Rust
uses: dtolnay/rust-toolchain@stable
@ -156,16 +163,19 @@ jobs:
database: Postgres
steps:
- name: Run actions/checkout@v3 for synapse
uses: actions/checkout@v3
- name: Run actions/checkout@v4 for synapse
uses: actions/checkout@v4
with:
path: synapse
- uses: actions/setup-go@v4
- name: Prepare Complement's Prerequisites
run: synapse/.ci/scripts/setup_complement_prerequisites.sh
- uses: actions/setup-go@v4
with:
cache-dependency-path: complement/go.sum
go-version-file: complement/go.mod
# This step is specific to the 'Twisted trunk' test run:
- name: Patch dependencies
run: |
@ -196,7 +206,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- uses: JasonEtco/create-an-issue@e27dddc79c92bc6e4562f268fffa5ed752639abd # v2.9.1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

1
.gitignore vendored
View File

@ -34,6 +34,7 @@ __pycache__/
/logs
/media_store/
/uploads
/homeserver-config-overrides.d
# For direnv users
/.envrc

3773
CHANGES.md

File diff suppressed because it is too large Load Diff

120
Cargo.lock generated
View File

@ -13,9 +13,9 @@ dependencies = [
[[package]]
name = "anyhow"
version = "1.0.71"
version = "1.0.75"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8"
checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6"
[[package]]
name = "arc-swap"
@ -90,6 +90,12 @@ dependencies = [
"version_check",
]
[[package]]
name = "heck"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8"
[[package]]
name = "hex"
version = "0.4.3"
@ -98,9 +104,9 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
[[package]]
name = "indoc"
version = "1.0.7"
version = "2.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "adab1eaa3408fb7f0c777a73e7465fd5656136fc93b670eb6df3c88c2c1344e3"
checksum = "1e186cfbae8084e513daff4240b4797e342f988cecda4fb6c939150f96315fd8"
[[package]]
name = "itoa"
@ -132,21 +138,21 @@ dependencies = [
[[package]]
name = "log"
version = "0.4.19"
version = "0.4.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4"
checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f"
[[package]]
name = "memchr"
version = "2.5.0"
version = "2.6.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c"
[[package]]
name = "memoffset"
version = "0.6.5"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce"
checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c"
dependencies = [
"autocfg",
]
@ -182,18 +188,18 @@ dependencies = [
[[package]]
name = "proc-macro2"
version = "1.0.52"
version = "1.0.64"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d0e1ae9e836cc3beddd63db0df682593d7e2d3d891ae8c9083d2113e1744224"
checksum = "78803b62cbf1f46fde80d7c0e803111524b9877184cfe7c3033659490ac7a7da"
dependencies = [
"unicode-ident",
]
[[package]]
name = "pyo3"
version = "0.17.3"
version = "0.20.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "268be0c73583c183f2b14052337465768c07726936a260f480f0857cb95ba543"
checksum = "04e8453b658fe480c3e70c8ed4e3d3ec33eb74988bd186561b0cc66b85c3bc4b"
dependencies = [
"anyhow",
"cfg-if",
@ -209,9 +215,9 @@ dependencies = [
[[package]]
name = "pyo3-build-config"
version = "0.17.3"
version = "0.20.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "28fcd1e73f06ec85bf3280c48c67e731d8290ad3d730f8be9dc07946923005c8"
checksum = "a96fe70b176a89cff78f2fa7b3c930081e163d5379b4dcdf993e3ae29ca662e5"
dependencies = [
"once_cell",
"target-lexicon",
@ -219,9 +225,9 @@ dependencies = [
[[package]]
name = "pyo3-ffi"
version = "0.17.3"
version = "0.20.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0f6cb136e222e49115b3c51c32792886defbfb0adead26a688142b346a0b9ffc"
checksum = "214929900fd25e6604661ed9cf349727c8920d47deff196c4e28165a6ef2a96b"
dependencies = [
"libc",
"pyo3-build-config",
@ -229,9 +235,9 @@ dependencies = [
[[package]]
name = "pyo3-log"
version = "0.8.2"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c94ff6535a6bae58d7d0b85e60d4c53f7f84d0d0aa35d6a28c3f3e70bfe51444"
checksum = "4c10808ee7250403bedb24bc30c32493e93875fef7ba3e4292226fe924f398bd"
dependencies = [
"arc-swap",
"log",
@ -240,32 +246,33 @@ dependencies = [
[[package]]
name = "pyo3-macros"
version = "0.17.3"
version = "0.20.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94144a1266e236b1c932682136dc35a9dee8d3589728f68130c7c3861ef96b28"
checksum = "dac53072f717aa1bfa4db832b39de8c875b7c7af4f4a6fe93cdbf9264cf8383b"
dependencies = [
"proc-macro2",
"pyo3-macros-backend",
"quote",
"syn 1.0.104",
"syn",
]
[[package]]
name = "pyo3-macros-backend"
version = "0.17.3"
version = "0.20.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c8df9be978a2d2f0cdebabb03206ed73b11314701a5bfe71b0d753b81997777f"
checksum = "7774b5a8282bd4f25f803b1f0d945120be959a36c72e08e7cd031c792fdfd424"
dependencies = [
"heck",
"proc-macro2",
"quote",
"syn 1.0.104",
"syn",
]
[[package]]
name = "pythonize"
version = "0.17.0"
version = "0.20.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0f7f0c136f5fbc01868185eef462800e49659eb23acca83b9e884367a006acb6"
checksum = "ffd1c3ef39c725d63db5f9bc455461bafd80540cb7824c61afb823501921a850"
dependencies = [
"pyo3",
"serde",
@ -273,9 +280,9 @@ dependencies = [
[[package]]
name = "quote"
version = "1.0.26"
version = "1.0.29"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc"
checksum = "573015e8ab27661678357f27dc26460738fd2b6c86e46f386fde94cb5d913105"
dependencies = [
"proc-macro2",
]
@ -291,9 +298,21 @@ dependencies = [
[[package]]
name = "regex"
version = "1.8.4"
version = "1.9.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d0ab3ca65655bb1e41f2a8c8cd662eb4fb035e67c3f78da1d61dffe89d07300f"
checksum = "ebee201405406dbf528b8b672104ae6d6d63e6d118cb10e4d51abbc7b58044ff"
dependencies = [
"aho-corasick",
"memchr",
"regex-automata",
"regex-syntax",
]
[[package]]
name = "regex-automata"
version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "59b23e92ee4318893fa3fe3e6fb365258efbfe6ac6ab30f090cdcbb7aa37efa9"
dependencies = [
"aho-corasick",
"memchr",
@ -302,9 +321,9 @@ dependencies = [
[[package]]
name = "regex-syntax"
version = "0.7.2"
version = "0.7.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "436b050e76ed2903236f032a59761c1eb99e1b0aead2c257922771dab1fc8c78"
checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da"
[[package]]
name = "ryu"
@ -320,29 +339,29 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
[[package]]
name = "serde"
version = "1.0.164"
version = "1.0.193"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9e8c8cf938e98f769bc164923b06dce91cea1751522f46f8466461af04c9027d"
checksum = "25dd9975e68d0cb5aa1120c288333fc98731bd1dd12f561e468ea4728c042b89"
dependencies = [
"serde_derive",
]
[[package]]
name = "serde_derive"
version = "1.0.164"
version = "1.0.193"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d9735b638ccc51c28bf6914d90a2e9725b377144fc612c49a611fddd1b631d68"
checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.10",
"syn",
]
[[package]]
name = "serde_json"
version = "1.0.96"
version = "1.0.108"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "057d394a50403bcac12672b2b18fb387ab6d289d957dab67dd201875391e52f1"
checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b"
dependencies = [
"itoa",
"ryu",
@ -363,20 +382,9 @@ checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601"
[[package]]
name = "syn"
version = "1.0.104"
version = "2.0.28"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4ae548ec36cf198c0ef7710d3c230987c2d6d7bd98ad6edc0274462724c585ce"
dependencies = [
"proc-macro2",
"quote",
"unicode-ident",
]
[[package]]
name = "syn"
version = "2.0.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5aad1363ed6d37b84299588d62d3a7d95b5a5c2d9aad5c85609fda12afaa1f40"
checksum = "04361975b3f5e348b2189d8dc55bc942f278b2d482a6a0365de5bdd62d351567"
dependencies = [
"proc-macro2",
"quote",
@ -420,9 +428,9 @@ checksum = "6ceab39d59e4c9499d4e5a8ee0e2735b891bb7308ac83dfb4e80cad195c9f6f3"
[[package]]
name = "unindent"
version = "0.1.10"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "58ee9362deb4a96cef4d437d1ad49cffc9b9e92d202b6995674e928ce684f112"
checksum = "c7de7d73e1754487cb58364ee906a499937a0dfabd86bcb980fa99ec8c8fa2ce"
[[package]]
name = "version_check"

View File

@ -3,3 +3,4 @@
[workspace]
members = ["rust"]
resolver = "2"

View File

@ -2,10 +2,19 @@
Synapse |support| |development| |documentation| |license| |pypi| |python|
=========================================================================
Synapse is an open-source `Matrix <https://matrix.org/>`_ homeserver written and
maintained by the Matrix.org Foundation. We began rapid development in 2014,
reaching v1.0.0 in 2019. Development on Synapse and the Matrix protocol itself continues
in earnest today.
Synapse is now actively maintained at `element-hq/synapse <https://github.com/element-hq/synapse>`_
=================================================================================================
Synapse is an open-source `Matrix <https://matrix.org/>`_ homeserver developed
from 2019 through 2023 as part of the Matrix.org Foundation. The Matrix.org
Foundation is not able to resource maintenance of Synapse and it
`continues to be developed by Element <https://github.com/element-hq/synapse>`_;
additionally you have the choice of `other Matrix homeservers <https://matrix.org/ecosystem/servers/>`_.
See `The future of Synapse and Dendrite <https://matrix.org/blog/2023/11/06/future-of-synapse-dendrite/>`_
blog post for more information.
=========================================================================
Briefly, Matrix is an open standard for communications on the internet, supporting
federation, encryption and VoIP. Matrix.org has more to say about the `goals of the
@ -122,7 +131,7 @@ You will need to change the server you are logging into from ``matrix.org``
and instead specify a Homeserver URL of ``https://<server_name>:8448``
(or just ``https://<server_name>`` if you are using a reverse proxy).
If you prefer to use another client, refer to our
`client breakdown <https://matrix.org/docs/projects/clients-matrix>`_.
`client breakdown <https://matrix.org/ecosystem/clients/>`_.
If all goes well you should at least be able to log in, create a room, and
start sending messages.

View File

@ -34,6 +34,14 @@ additional-css = [
"docs/website_files/table-of-contents.css",
"docs/website_files/remove-nav-buttons.css",
"docs/website_files/indent-section-headers.css",
"docs/website_files/version-picker.css",
]
additional-js = ["docs/website_files/table-of-contents.js"]
theme = "docs/website_files/theme"
additional-js = [
"docs/website_files/table-of-contents.js",
"docs/website_files/version-picker.js",
"docs/website_files/version.js",
]
theme = "docs/website_files/theme"
[preprocessor.schema_versions]
command = "./scripts-dev/schema_versions.py"

View File

@ -769,7 +769,7 @@ def main(server_url, identity_server_url, username, token, config_path):
global CONFIG_JSON
CONFIG_JSON = config_path # bit cheeky, but just overwrite the global
try:
with open(config_path, "r") as config:
with open(config_path) as config:
syn_cmd.config = json.load(config)
try:
http_client.verbose = "on" == syn_cmd.config["verbose"]

View File

@ -37,7 +37,6 @@ class HttpClient:
Deferred: Succeeds when we get a 2xx HTTP response. The result
will be the decoded JSON body.
"""
pass
def get_json(self, url, args=None):
"""Gets some json from the given host homeserver and path
@ -53,7 +52,6 @@ class HttpClient:
Deferred: Succeeds when we get a 2xx HTTP response. The result
will be the decoded JSON body.
"""
pass
class TwistedHttpClient(HttpClient):

File diff suppressed because it is too large Load Diff

View File

@ -29,7 +29,7 @@
"level": "error"
},
{
"line": "my-matrix-server-federation-sender-1 | 2023-01-25 20:56:20,995 - synapse.http.matrixfederationclient - 709 - WARNING - federation_transaction_transmission_loop-3 - {PUT-O-3} [example.com] Request failed: PUT matrix://example.com/_matrix/federation/v1/send/1674680155797: HttpResponseException('403: Forbidden')",
"line": "my-matrix-server-federation-sender-1 | 2023-01-25 20:56:20,995 - synapse.http.matrixfederationclient - 709 - WARNING - federation_transaction_transmission_loop-3 - {PUT-O-3} [example.com] Request failed: PUT matrix-federation://example.com/_matrix/federation/v1/send/1674680155797: HttpResponseException('403: Forbidden')",
"level": "warning"
},
{

194
debian/changelog vendored
View File

@ -1,3 +1,195 @@
matrix-synapse-py3 (1.98.0) stable; urgency=medium
* New Synapse release 1.98.0.
-- Synapse Packaging team <packages@matrix.org> Tue, 12 Dec 2023 15:04:31 +0000
matrix-synapse-py3 (1.98.0~rc1) stable; urgency=medium
* New Synapse release 1.98.0rc1.
-- Synapse Packaging team <packages@matrix.org> Tue, 05 Dec 2023 13:08:42 +0000
matrix-synapse-py3 (1.97.0) stable; urgency=medium
* New Synapse release 1.97.0.
-- Synapse Packaging team <packages@matrix.org> Tue, 28 Nov 2023 14:08:58 +0000
matrix-synapse-py3 (1.97.0~rc1) stable; urgency=medium
* New Synapse release 1.97.0rc1.
-- Synapse Packaging team <packages@matrix.org> Tue, 21 Nov 2023 12:32:03 +0000
matrix-synapse-py3 (1.96.1) stable; urgency=medium
* New synapse release 1.96.1.
-- Synapse Packaging team <packages@matrix.org> Fri, 17 Nov 2023 12:48:45 +0000
matrix-synapse-py3 (1.96.0) stable; urgency=medium
* New synapse release 1.96.0.
-- Synapse Packaging team <packages@matrix.org> Thu, 16 Nov 2023 17:54:26 +0000
matrix-synapse-py3 (1.96.0~rc1) stable; urgency=medium
* New Synapse release 1.96.0rc1.
-- Synapse Packaging team <packages@matrix.org> Tue, 31 Oct 2023 14:09:09 +0000
matrix-synapse-py3 (1.95.1) stable; urgency=medium
* New Synapse release 1.95.1.
-- Synapse Packaging team <packages@matrix.org> Tue, 31 Oct 2023 14:00:00 +0000
matrix-synapse-py3 (1.95.0) stable; urgency=medium
* New Synapse release 1.95.0.
-- Synapse Packaging team <packages@matrix.org> Tue, 24 Oct 2023 13:00:46 +0100
matrix-synapse-py3 (1.95.0~rc1) stable; urgency=medium
* New synapse release 1.95.0rc1.
-- Synapse Packaging team <packages@matrix.org> Tue, 17 Oct 2023 15:50:17 +0000
matrix-synapse-py3 (1.94.0) stable; urgency=medium
* New Synapse release 1.94.0.
-- Synapse Packaging team <packages@matrix.org> Tue, 10 Oct 2023 10:57:41 +0100
matrix-synapse-py3 (1.94.0~rc1) stable; urgency=medium
* New Synapse release 1.94.0rc1.
-- Synapse Packaging team <packages@matrix.org> Tue, 03 Oct 2023 11:48:18 +0100
matrix-synapse-py3 (1.93.0) stable; urgency=medium
* New Synapse release 1.93.0.
-- Synapse Packaging team <packages@matrix.org> Tue, 26 Sep 2023 15:54:40 +0100
matrix-synapse-py3 (1.93.0~rc1) stable; urgency=medium
* New synapse release 1.93.0rc1.
-- Synapse Packaging team <packages@matrix.org> Tue, 19 Sep 2023 11:55:00 +0000
matrix-synapse-py3 (1.92.3) stable; urgency=medium
* New Synapse release 1.92.3.
-- Synapse Packaging team <packages@matrix.org> Mon, 18 Sep 2023 15:05:04 +0200
matrix-synapse-py3 (1.92.2) stable; urgency=medium
* New Synapse release 1.92.2.
-- Synapse Packaging team <packages@matrix.org> Fri, 15 Sep 2023 13:17:41 +0100
matrix-synapse-py3 (1.92.1) stable; urgency=medium
* New Synapse release 1.92.1.
-- Synapse Packaging team <packages@matrix.org> Tue, 12 Sep 2023 13:19:42 +0200
matrix-synapse-py3 (1.92.0) stable; urgency=medium
* New Synapse release 1.92.0.
-- Synapse Packaging team <packages@matrix.org> Tue, 12 Sep 2023 11:59:23 +0200
matrix-synapse-py3 (1.91.2) stable; urgency=medium
* New synapse release 1.91.2.
-- Synapse Packaging team <packages@matrix.org> Wed, 06 Sep 2023 14:59:30 +0000
matrix-synapse-py3 (1.92.0~rc1) stable; urgency=medium
* New Synapse release 1.92.0rc1.
-- Synapse Packaging team <packages@matrix.org> Tue, 05 Sep 2023 11:21:43 +0100
matrix-synapse-py3 (1.91.1) stable; urgency=medium
* New Synapse release 1.91.1.
-- Synapse Packaging team <packages@matrix.org> Mon, 04 Sep 2023 14:03:18 +0100
matrix-synapse-py3 (1.91.0) stable; urgency=medium
* New Synapse release 1.91.0.
-- Synapse Packaging team <packages@matrix.org> Wed, 30 Aug 2023 11:18:10 +0100
matrix-synapse-py3 (1.91.0~rc1) stable; urgency=medium
* New Synapse release 1.91.0rc1.
-- Synapse Packaging team <packages@matrix.org> Wed, 23 Aug 2023 09:47:18 -0700
matrix-synapse-py3 (1.90.0) stable; urgency=medium
* New Synapse release 1.90.0.
-- Synapse Packaging team <packages@matrix.org> Tue, 15 Aug 2023 11:17:34 +0100
matrix-synapse-py3 (1.90.0~rc1) stable; urgency=medium
* New Synapse release 1.90.0rc1.
-- Synapse Packaging team <packages@matrix.org> Tue, 08 Aug 2023 15:29:34 +0100
matrix-synapse-py3 (1.89.0) stable; urgency=medium
* New Synapse release 1.89.0.
-- Synapse Packaging team <packages@matrix.org> Tue, 01 Aug 2023 11:07:15 +0100
matrix-synapse-py3 (1.89.0~rc1) stable; urgency=medium
* New Synapse release 1.89.0rc1.
-- Synapse Packaging team <packages@matrix.org> Tue, 25 Jul 2023 14:31:07 +0200
matrix-synapse-py3 (1.88.0) stable; urgency=medium
* New Synapse release 1.88.0.
-- Synapse Packaging team <packages@matrix.org> Tue, 18 Jul 2023 13:59:28 +0100
matrix-synapse-py3 (1.88.0~rc1) stable; urgency=medium
* New Synapse release 1.88.0rc1.
-- Synapse Packaging team <packages@matrix.org> Tue, 11 Jul 2023 10:20:19 +0100
matrix-synapse-py3 (1.87.0) stable; urgency=medium
* New Synapse release 1.87.0.
-- Synapse Packaging team <packages@matrix.org> Tue, 04 Jul 2023 16:24:00 +0100
matrix-synapse-py3 (1.87.0~rc1) stable; urgency=medium
* New synapse release 1.87.0rc1.
-- Synapse Packaging team <packages@matrix.org> Tue, 27 Jun 2023 15:27:04 +0000
matrix-synapse-py3 (1.86.0) stable; urgency=medium
* New Synapse release 1.86.0.
-- Synapse Packaging team <packages@matrix.org> Tue, 20 Jun 2023 17:22:46 +0200
matrix-synapse-py3 (1.86.0~rc2) stable; urgency=medium
* New Synapse release 1.86.0rc2.
@ -1481,7 +1673,7 @@ matrix-synapse-py3 (0.99.3.1) stable; urgency=medium
matrix-synapse-py3 (0.99.3) stable; urgency=medium
[ Richard van der Hoff ]
* Fix warning during preconfiguration. (Fixes: #4819)
* Fix warning during preconfiguration. (Fixes: https://github.com/matrix-org/synapse/issues/4819)
[ Synapse Packaging team ]
* New synapse release 0.99.3.

View File

@ -25,9 +25,9 @@ ARG PYTHON_VERSION=3.11
###
### Stage 0: generate requirements.txt
###
# We hardcode the use of Debian bullseye here because this could change upstream
# and other Dockerfiles used for testing are expecting bullseye.
FROM docker.io/library/python:${PYTHON_VERSION}-slim-bullseye as requirements
# We hardcode the use of Debian bookworm here because this could change upstream
# and other Dockerfiles used for testing are expecting bookworm.
FROM docker.io/library/python:${PYTHON_VERSION}-slim-bookworm as requirements
# RUN --mount is specific to buildkit and is documented at
# https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/syntax.md#build-mounts-run---mount.
@ -87,7 +87,7 @@ RUN if [ -z "$TEST_ONLY_IGNORE_POETRY_LOCKFILE" ]; then \
###
### Stage 1: builder
###
FROM docker.io/library/python:${PYTHON_VERSION}-slim-bullseye as builder
FROM docker.io/library/python:${PYTHON_VERSION}-slim-bookworm as builder
# install the OS build deps
RUN \
@ -158,7 +158,7 @@ RUN --mount=type=cache,target=/synapse/target,sharing=locked \
### Stage 2: runtime
###
FROM docker.io/library/python:${PYTHON_VERSION}-slim-bullseye
FROM docker.io/library/python:${PYTHON_VERSION}-slim-bookworm
LABEL org.opencontainers.image.url='https://matrix.org/docs/projects/server/synapse'
LABEL org.opencontainers.image.documentation='https://github.com/matrix-org/synapse/blob/master/docker/README.md'
@ -173,10 +173,10 @@ RUN \
gosu \
libjpeg62-turbo \
libpq5 \
libwebp6 \
libwebp7 \
xmlsec1 \
libjemalloc2 \
libicu67 \
libicu72 \
libssl-dev \
openssl \
&& rm -rf /var/lib/apt/lists/*

View File

@ -28,12 +28,12 @@ FROM docker.io/library/${distro} as builder
RUN apt-get update -qq -o Acquire::Languages=none
RUN env DEBIAN_FRONTEND=noninteractive apt-get install \
-yqq --no-install-recommends \
build-essential \
ca-certificates \
devscripts \
equivs \
wget
-yqq --no-install-recommends \
build-essential \
ca-certificates \
devscripts \
equivs \
wget
# fetch and unpack the package
# We are temporarily using a fork of dh-virtualenv due to an incompatibility with Python 3.11, which ships with
@ -62,33 +62,29 @@ FROM docker.io/library/${distro}
ARG distro=""
ENV distro ${distro}
# Python < 3.7 assumes LANG="C" means ASCII-only and throws on printing unicode
# http://bugs.python.org/issue19846
ENV LANG C.UTF-8
# Install the build dependencies
#
# NB: keep this list in sync with the list of build-deps in debian/control
# TODO: it would be nice to do that automatically.
RUN apt-get update -qq -o Acquire::Languages=none \
&& env DEBIAN_FRONTEND=noninteractive apt-get install \
-yqq --no-install-recommends -o Dpkg::Options::=--force-unsafe-io \
build-essential \
curl \
debhelper \
devscripts \
libsystemd-dev \
lsb-release \
pkg-config \
python3-dev \
python3-pip \
python3-setuptools \
python3-venv \
sqlite3 \
libpq-dev \
libicu-dev \
pkg-config \
xmlsec1
-yqq --no-install-recommends -o Dpkg::Options::=--force-unsafe-io \
build-essential \
curl \
debhelper \
devscripts \
libsystemd-dev \
lsb-release \
pkg-config \
python3-dev \
python3-pip \
python3-setuptools \
python3-venv \
sqlite3 \
libpq-dev \
libicu-dev \
pkg-config \
xmlsec1
# Install rust and ensure it's in the PATH
ENV RUSTUP_HOME=/rust

View File

@ -7,7 +7,7 @@ ARG FROM=matrixdotorg/synapse:$SYNAPSE_VERSION
# target image. For repeated rebuilds, this is much faster than apt installing
# each time.
FROM docker.io/library/debian:bullseye-slim AS deps_base
FROM docker.io/library/debian:bookworm-slim AS deps_base
RUN \
--mount=type=cache,target=/var/cache/apt,sharing=locked \
--mount=type=cache,target=/var/lib/apt,sharing=locked \
@ -21,7 +21,7 @@ FROM docker.io/library/debian:bullseye-slim AS deps_base
# which makes it much easier to copy (but we need to make sure we use an image
# based on the same debian version as the synapse image, to make sure we get
# the expected version of libc.
FROM docker.io/library/redis:7-bullseye AS redis_base
FROM docker.io/library/redis:7-bookworm AS redis_base
# now build the final image, based on the the regular Synapse docker image
FROM $FROM

View File

@ -20,8 +20,8 @@ FROM $FROM
# the same debian version as Synapse's docker image (so the versions of the
# shared libraries match).
RUN adduser --system --uid 999 postgres --home /var/lib/postgresql
COPY --from=docker.io/library/postgres:13-bullseye /usr/lib/postgresql /usr/lib/postgresql
COPY --from=docker.io/library/postgres:13-bullseye /usr/share/postgresql /usr/share/postgresql
COPY --from=docker.io/library/postgres:13-bookworm /usr/lib/postgresql /usr/lib/postgresql
COPY --from=docker.io/library/postgres:13-bookworm /usr/share/postgresql /usr/share/postgresql
RUN mkdir /var/run/postgresql && chown postgres /var/run/postgresql
ENV PATH="${PATH}:/usr/lib/postgresql/13/bin"
ENV PGDATA=/var/lib/postgresql/data

View File

@ -68,6 +68,11 @@ if [[ -n "$SYNAPSE_COMPLEMENT_USE_WORKERS" ]]; then
fi
log "Workers requested: $SYNAPSE_WORKER_TYPES"
# adjust connection pool limits on worker mode as otherwise running lots of worker synapses
# can make docker unhappy (in GHA)
export POSTGRES_CP_MIN=1
export POSTGRES_CP_MAX=3
echo "using reduced connection pool limits for worker mode"
# Improve startup times by using a launcher based on fork()
export SYNAPSE_USE_EXPERIMENTAL_FORKING_LAUNCHER=1
else

View File

@ -92,8 +92,6 @@ allow_device_name_lookup_over_federation: true
## Experimental Features ##
experimental_features:
# Enable history backfilling support
msc2716_enabled: true
# client-side support for partial state in /send_join responses
faster_joins: true
# Enable support for polls

View File

@ -35,7 +35,11 @@ server {
# Send all other traffic to the main process
location ~* ^(\\/_matrix|\\/_synapse) {
{% if using_unix_sockets %}
proxy_pass http://unix:/run/main_public.sock;
{% else %}
proxy_pass http://localhost:8080;
{% endif %}
proxy_set_header X-Forwarded-For $remote_addr;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header Host $host;

View File

@ -6,6 +6,9 @@
{% if enable_redis %}
redis:
enabled: true
{% if using_unix_sockets %}
path: /tmp/redis.sock
{% endif %}
{% endif %}
{% if appservice_registrations is not none %}

View File

@ -19,7 +19,11 @@ username=www-data
autorestart=true
[program:redis]
{% if using_unix_sockets %}
command=/usr/local/bin/prefix-log /usr/local/bin/redis-server --unixsocket /tmp/redis.sock
{% else %}
command=/usr/local/bin/prefix-log /usr/local/bin/redis-server
{% endif %}
priority=1
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0

View File

@ -8,7 +8,11 @@ worker_name: "{{ name }}"
worker_listeners:
- type: http
{% if using_unix_sockets %}
path: "/run/worker.{{ port }}"
{% else %}
port: {{ port }}
{% endif %}
{% if listener_resources %}
resources:
- names:

View File

@ -36,12 +36,17 @@ listeners:
# Allow configuring in case we want to reverse proxy 8008
# using another process in the same container
{% if SYNAPSE_USE_UNIX_SOCKET %}
# Unix sockets don't care about TLS or IP addresses or ports
- path: '/run/main_public.sock'
type: http
{% else %}
- port: {{ SYNAPSE_HTTP_PORT or 8008 }}
tls: false
bind_addresses: ['::']
type: http
x_forwarded: false
{% endif %}
resources:
- names: [client]
compress: true
@ -57,10 +62,13 @@ database:
user: "{{ POSTGRES_USER or "synapse" }}"
password: "{{ POSTGRES_PASSWORD }}"
database: "{{ POSTGRES_DB or "synapse" }}"
{% if not SYNAPSE_USE_UNIX_SOCKET %}
{# Synapse will use a default unix socket for Postgres when host/port is not specified (behavior from `psycopg2`). #}
host: "{{ POSTGRES_HOST or "db" }}"
port: "{{ POSTGRES_PORT or "5432" }}"
cp_min: 5
cp_max: 10
{% endif %}
cp_min: {{ POSTGRES_CP_MIN or 5 }}
cp_max: {{ POSTGRES_CP_MAX or 10 }}
{% else %}
database:
name: "sqlite3"

View File

@ -74,6 +74,9 @@ MAIN_PROCESS_HTTP_LISTENER_PORT = 8080
MAIN_PROCESS_INSTANCE_NAME = "main"
MAIN_PROCESS_LOCALHOST_ADDRESS = "127.0.0.1"
MAIN_PROCESS_REPLICATION_PORT = 9093
# Obviously, these would only be used with the UNIX socket option
MAIN_PROCESS_UNIX_SOCKET_PUBLIC_PATH = "/run/main_public.sock"
MAIN_PROCESS_UNIX_SOCKET_PRIVATE_PATH = "/run/main_private.sock"
# A simple name used as a placeholder in the WORKERS_CONFIG below. This will be replaced
# during processing with the name of the worker.
@ -180,6 +183,7 @@ WORKERS_CONFIG: Dict[str, Dict[str, Any]] = {
"^/_matrix/client/(r0|v3|unstable)/password_policy$",
"^/_matrix/client/(api/v1|r0|v3|unstable)/directory/room/.*$",
"^/_matrix/client/(r0|v3|unstable)/capabilities$",
"^/_matrix/client/(r0|v3|unstable)/notifications$",
],
"shared_extra_conf": {},
"worker_extra_conf": "",
@ -244,7 +248,6 @@ WORKERS_CONFIG: Dict[str, Dict[str, Any]] = {
"^/_matrix/client/(api/v1|r0|v3|unstable)/join/",
"^/_matrix/client/(api/v1|r0|v3|unstable)/knock/",
"^/_matrix/client/(api/v1|r0|v3|unstable)/profile/",
"^/_matrix/client/(v1|unstable/org.matrix.msc2716)/rooms/.*/batch_send",
],
"shared_extra_conf": {},
"worker_extra_conf": "",
@ -408,11 +411,15 @@ def add_worker_roles_to_shared_config(
)
# Map of stream writer instance names to host/ports combos
instance_map[worker_name] = {
"host": "localhost",
"port": worker_port,
}
if os.environ.get("SYNAPSE_USE_UNIX_SOCKET", False):
instance_map[worker_name] = {
"path": f"/run/worker.{worker_port}",
}
else:
instance_map[worker_name] = {
"host": "localhost",
"port": worker_port,
}
# Update the list of stream writers. It's convenient that the name of the worker
# type is the same as the stream to write. Iterate over the whole list in case there
# is more than one.
@ -424,10 +431,15 @@ def add_worker_roles_to_shared_config(
# Map of stream writer instance names to host/ports combos
# For now, all stream writers need http replication ports
instance_map[worker_name] = {
"host": "localhost",
"port": worker_port,
}
if os.environ.get("SYNAPSE_USE_UNIX_SOCKET", False):
instance_map[worker_name] = {
"path": f"/run/worker.{worker_port}",
}
else:
instance_map[worker_name] = {
"host": "localhost",
"port": worker_port,
}
def merge_worker_template_configs(
@ -719,17 +731,29 @@ def generate_worker_files(
# Note that yaml cares about indentation, so care should be taken to insert lines
# into files at the correct indentation below.
# Convenience helper for if using unix sockets instead of host:port
using_unix_sockets = environ.get("SYNAPSE_USE_UNIX_SOCKET", False)
# First read the original config file and extract the listeners block. Then we'll
# add another listener for replication. Later we'll write out the result to the
# shared config file.
listeners = [
{
"port": MAIN_PROCESS_REPLICATION_PORT,
"bind_address": MAIN_PROCESS_LOCALHOST_ADDRESS,
"type": "http",
"resources": [{"names": ["replication"]}],
}
]
listeners: List[Any]
if using_unix_sockets:
listeners = [
{
"path": MAIN_PROCESS_UNIX_SOCKET_PRIVATE_PATH,
"type": "http",
"resources": [{"names": ["replication"]}],
}
]
else:
listeners = [
{
"port": MAIN_PROCESS_REPLICATION_PORT,
"bind_address": MAIN_PROCESS_LOCALHOST_ADDRESS,
"type": "http",
"resources": [{"names": ["replication"]}],
}
]
with open(config_path) as file_stream:
original_config = yaml.safe_load(file_stream)
original_listeners = original_config.get("listeners")
@ -770,7 +794,17 @@ def generate_worker_files(
# A list of internal endpoints to healthcheck, starting with the main process
# which exists even if no workers do.
healthcheck_urls = ["http://localhost:8080/health"]
# This list ends up being part of the command line to curl, (curl added support for
# Unix sockets in version 7.40).
if using_unix_sockets:
healthcheck_urls = [
f"--unix-socket {MAIN_PROCESS_UNIX_SOCKET_PUBLIC_PATH} "
# The scheme and hostname from the following URL are ignored.
# The only thing that matters is the path `/health`
"http://localhost/health"
]
else:
healthcheck_urls = ["http://localhost:8080/health"]
# Get the set of all worker types that we have configured
all_worker_types_in_use = set(chain(*requested_worker_types.values()))
@ -807,8 +841,12 @@ def generate_worker_files(
# given worker_type needs to stay assigned and not be replaced.
worker_config["shared_extra_conf"].update(shared_config)
shared_config = worker_config["shared_extra_conf"]
healthcheck_urls.append("http://localhost:%d/health" % (worker_port,))
if using_unix_sockets:
healthcheck_urls.append(
f"--unix-socket /run/worker.{worker_port} http://localhost/health"
)
else:
healthcheck_urls.append("http://localhost:%d/health" % (worker_port,))
# Update the shared config with sharding-related options if necessary
add_worker_roles_to_shared_config(
@ -824,9 +862,10 @@ def generate_worker_files(
# Then a worker config file
convert(
"/conf/worker.yaml.j2",
"/conf/workers/{name}.yaml".format(name=worker_name),
f"/conf/workers/{worker_name}.yaml",
**worker_config,
worker_log_config_filepath=log_config_filepath,
using_unix_sockets=using_unix_sockets,
)
# Save this worker's port number to the correct nginx upstreams
@ -847,8 +886,13 @@ def generate_worker_files(
nginx_upstream_config = ""
for upstream_worker_base_name, upstream_worker_ports in nginx_upstreams.items():
body = ""
for port in upstream_worker_ports:
body += f" server localhost:{port};\n"
if using_unix_sockets:
for port in upstream_worker_ports:
body += f" server unix:/run/worker.{port};\n"
else:
for port in upstream_worker_ports:
body += f" server localhost:{port};\n"
# Add to the list of configured upstreams
nginx_upstream_config += NGINX_UPSTREAM_CONFIG_BLOCK.format(
@ -878,10 +922,15 @@ def generate_worker_files(
# If there are workers, add the main process to the instance_map too.
if workers_in_use:
instance_map = shared_config.setdefault("instance_map", {})
instance_map[MAIN_PROCESS_INSTANCE_NAME] = {
"host": MAIN_PROCESS_LOCALHOST_ADDRESS,
"port": MAIN_PROCESS_REPLICATION_PORT,
}
if using_unix_sockets:
instance_map[MAIN_PROCESS_INSTANCE_NAME] = {
"path": MAIN_PROCESS_UNIX_SOCKET_PRIVATE_PATH,
}
else:
instance_map[MAIN_PROCESS_INSTANCE_NAME] = {
"host": MAIN_PROCESS_LOCALHOST_ADDRESS,
"port": MAIN_PROCESS_REPLICATION_PORT,
}
# Shared homeserver config
convert(
@ -891,6 +940,7 @@ def generate_worker_files(
appservice_registrations=appservice_registrations,
enable_redis=workers_in_use,
workers_in_use=workers_in_use,
using_unix_sockets=using_unix_sockets,
)
# Nginx config
@ -901,6 +951,7 @@ def generate_worker_files(
upstream_directives=nginx_upstream_config,
tls_cert_path=os.environ.get("SYNAPSE_TLS_CERT"),
tls_key_path=os.environ.get("SYNAPSE_TLS_KEY"),
using_unix_sockets=using_unix_sockets,
)
# Supervisord config
@ -910,6 +961,7 @@ def generate_worker_files(
"/etc/supervisor/supervisord.conf",
main_config_path=config_path,
enable_redis=workers_in_use,
using_unix_sockets=using_unix_sockets,
)
convert(

View File

@ -8,9 +8,9 @@ ARG PYTHON_VERSION=3.9
###
### Stage 0: generate requirements.txt
###
# We hardcode the use of Debian bullseye here because this could change upstream
# and other Dockerfiles used for testing are expecting bullseye.
FROM docker.io/library/python:${PYTHON_VERSION}-slim-bullseye
# We hardcode the use of Debian bookworm here because this could change upstream
# and other Dockerfiles used for testing are expecting bookworm.
FROM docker.io/library/python:${PYTHON_VERSION}-slim-bookworm
# Install Rust and other dependencies (stolen from normal Dockerfile)
# install the OS build deps
@ -33,7 +33,7 @@ RUN \
gosu \
libjpeg62-turbo \
libpq5 \
libwebp6 \
libwebp7 \
xmlsec1 \
libjemalloc2 \
&& rm -rf /var/lib/apt/lists/*

View File

@ -82,7 +82,7 @@ def generate_config_from_template(
with open(filename) as handle:
value = handle.read()
else:
log("Generating a random secret for {}".format(secret))
log(f"Generating a random secret for {secret}")
value = codecs.encode(os.urandom(32), "hex").decode()
with open(filename, "w") as handle:
handle.write(value)
@ -239,7 +239,7 @@ def main(args: List[str], environ: MutableMapping[str, str]) -> None:
log("Could not find %s, will not use" % (jemallocpath,))
# if there are no config files passed to synapse, try adding the default file
if not any(p.startswith("--config-path") or p.startswith("-c") for p in args):
if not any(p.startswith(("--config-path", "-c")) for p in args):
config_dir = environ.get("SYNAPSE_CONFIG_DIR", "/data")
config_path = environ.get(
"SYNAPSE_CONFIG_PATH", config_dir + "/homeserver.yaml"

View File

@ -19,7 +19,7 @@
# Usage
- [Federation](federate.md)
- [Configuration](usage/configuration/README.md)
- [Configuration Manual](usage/configuration/config_documentation.md)
- [Configuration Manual](usage/configuration/config_documentation.md)
- [Homeserver Sample Config File](usage/configuration/homeserver_sample_config.md)
- [Logging Sample Config File](usage/configuration/logging_sample_config.md)
- [Structured Logging](structured_logging.md)
@ -48,6 +48,7 @@
- [Password auth provider callbacks](modules/password_auth_provider_callbacks.md)
- [Background update controller callbacks](modules/background_update_controller_callbacks.md)
- [Account data callbacks](modules/account_data_callbacks.md)
- [Add extra fields to client events unsigned section callbacks](modules/add_extra_fields_to_client_events_unsigned.md)
- [Porting a legacy module to the new interface](modules/porting_legacy_module.md)
- [Workers](workers.md)
- [Using `synctl` with Workers](synctl_workers.md)
@ -97,6 +98,7 @@
- [Cancellation](development/synapse_architecture/cancellation.md)
- [Log Contexts](log_contexts.md)
- [Replication](replication.md)
- [Streams](development/synapse_architecture/streams.md)
- [TCP Replication](tcp_replication.md)
- [Faster remote joins](development/synapse_architecture/faster_joins.md)
- [Internal Documentation](development/internal_documentation/README.md)

View File

@ -1,5 +1,7 @@
# Account validity API
**Note:** This API is disabled when MSC3861 is enabled. [See #15582](https://github.com/matrix-org/synapse/pull/15582)
This API allows a server administrator to manage the validity of an account. To
use it, you must enable the account validity feature (under
`account_validity`) in Synapse's configuration.

View File

@ -1,5 +1,7 @@
# Shared-Secret Registration
**Note:** This API is disabled when MSC3861 is enabled. [See #15582](https://github.com/matrix-org/synapse/pull/15582)
This API allows for the creation of users in an administrative and
non-interactive way. This is generally used for bootstrapping a Synapse
instance with administrator accounts.

View File

@ -419,7 +419,7 @@ The following query parameters are available:
* `from` (required) - The token to start returning events from. This token can be obtained from a prev_batch
or next_batch token returned by the /sync endpoint, or from an end token returned by a previous request to this endpoint.
* `to` - The token to spot returning events at.
* `to` - The token to stop returning events at.
* `limit` - The maximum number of events to return. Defaults to `10`.
* `filter` - A JSON RoomEventFilter to filter returned events with.
* `dir` - The direction to return events from. Either `f` for forwards or `b` for backwards. Setting
@ -536,7 +536,8 @@ The following query parameters are available:
**Response**
* `event_id` - converted from timestamp
* `event_id` - The event ID closest to the given timestamp.
* `origin_server_ts` - The timestamp of the event in milliseconds since the Unix epoch.
# Block Room API
The Block Room admin API allows server admins to block and unblock rooms,

View File

@ -54,7 +54,8 @@ It returns a JSON body like the following:
"external_id": "<user_id_provider_2>"
}
],
"user_type": null
"user_type": null,
"locked": false
}
```
@ -103,7 +104,8 @@ with a body of:
],
"admin": false,
"deactivated": false,
"user_type": null
"user_type": null,
"locked": false
}
```
@ -155,6 +157,7 @@ Body parameters:
Note: a user cannot be erased with this API. For more details on
deactivating and erasing users see [Deactivate Account](#deactivate-account).
- `locked` - **bool**, optional. If unspecified, locked state will be left unchanged.
- `user_type` - **string** or null, optional. If not provided, the user type will be
not be changed. If `null` is given, the user type will be cleared.
Other allowed options are: `bot` and `support`.
@ -183,7 +186,8 @@ A response body like the following is returned:
"shadow_banned": 0,
"displayname": "<User One>",
"avatar_url": null,
"creation_ts": 1560432668000
"creation_ts": 1560432668000,
"locked": false
}, {
"name": "<user_id2>",
"is_guest": 0,
@ -194,7 +198,8 @@ A response body like the following is returned:
"shadow_banned": 0,
"displayname": "<User Two>",
"avatar_url": "<avatar_url>",
"creation_ts": 1561550621000
"creation_ts": 1561550621000,
"locked": false
}
],
"next_token": "100",
@ -217,7 +222,9 @@ The following parameters should be set in the URL:
- `name` - Is optional and filters to only return users with user ID localparts
**or** displaynames that contain this value.
- `guests` - string representing a bool - Is optional and if `false` will **exclude** guest users.
Defaults to `true` to include guest users.
Defaults to `true` to include guest users. This parameter is not supported when MSC3861 is enabled. [See #15582](https://github.com/matrix-org/synapse/pull/15582)
- `admins` - Optional flag to filter admins. If `true`, only admins are queried. If `false`, admins are excluded from
the query. When the flag is absent (the default), **both** admins and non-admins are included in the search results.
- `deactivated` - string representing a bool - Is optional and if `true` will **include** deactivated users.
Defaults to `false` to exclude deactivated users.
- `limit` - string representing a positive integer - Is optional but is used for pagination,
@ -239,9 +246,15 @@ The following parameters should be set in the URL:
- `displayname` - Users are ordered alphabetically by `displayname`.
- `avatar_url` - Users are ordered alphabetically by avatar URL.
- `creation_ts` - Users are ordered by when the users was created in ms.
- `last_seen_ts` - Users are ordered by when the user was lastly seen in ms.
- `dir` - Direction of media order. Either `f` for forwards or `b` for backwards.
Setting this value to `b` will reverse the above sort order. Defaults to `f`.
- `not_user_type` - Exclude certain user types, such as bot users, from the request.
Can be provided multiple times. Possible values are `bot`, `support` or "empty string".
"empty string" here means to exclude users without a type.
- `locked` - string representing a bool - Is optional and if `true` will **include** locked users.
Defaults to `false` to exclude locked users. Note: Introduced in v1.93.
Caution. The database only has indexes on the columns `name` and `creation_ts`.
This means that if a different sort order is used (`is_guest`, `admin`,
@ -266,10 +279,12 @@ The following fields are returned in the JSON response body:
- `displayname` - string - The user's display name if they have set one.
- `avatar_url` - string - The user's avatar URL if they have set one.
- `creation_ts` - integer - The user's creation timestamp in ms.
- `last_seen_ts` - integer - The user's last activity timestamp in ms.
- `locked` - bool - Status if that user has been marked as locked. Note: Introduced in v1.93.
- `next_token`: string representing a positive integer - Indication for pagination. See above.
- `total` - integer - Total number of media.
*Added in Synapse 1.93:* the `locked` query parameter and response field.
## Query current sessions for a user
@ -384,6 +399,8 @@ The following actions are **NOT** performed. The list may be incomplete.
## Reset password
**Note:** This API is disabled when MSC3861 is enabled. [See #15582](https://github.com/matrix-org/synapse/pull/15582)
Changes the password of another user. This will automatically log the user out of all their devices.
The api is:
@ -407,6 +424,8 @@ The parameter `logout_devices` is optional and defaults to `true`.
## Get whether a user is a server administrator or not
**Note:** This API is disabled when MSC3861 is enabled. [See #15582](https://github.com/matrix-org/synapse/pull/15582)
The api is:
```
@ -424,6 +443,8 @@ A response body like the following is returned:
## Change whether a user is a server administrator or not
**Note:** This API is disabled when MSC3861 is enabled. [See #15582](https://github.com/matrix-org/synapse/pull/15582)
Note that you cannot demote yourself.
The api is:
@ -597,6 +618,16 @@ A response body like the following is returned:
"quarantined_by": null,
"safe_from_quarantine": false,
"upload_name": "test2.png"
},
{
"created_ts": 300400,
"last_access_ts": 300700,
"media_id": "BzYNLRUgGHphBkdKGbzXwbjX",
"media_length": 1337,
"media_type": "application/octet-stream",
"quarantined_by": null,
"safe_from_quarantine": false,
"upload_name": null
}
],
"next_token": 3,
@ -658,16 +689,17 @@ The following fields are returned in the JSON response body:
- `media` - An array of objects, each containing information about a media.
Media objects contain the following fields:
- `created_ts` - integer - Timestamp when the content was uploaded in ms.
- `last_access_ts` - integer - Timestamp when the content was last accessed in ms.
- `last_access_ts` - integer or null - Timestamp when the content was last accessed in ms.
Null if there was no access, yet.
- `media_id` - string - The id used to refer to the media. Details about the format
are documented under
[media repository](../media_repository.md).
- `media_length` - integer - Length of the media in bytes.
- `media_type` - string - The MIME-type of the media.
- `quarantined_by` - string - The user ID that initiated the quarantine request
for this media.
- `quarantined_by` - string or null - The user ID that initiated the quarantine request
for this media. Null if not quarantined.
- `safe_from_quarantine` - bool - Status if this media is safe from quarantining.
- `upload_name` - string - The name the media was uploaded with.
- `upload_name` - string or null - The name the media was uploaded with. Null if not provided during upload.
- `next_token`: integer - Indication for pagination. See above.
- `total` - integer - Total number of media.
@ -717,6 +749,8 @@ delete largest/smallest or newest/oldest files first.
## Login as a user
**Note:** This API is disabled when MSC3861 is enabled. [See #15582](https://github.com/matrix-org/synapse/pull/15582)
Get an access token that can be used to authenticate as that user. Useful for
when admins wish to do actions on behalf of a user.
@ -729,7 +763,8 @@ POST /_synapse/admin/v1/users/<user_id>/login
An optional `valid_until_ms` field can be specified in the request body as an
integer timestamp that specifies when the token should expire. By default tokens
do not expire.
do not expire. Note that this API does not allow a user to login as themselves
(to create more tokens).
A response body like the following is returned:
@ -749,6 +784,43 @@ Note: The token will expire if the *admin* user calls `/logout/all` from any
of their devices, but the token will *not* expire if the target user does the
same.
## Allow replacing master cross-signing key without User-Interactive Auth
This endpoint is not intended for server administrator usage;
we describe it here for completeness.
This API temporarily permits a user to replace their master cross-signing key
without going through
[user-interactive authentication](https://spec.matrix.org/v1.8/client-server-api/#user-interactive-authentication-api) (UIA).
This is useful when Synapse has delegated its authentication to the
[Matrix Authentication Service](https://github.com/matrix-org/matrix-authentication-service/);
as Synapse cannot perform UIA is not possible in these circumstances.
The API is
```http request
POST /_synapse/admin/v1/users/<user_id>/_allow_cross_signing_replacement_without_uia
{}
```
If the user does not exist, or does exist but has no master cross-signing key,
this will return with status code `404 Not Found`.
Otherwise, a response body like the following is returned, with status `200 OK`:
```json
{
"updatable_without_uia_before_ms": 1234567890
}
```
The response body is a JSON object with a single field:
- `updatable_without_uia_before_ms`: integer. The timestamp in milliseconds
before which the user is permitted to replace their cross-signing key without
going through UIA.
_Added in Synapse 1.97.0._
## User devices
@ -1180,7 +1252,7 @@ The following parameters should be set in the URL:
- `user_id` - The fully qualified MXID: for example, `@user:server.com`. The user must
be local.
### Check username availability
## Check username availability
Checks to see if a username is available, and valid, for the server. See [the client-server
API](https://matrix.org/docs/spec/client_server/r0.6.0#get-matrix-client-r0-register-available)
@ -1198,7 +1270,7 @@ GET /_synapse/admin/v1/username_available?username=$localpart
The request and response format is the same as the
[/_matrix/client/r0/register/available](https://matrix.org/docs/spec/client_server/r0.6.0#get-matrix-client-r0-register-available) API.
### Find a user based on their ID in an auth provider
## Find a user based on their ID in an auth provider
The API is:
@ -1237,7 +1309,7 @@ Returns a `404` HTTP status code if no user was found, with a response body like
_Added in Synapse 1.68.0._
### Find a user based on their Third Party ID (ThreePID or 3PID)
## Find a user based on their Third Party ID (ThreePID or 3PID)
The API is:

View File

@ -1,7 +1,7 @@
# Version API
This API returns the running Synapse version and the Python version
on which Synapse is being run. This is useful when a Synapse instance
This API returns the running Synapse version.
This is useful when a Synapse instance
is behind a proxy that does not forward the 'Server' header (which also
contains Synapse version information).
@ -15,7 +15,9 @@ It returns a JSON body like the following:
```json
{
"server_version": "0.99.2rc1 (b=develop, abcdef123)",
"python_version": "3.7.8"
"server_version": "0.99.2rc1 (b=develop, abcdef123)"
}
```
*Changed in Synapse 1.94.0:* The `python_version` key was removed from the
response body.

View File

@ -24,7 +24,7 @@ Server with a domain specific API.
1. **Messaging Layer**
This is what the rest of the homeserver hits to send messages, join rooms,
etc. It also allows you to register callbacks for when it get's notified by
etc. It also allows you to register callbacks for when it gets notified by
lower levels that e.g. a new message has been received.
It is responsible for serializing requests to send to the data

View File

@ -164,7 +164,7 @@ Synapse 1.6.0rc2 (2019-11-25)
Bugfixes
--------
- Fix a bug which could cause the background database update hander for event labels to get stuck in a loop raising exceptions. ([\#6407](https://github.com/matrix-org/synapse/issues/6407))
- Fix a bug which could cause the background database update handler for event labels to get stuck in a loop raising exceptions. ([\#6407](https://github.com/matrix-org/synapse/issues/6407))
Synapse 1.6.0rc1 (2019-11-20)
@ -191,7 +191,7 @@ Bugfixes
- Appservice requests will no longer contain a double slash prefix when the appservice url provided ends in a slash. ([\#6306](https://github.com/matrix-org/synapse/issues/6306))
- Fix `/purge_room` admin API. ([\#6307](https://github.com/matrix-org/synapse/issues/6307))
- Fix the `hidden` field in the `devices` table for SQLite versions prior to 3.23.0. ([\#6313](https://github.com/matrix-org/synapse/issues/6313))
- Fix bug which casued rejected events to be persisted with the wrong room state. ([\#6320](https://github.com/matrix-org/synapse/issues/6320))
- Fix bug which caused rejected events to be persisted with the wrong room state. ([\#6320](https://github.com/matrix-org/synapse/issues/6320))
- Fix bug where `rc_login` ratelimiting would prematurely kick in. ([\#6335](https://github.com/matrix-org/synapse/issues/6335))
- Prevent the server taking a long time to start up when guest registration is enabled. ([\#6338](https://github.com/matrix-org/synapse/issues/6338))
- Fix bug where upgrading a guest account to a full user would fail when account validity is enabled. ([\#6359](https://github.com/matrix-org/synapse/issues/6359))
@ -232,7 +232,7 @@ Internal Changes
- Add some documentation about worker replication. ([\#6305](https://github.com/matrix-org/synapse/issues/6305))
- Move admin endpoints into separate files. Contributed by Awesome Technologies Innovationslabor GmbH. ([\#6308](https://github.com/matrix-org/synapse/issues/6308))
- Document the use of `lint.sh` for code style enforcement & extend it to run on specified paths only. ([\#6312](https://github.com/matrix-org/synapse/issues/6312))
- Add optional python dependencies and dependant binary libraries to snapcraft packaging. ([\#6317](https://github.com/matrix-org/synapse/issues/6317))
- Add optional python dependencies and dependent binary libraries to snapcraft packaging. ([\#6317](https://github.com/matrix-org/synapse/issues/6317))
- Remove the dependency on psutil and replace functionality with the stdlib `resource` module. ([\#6318](https://github.com/matrix-org/synapse/issues/6318), [\#6336](https://github.com/matrix-org/synapse/issues/6336))
- Improve documentation for EventContext fields. ([\#6319](https://github.com/matrix-org/synapse/issues/6319))
- Add some checks that we aren't using state from rejected events. ([\#6330](https://github.com/matrix-org/synapse/issues/6330))
@ -653,7 +653,7 @@ Internal Changes
- Return 502 not 500 when failing to reach any remote server. ([\#5810](https://github.com/matrix-org/synapse/issues/5810))
- Reduce global pauses in the events stream caused by expensive state resolution during persistence. ([\#5826](https://github.com/matrix-org/synapse/issues/5826))
- Add a lower bound to well-known lookup cache time to avoid repeated lookups. ([\#5836](https://github.com/matrix-org/synapse/issues/5836))
- Whitelist history visbility sytests in worker mode tests. ([\#5843](https://github.com/matrix-org/synapse/issues/5843))
- Whitelist history visibility sytests in worker mode tests. ([\#5843](https://github.com/matrix-org/synapse/issues/5843))
Synapse 1.2.1 (2019-07-26)
@ -817,7 +817,7 @@ See the [upgrade notes](docs/upgrade.md#upgrading-to-v110) for more details.
Features
--------
- Added possibilty to disable local password authentication. Contributed by Daniel Hoffend. ([\#5092](https://github.com/matrix-org/synapse/issues/5092))
- Added possibility to disable local password authentication. Contributed by Daniel Hoffend. ([\#5092](https://github.com/matrix-org/synapse/issues/5092))
- Add monthly active users to phonehome stats. ([\#5252](https://github.com/matrix-org/synapse/issues/5252))
- Allow expired user to trigger renewal email sending manually. ([\#5363](https://github.com/matrix-org/synapse/issues/5363))
- Statistics on forward extremities per room are now exposed via Prometheus. ([\#5384](https://github.com/matrix-org/synapse/issues/5384), [\#5458](https://github.com/matrix-org/synapse/issues/5458), [\#5461](https://github.com/matrix-org/synapse/issues/5461))
@ -850,7 +850,7 @@ Bugfixes
- Fix bug where clients could tight loop calling `/sync` for a period. ([\#5507](https://github.com/matrix-org/synapse/issues/5507))
- Fix bug with `jinja2` preventing Synapse from starting. Users who had this problem should now simply need to run `pip install matrix-synapse`. ([\#5514](https://github.com/matrix-org/synapse/issues/5514))
- Fix a regression where homeservers on private IP addresses were incorrectly blacklisted. ([\#5523](https://github.com/matrix-org/synapse/issues/5523))
- Fixed m.login.jwt using unregistred user_id and added pyjwt>=1.6.4 as jwt conditional dependencies. Contributed by Pau Rodriguez-Estivill. ([\#5555](https://github.com/matrix-org/synapse/issues/5555), [\#5586](https://github.com/matrix-org/synapse/issues/5586))
- Fixed m.login.jwt using unregistered user_id and added pyjwt>=1.6.4 as jwt conditional dependencies. Contributed by Pau Rodriguez-Estivill. ([\#5555](https://github.com/matrix-org/synapse/issues/5555), [\#5586](https://github.com/matrix-org/synapse/issues/5586))
- Fix a bug that would cause invited users to receive several emails for a single 3PID invite in case the inviter is rate limited. ([\#5576](https://github.com/matrix-org/synapse/issues/5576))

View File

@ -251,7 +251,7 @@ Internal Changes
- Optimise `/createRoom` with multiple invited users. ([\#8559](https://github.com/matrix-org/synapse/issues/8559))
- Implement and use an `@lru_cache` decorator. ([\#8595](https://github.com/matrix-org/synapse/issues/8595))
- Don't instansiate Requester directly. ([\#8614](https://github.com/matrix-org/synapse/issues/8614))
- Don't instantiate Requester directly. ([\#8614](https://github.com/matrix-org/synapse/issues/8614))
- Type hints for `RegistrationStore`. ([\#8615](https://github.com/matrix-org/synapse/issues/8615))
- Change schema to support access tokens belonging to one user but granting access to another. ([\#8616](https://github.com/matrix-org/synapse/issues/8616))
- Remove unused OPTIONS handlers. ([\#8621](https://github.com/matrix-org/synapse/issues/8621))
@ -518,7 +518,7 @@ Bugfixes
- Fix a bug which cause the logging system to report errors, if `DEBUG` was enabled and no `context` filter was applied. ([\#8278](https://github.com/matrix-org/synapse/issues/8278))
- Fix edge case where push could get delayed for a user until a later event was pushed. ([\#8287](https://github.com/matrix-org/synapse/issues/8287))
- Fix fetching malformed events from remote servers. ([\#8324](https://github.com/matrix-org/synapse/issues/8324))
- Fix `UnboundLocalError` from occuring when appservices send a malformed register request. ([\#8329](https://github.com/matrix-org/synapse/issues/8329))
- Fix `UnboundLocalError` from occurring when appservices send a malformed register request. ([\#8329](https://github.com/matrix-org/synapse/issues/8329))
- Don't send push notifications to expired user accounts. ([\#8353](https://github.com/matrix-org/synapse/issues/8353))
- Fix a regression in v1.19.0 with reactivating users through the admin API. ([\#8362](https://github.com/matrix-org/synapse/issues/8362))
- Fix a bug where during device registration the length of the device name wasn't limited. ([\#8364](https://github.com/matrix-org/synapse/issues/8364))
@ -815,7 +815,7 @@ Bugfixes
- Fix a bug introduced in Synapse v1.7.2 which caused inaccurate membership counts in the room directory. ([\#7977](https://github.com/matrix-org/synapse/issues/7977))
- Fix a long standing bug: 'Duplicate key value violates unique constraint "event_relations_id"' when message retention is configured. ([\#7978](https://github.com/matrix-org/synapse/issues/7978))
- Fix "no create event in auth events" when trying to reject invitation after inviter leaves. Bug introduced in Synapse v1.10.0. ([\#7980](https://github.com/matrix-org/synapse/issues/7980))
- Fix various comments and minor discrepencies in server notices code. ([\#7996](https://github.com/matrix-org/synapse/issues/7996))
- Fix various comments and minor discrepancies in server notices code. ([\#7996](https://github.com/matrix-org/synapse/issues/7996))
- Fix a long standing bug where HTTP HEAD requests resulted in a 400 error. ([\#7999](https://github.com/matrix-org/synapse/issues/7999))
- Fix a long-standing bug which caused two copies of some log lines to be written when synctl was used along with a MemoryHandler logger. ([\#8011](https://github.com/matrix-org/synapse/issues/8011), [\#8012](https://github.com/matrix-org/synapse/issues/8012))
@ -1460,7 +1460,7 @@ Bugfixes
- Transfer alias mappings on room upgrade. ([\#6946](https://github.com/matrix-org/synapse/issues/6946))
- Ensure that a user interactive authentication session is tied to a single request. ([\#7068](https://github.com/matrix-org/synapse/issues/7068), [\#7455](https://github.com/matrix-org/synapse/issues/7455))
- Fix a bug in the federation API which could cause occasional "Failed to get PDU" errors. ([\#7089](https://github.com/matrix-org/synapse/issues/7089))
- Return the proper error (`M_BAD_ALIAS`) when a non-existant canonical alias is provided. ([\#7109](https://github.com/matrix-org/synapse/issues/7109))
- Return the proper error (`M_BAD_ALIAS`) when a non-existent canonical alias is provided. ([\#7109](https://github.com/matrix-org/synapse/issues/7109))
- Fix a bug which meant that groups updates were not correctly replicated between workers. ([\#7117](https://github.com/matrix-org/synapse/issues/7117))
- Fix starting workers when federation sending not split out. ([\#7133](https://github.com/matrix-org/synapse/issues/7133))
- Ensure `is_verified` is a boolean in responses to `GET /_matrix/client/r0/room_keys/keys`. Also warn the user if they forgot the `version` query param. ([\#7150](https://github.com/matrix-org/synapse/issues/7150))
@ -1482,7 +1482,7 @@ Bugfixes
- Fix bad error handling that would cause Synapse to crash if it's provided with a YAML configuration file that's either empty or doesn't parse into a key-value map. ([\#7341](https://github.com/matrix-org/synapse/issues/7341))
- Fix incorrect metrics reporting for `renew_attestations` background task. ([\#7344](https://github.com/matrix-org/synapse/issues/7344))
- Prevent non-federating rooms from appearing in responses to federated `POST /publicRoom` requests when a filter was included. ([\#7367](https://github.com/matrix-org/synapse/issues/7367))
- Fix a bug which would cause the room durectory to be incorrectly populated if Synapse was upgraded directly from v1.2.1 or earlier to v1.4.0 or later. Note that this fix does not apply retrospectively; see the [upgrade notes](docs/upgrade.md#upgrading-to-v1130) for more information. ([\#7387](https://github.com/matrix-org/synapse/issues/7387))
- Fix a bug which would cause the room directory to be incorrectly populated if Synapse was upgraded directly from v1.2.1 or earlier to v1.4.0 or later. Note that this fix does not apply retrospectively; see the [upgrade notes](docs/upgrade.md#upgrading-to-v1130) for more information. ([\#7387](https://github.com/matrix-org/synapse/issues/7387))
- Fix bug in `EventContext.deserialize`. ([\#7393](https://github.com/matrix-org/synapse/issues/7393))
@ -1638,7 +1638,7 @@ Security advisory
-----------------
Synapse may be vulnerable to request-smuggling attacks when it is used with a
reverse-proxy. The vulnerabilties are fixed in Twisted 20.3.0, and are
reverse-proxy. The vulnerabilities are fixed in Twisted 20.3.0, and are
described in
[CVE-2020-10108](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-10108)
and
@ -1748,7 +1748,7 @@ Internal Changes
- Refactoring work in preparation for changing the event redaction algorithm. ([\#6874](https://github.com/matrix-org/synapse/issues/6874), [\#6875](https://github.com/matrix-org/synapse/issues/6875), [\#6983](https://github.com/matrix-org/synapse/issues/6983), [\#7003](https://github.com/matrix-org/synapse/issues/7003))
- Improve performance of v2 state resolution for large rooms. ([\#6952](https://github.com/matrix-org/synapse/issues/6952), [\#7095](https://github.com/matrix-org/synapse/issues/7095))
- Reduce time spent doing GC, by freezing objects on startup. ([\#6953](https://github.com/matrix-org/synapse/issues/6953))
- Minor perfermance fixes to `get_auth_chain_ids`. ([\#6954](https://github.com/matrix-org/synapse/issues/6954))
- Minor performance fixes to `get_auth_chain_ids`. ([\#6954](https://github.com/matrix-org/synapse/issues/6954))
- Don't record remote cross-signing keys in the `devices` table. ([\#6956](https://github.com/matrix-org/synapse/issues/6956))
- Use flake8-comprehensions to enforce good hygiene of list/set/dict comprehensions. ([\#6957](https://github.com/matrix-org/synapse/issues/6957))
- Merge worker apps together. ([\#6964](https://github.com/matrix-org/synapse/issues/6964), [\#7002](https://github.com/matrix-org/synapse/issues/7002), [\#7055](https://github.com/matrix-org/synapse/issues/7055), [\#7104](https://github.com/matrix-org/synapse/issues/7104))
@ -1809,7 +1809,7 @@ Bugfixes
- Allow URL-encoded User IDs on `/_synapse/admin/v2/users/<user_id>[/admin]` endpoints. Thanks to @NHAS for reporting. ([\#6825](https://github.com/matrix-org/synapse/issues/6825))
- Fix Synapse refusing to start if `federation_certificate_verification_whitelist` option is blank. ([\#6849](https://github.com/matrix-org/synapse/issues/6849))
- Fix errors from logging in the purge jobs related to the message retention policies support. ([\#6945](https://github.com/matrix-org/synapse/issues/6945))
- Return a 404 instead of 200 for querying information of a non-existant user through the admin API. ([\#6901](https://github.com/matrix-org/synapse/issues/6901))
- Return a 404 instead of 200 for querying information of a non-existent user through the admin API. ([\#6901](https://github.com/matrix-org/synapse/issues/6901))
Updates to the Docker image
@ -1889,7 +1889,7 @@ Bugfixes
Synapse 1.10.0rc4 (2020-02-11)
==============================
This release candidate was built incorrectly and is superceded by 1.10.0rc5.
This release candidate was built incorrectly and is superseded by 1.10.0rc5.
Synapse 1.10.0rc3 (2020-02-10)
==============================

View File

@ -2270,7 +2270,7 @@ Features
Bugfixes
--------
- Fix spurious errors in logs when deleting a non-existant pusher. ([\#9121](https://github.com/matrix-org/synapse/issues/9121))
- Fix spurious errors in logs when deleting a non-existent pusher. ([\#9121](https://github.com/matrix-org/synapse/issues/9121))
- Fix a long-standing bug where Synapse would return a 500 error when a thumbnail did not exist (and auto-generation of thumbnails was not enabled). ([\#9163](https://github.com/matrix-org/synapse/issues/9163))
- Fix a long-standing bug where an internal server error was raised when attempting to preview an HTML document in an unknown character encoding. ([\#9164](https://github.com/matrix-org/synapse/issues/9164))
- Fix a long-standing bug where invalid data could cause errors when calculating the presentable room name for push. ([\#9165](https://github.com/matrix-org/synapse/issues/9165))
@ -2522,7 +2522,7 @@ Bugfixes
- Fix a long-standing bug where a `m.image` event without a `url` would cause errors on push. ([\#8965](https://github.com/matrix-org/synapse/issues/8965))
- Fix a small bug in v2 state resolution algorithm, which could also cause performance issues for rooms with large numbers of power levels. ([\#8971](https://github.com/matrix-org/synapse/issues/8971))
- Add validation to the `sendToDevice` API to raise a missing parameters error instead of a 500 error. ([\#8975](https://github.com/matrix-org/synapse/issues/8975))
- Add validation of group IDs to raise a 400 error instead of a 500 eror. ([\#8977](https://github.com/matrix-org/synapse/issues/8977))
- Add validation of group IDs to raise a 400 error instead of a 500 error. ([\#8977](https://github.com/matrix-org/synapse/issues/8977))
Improved Documentation

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@ -8,9 +8,9 @@ to the server until they have.
There are several parts to this functionality; each requires some specific
configuration in `homeserver.yaml` to be enabled.
Note that various parts of the configuation and this document refer to the
Note that various parts of the configuration and this document refer to the
"privacy policy": agreement with a privacy policy is one particular use of this
feature, but of course adminstrators can specify other terms and conditions
feature, but of course administrators can specify other terms and conditions
unrelated to "privacy" per se.
Collecting policy agreement from a user

View File

@ -23,7 +23,7 @@ people building from source should ensure they can fetch recent versions of Rust
(e.g. by using [rustup](https://rustup.rs/)).
The oldest supported version of SQLite is the version
[provided](https://packages.debian.org/buster/libsqlite3-0) by
[provided](https://packages.debian.org/bullseye/libsqlite3-0) by
[Debian oldstable](https://wiki.debian.org/DebianOldStable).
Context

View File

@ -66,7 +66,7 @@ Of their installation methods, we recommend
```shell
pip install --user pipx
pipx install poetry
pipx install poetry==1.5.1 # Problems with Poetry 1.6, see https://github.com/matrix-org/synapse/issues/16147
```
but see poetry's [installation instructions](https://python-poetry.org/docs/#installation)
@ -266,7 +266,7 @@ The easiest way to do so is to run Postgres via a docker container. In one
terminal:
```shell
docker run --rm -e POSTGRES_PASSWORD=mysecretpassword -e POSTGRES_USER=postgres -e POSTGRES_DB=postgress -p 5432:5432 postgres:14
docker run --rm -e POSTGRES_PASSWORD=mysecretpassword -e POSTGRES_USER=postgres -e POSTGRES_DB=postgres -p 5432:5432 postgres:14
```
If you see an error like
@ -322,7 +322,7 @@ The following command will let you run the integration test with the most common
configuration:
```sh
$ docker run --rm -it -v /path/where/you/have/cloned/the/repository\:/src:ro -v /path/to/where/you/want/logs\:/logs matrixdotorg/sytest-synapse:buster
$ docker run --rm -it -v /path/where/you/have/cloned/the/repository\:/src:ro -v /path/to/where/you/want/logs\:/logs matrixdotorg/sytest-synapse:focal
```
(Note that the paths must be full paths! You could also write `$(realpath relative/path)` if needed.)
@ -370,6 +370,7 @@ The above will run a monolithic (single-process) Synapse with SQLite as the data
See the [worker documentation](../workers.md) for additional information on workers.
- Passing `ASYNCIO_REACTOR=1` as an environment variable to use the Twisted asyncio reactor instead of the default one.
- Passing `PODMAN=1` will use the [podman](https://podman.io/) container runtime, instead of docker.
- Passing `UNIX_SOCKETS=1` will utilise Unix socket functionality for Synapse, Redis, and Postgres(when applicable).
To increase the log level for the tests, set `SYNAPSE_TEST_LOG_LEVEL`, e.g:
```sh

View File

@ -150,6 +150,67 @@ def run_upgrade(
...
```
## Background updates
It is sometimes appropriate to perform database migrations as part of a background
process (instead of blocking Synapse until the migration is done). In particular,
this is useful for migrating data when adding new columns or tables.
Pending background updates stored in the `background_updates` table and are denoted
by a unique name, the current status (stored in JSON), and some dependency information:
* Whether the update requires a previous update to be complete.
* A rough ordering for which to complete updates.
A new background updates needs to be added to the `background_updates` table:
```sql
INSERT INTO background_updates (ordering, update_name, depends_on, progress_json) VALUES
(7706, 'my_background_update', 'a_previous_background_update' '{}');
```
And then needs an associated handler in the appropriate datastore:
```python
self.db_pool.updates.register_background_update_handler(
"my_background_update",
update_handler=self._my_background_update,
)
```
There are a few types of updates that can be performed, see the `BackgroundUpdater`:
* `register_background_update_handler`: A generic handler for custom SQL
* `register_background_index_update`: Create an index in the background
* `register_background_validate_constraint`: Validate a constraint in the background
(PostgreSQL-only)
* `register_background_validate_constraint_and_delete_rows`: Similar to
`register_background_validate_constraint`, but deletes rows which don't fit
the constraint.
For `register_background_update_handler`, the generic handler must track progress
and then finalize the background update:
```python
async def _my_background_update(self, progress: JsonDict, batch_size: int) -> int:
def _do_something(txn: LoggingTransaction) -> int:
...
self.db_pool.updates._background_update_progress_txn(
txn, "my_background_update", {"last_processed": last_processed}
)
return last_processed - prev_last_processed
num_processed = await self.db_pool.runInteraction("_do_something", _do_something)
await self.db_pool.updates._end_background_update("my_background_update")
return num_processed
```
Synapse will attempt to rate-limit how often background updates are run via the
given batch-size and the returned number of processed entries (and how long the
function took to run). See
[background update controller callbacks](../modules/background_update_controller_callbacks.md).
## Boolean columns
Boolean columns require special treatment, since SQLite treats booleans the
@ -184,3 +245,160 @@ version `3`, that can only happen with a hash collision, which we basically hope
will never happen (SHA256 has a massive big key space).
## Worked examples of gradual migrations
Some migrations need to be performed gradually. A prime example of this is anything
which would need to do a large table scan — including adding columns, indices or
`NOT NULL` constraints to non-empty tables — such a migration should be done as a
background update where possible, at least on Postgres.
We can afford to be more relaxed about SQLite databases since they are usually
used on smaller deployments and SQLite does not support the same concurrent
DDL operations as Postgres.
We also typically insist on having at least one Synapse version's worth of
backwards compatibility, so that administrators can roll back Synapse if an upgrade
did not go smoothly.
This sometimes results in having to plan a migration across multiple versions
of Synapse.
This section includes an example and may include more in the future.
### Transforming a column into another one, with `NOT NULL` constraints
This example illustrates how you would introduce a new column, write data into it
based on data from an old column and then drop the old column.
We are aiming for semantic equivalence to:
```sql
ALTER TABLE mytable ADD COLUMN new_column INTEGER;
UPDATE mytable SET new_column = old_column * 100;
ALTER TABLE mytable ALTER COLUMN new_column ADD CONSTRAINT NOT NULL;
ALTER TABLE mytable DROP COLUMN old_column;
```
#### Synapse version `N`
```python
SCHEMA_VERSION = S
SCHEMA_COMPAT_VERSION = ... # unimportant at this stage
```
**Invariants:**
1. `old_column` is read by Synapse and written to by Synapse.
#### Synapse version `N + 1`
```python
SCHEMA_VERSION = S + 1
SCHEMA_COMPAT_VERSION = ... # unimportant at this stage
```
**Changes:**
1.
```sql
ALTER TABLE mytable ADD COLUMN new_column INTEGER;
```
**Invariants:**
1. `old_column` is read by Synapse and written to by Synapse.
2. `new_column` is written to by Synapse.
**Notes:**
1. `new_column` can't have a `NOT NULL NOT VALID` constraint yet, because the previous Synapse version did not write to the new column (since we haven't bumped the `SCHEMA_COMPAT_VERSION` yet, we still need to be compatible with the previous version).
#### Synapse version `N + 2`
```python
SCHEMA_VERSION = S + 2
SCHEMA_COMPAT_VERSION = S + 1 # this signals that we can't roll back to a time before new_column existed
```
**Changes:**
1. On Postgres, add a `NOT VALID` constraint to ensure new rows are compliant. *SQLite does not have such a construct, but it would be unnecessary anyway since there is no way to concurrently perform this migration on SQLite.*
```sql
ALTER TABLE mytable ADD CONSTRAINT CHECK new_column_not_null (new_column IS NOT NULL) NOT VALID;
```
2. Start a background update to perform migration: it should gradually run e.g.
```sql
UPDATE mytable SET new_column = old_column * 100 WHERE 0 < mytable_id AND mytable_id <= 5;
```
This background update is technically pointless on SQLite, but you must schedule it anyway so that the `portdb` script to migrate to Postgres still works.
3. Upon completion of the background update, you should run `VALIDATE CONSTRAINT` on Postgres to turn the `NOT VALID` constraint into a valid one.
```sql
ALTER TABLE mytable VALIDATE CONSTRAINT new_column_not_null;
```
This will take some time but does **NOT** hold an exclusive lock over the table.
**Invariants:**
1. `old_column` is read by Synapse and written to by Synapse.
2. `new_column` is written to by Synapse and new rows always have a non-`NULL` value in this field.
**Notes:**
1. If you wish, you can convert the `CHECK (new_column IS NOT NULL)` to a `NOT NULL` constraint free of charge in Postgres by adding the `NOT NULL` constraint and then dropping the `CHECK` constraint, because Postgres can statically verify that the `NOT NULL` constraint is implied by the `CHECK` constraint without performing a table scan.
2. It might be tempting to make version `N + 2` redundant by moving the background update to `N + 1` and delaying adding the `NOT NULL` constraint to `N + 3`, but that would mean the constraint would always be validated in the foreground in `N + 3`. Whereas if the `N + 2` step is kept, the migration in `N + 3` would be fast in the happy case.
#### Synapse version `N + 3`
```python
SCHEMA_VERSION = S + 3
SCHEMA_COMPAT_VERSION = S + 1 # we can't roll back to a time before new_column existed
```
**Changes:**
1. (Postgres) Update the table to populate values of `new_column` in case the background update had not completed. Additionally, `VALIDATE CONSTRAINT` to make the check fully valid.
```sql
-- you ideally want an index on `new_column` or e.g. `(new_column) WHERE new_column IS NULL` first, or perhaps you can find a way to skip this if the `NOT NULL` constraint has already been validated.
UPDATE mytable SET new_column = old_column * 100 WHERE new_column IS NULL;
-- this is a no-op if it already ran as part of the background update
ALTER TABLE mytable VALIDATE CONSTRAINT new_column_not_null;
```
2. (SQLite) Recreate the table by precisely following [the 12-step procedure for SQLite table schema changes](https://www.sqlite.org/lang_altertable.html#otheralter).
During this table rewrite, you should recreate `new_column` as `NOT NULL` and populate any outstanding `NULL` values at the same time.
Unfortunately, you can't drop `old_column` yet because it must be present for compatibility with the Postgres schema, as needed by `portdb`.
(Otherwise you could do this all in one go with SQLite!)
**Invariants:**
1. `old_column` is written to by Synapse (but no longer read by Synapse!).
2. `new_column` is read by Synapse and written to by Synapse. Moreover, all rows have a non-`NULL` value in this field, as guaranteed by a schema constraint.
**Notes:**
1. We can't drop `old_column` yet, or even stop writing to it, because that would break a rollback to the previous version of Synapse.
2. Application code can now rely on `new_column` being populated. The remaining steps are only motivated by the wish to clean-up old columns.
#### Synapse version `N + 4`
```python
SCHEMA_VERSION = S + 4
SCHEMA_COMPAT_VERSION = S + 3 # we can't roll back to a time before new_column was entirely non-NULL
```
**Invariants:**
1. `old_column` exists but is not written to or read from by Synapse.
2. `new_column` is read by Synapse and written to by Synapse. Moreover, all rows have a non-`NULL` value in this field, as guaranteed by a schema constraint.
**Notes:**
1. We can't drop `old_column` yet because that would break a rollback to the previous version of Synapse. \
**TODO:** It may be possible to relax this and drop the column straight away as long as the previous version of Synapse detected a rollback occurred and stopped attempting to write to the column. This could possibly be done by checking whether the database's schema compatibility version was `S + 3`.
#### Synapse version `N + 5`
```python
SCHEMA_VERSION = S + 5
SCHEMA_COMPAT_VERSION = S + 4 # we can't roll back to a time before old_column was no longer being touched
```
**Changes:**
1.
```sql
ALTER TABLE mytable DROP COLUMN old_column;
```

View File

@ -12,7 +12,7 @@ Note that this schedule might be modified depending on the availability of the
Synapse team, e.g. releases may be skipped to avoid holidays.
Release announcements can be found in the
[release category of the Matrix blog](https://matrix.org/blog/category/releases).
[release category of the Matrix blog](https://matrix.org/category/releases).
## Bugfix releases
@ -34,4 +34,4 @@ be held to be released together.
In some cases, a pre-disclosure of a security release will be issued as a notice
to Synapse operators that there is an upcoming security release. These can be
found in the [security category of the Matrix blog](https://matrix.org/blog/category/security).
found in the [security category of the Matrix blog](https://matrix.org/category/security).

View File

@ -6,7 +6,7 @@ This is a work-in-progress set of notes with two goals:
See also [MSC3902](https://github.com/matrix-org/matrix-spec-proposals/pull/3902).
The key idea is described by [MSC706](https://github.com/matrix-org/matrix-spec-proposals/pull/3902). This allows servers to
The key idea is described by [MSC3706](https://github.com/matrix-org/matrix-spec-proposals/pull/3706). This allows servers to
request a lightweight response to the federation `/send_join` endpoint.
This is called a **faster join**, also known as a **partial join**. In these
notes we'll usually use the word "partial" as it matches the database schema.
@ -264,7 +264,7 @@ But don't want to send out sensitive data in other HS's events in this way.
Suppose we discover after resync that we shouldn't have sent out one our events (not a prev_event) to a target HS. Not much we can do.
What about if we didn't send them an event but shouldn't've?
E.g. what if someone joined from a new HS shortly after you did? We wouldn't talk to them.
Could imagine sending out the "Missed" events after the resync but... painful to work out what they shuld have seen if they joined/left.
Could imagine sending out the "Missed" events after the resync but... painful to work out what they should have seen if they joined/left.
Instead, just send them the latest event (if they're still in the room after resync) and let them backfill.(?)
- Don't do this currently.
- If anyone who has received our messages sends a message to a HS we missed, they can backfill our messages

View File

@ -0,0 +1,164 @@
## Streams
Synapse has a concept of "streams", which are roughly described in [`id_generators.py`](
https://github.com/matrix-org/synapse/blob/develop/synapse/storage/util/id_generators.py
).
Generally speaking, streams are a series of notifications that something in Synapse's database has changed that the application might need to respond to.
For example:
- The events stream reports new events (PDUs) that Synapse creates, or that Synapse accepts from another homeserver.
- The account data stream reports changes to users' [account data](https://spec.matrix.org/v1.7/client-server-api/#client-config).
- The to-device stream reports when a device has a new [to-device message](https://spec.matrix.org/v1.7/client-server-api/#send-to-device-messaging).
See [`synapse.replication.tcp.streams`](
https://github.com/matrix-org/synapse/blob/develop/synapse/replication/tcp/streams/__init__.py
) for the full list of streams.
It is very helpful to understand the streams mechanism when working on any part of Synapse that needs to respond to changes—especially if those changes are made by different workers.
To that end, let's describe streams formally, paraphrasing from the docstring of [`AbstractStreamIdGenerator`](
https://github.com/matrix-org/synapse/blob/a719b703d9bd0dade2565ddcad0e2f3a7a9d4c37/synapse/storage/util/id_generators.py#L96
).
### Definition
A stream is an append-only log `T1, T2, ..., Tn, ...` of facts[^1] which grows over time.
Only "writers" can add facts to a stream, and there may be multiple writers.
Each fact has an ID, called its "stream ID".
Readers should only process facts in ascending stream ID order.
Roughly speaking, each stream is backed by a database table.
It should have a `stream_id` (or similar) bigint column holding stream IDs, plus additional columns as necessary to describe the fact.
Typically, a fact is expressed with a single row in its backing table.[^2]
Within a stream, no two facts may have the same stream_id.
> _Aside_. Some additional notes on streams' backing tables.
>
> 1. Rich would like to [ditch the backing tables](https://github.com/matrix-org/synapse/issues/13456).
> 2. The backing tables may have other uses.
> For example, the events table serves backs the events stream, and is read when processing new events.
> But old rows are read from the table all the time, whenever Synapse needs to lookup some facts about an event.
> 3. Rich suspects that sometimes the stream is backed by multiple tables, so the stream proper is the union of those tables.
Stream writers can "reserve" a stream ID, and then later mark it as having being completed.
Stream writers need to track the completion of each stream fact.
In the happy case, completion means a fact has been written to the stream table.
But unhappy cases (e.g. transaction rollback due to an error) also count as completion.
Once completed, the rows written with that stream ID are fixed, and no new rows
will be inserted with that ID.
### Current stream ID
For any given stream reader (including writers themselves), we may define a per-writer current stream ID:
> A current stream ID _for a writer W_ is the largest stream ID such that
> all transactions added by W with equal or smaller ID have completed.
Similarly, there is a "linear" notion of current stream ID:
> A "linear" current stream ID is the largest stream ID such that
> all facts (added by any writer) with equal or smaller ID have completed.
Because different stream readers A and B learn about new facts at different times, A and B may disagree about current stream IDs.
Put differently: we should think of stream readers as being independent of each other, proceeding through a stream of facts at different rates.
The above definition does not give a unique current stream ID, in fact there can
be a range of current stream IDs. Synapse uses both the minimum and maximum IDs
for different purposes. Most often the maximum is used, as its generally
beneficial for workers to advance their IDs as soon as possible. However, the
minimum is used in situations where e.g. another worker is going to wait until
the stream advances past a position.
**NB.** For both senses of "current", that if a writer opens a transaction that never completes, the current stream ID will never advance beyond that writer's last written stream ID.
For single-writer streams, the per-writer current ID and the linear current ID are the same.
Both senses of current ID are monotonic, but they may "skip" or jump over IDs because facts complete out of order.
_Example_.
Consider a single-writer stream which is initially at ID 1.
| Action | Current stream ID | Notes |
|------------|-------------------|-------------------------------------------------|
| | 1 | |
| Reserve 2 | 1 | |
| Reserve 3 | 1 | |
| Complete 3 | 1 | current ID unchanged, waiting for 2 to complete |
| Complete 2 | 3 | current ID jumps from 1 -> 3 |
| Reserve 4 | 3 | |
| Reserve 5 | 3 | |
| Reserve 6 | 3 | |
| Complete 5 | 3 | |
| Complete 4 | 5 | current ID jumps 3->5, even though 6 is pending |
| Complete 6 | 6 | |
### Multi-writer streams
There are two ways to view a multi-writer stream.
1. Treat it as a collection of distinct single-writer streams, one
for each writer.
2. Treat it as a single stream.
The single stream (option 2) is conceptually simpler, and easier to represent (a single stream id).
However, it requires each reader to know about the entire set of writers, to ensures that readers don't erroneously advance their current stream position too early and miss a fact from an unknown writer.
In contrast, multiple parallel streams (option 1) are more complex, requiring more state to represent (map from writer to stream id).
The payoff for doing so is that readers can "peek" ahead to facts that completed on one writer no matter the state of the others, reducing latency.
Note that a multi-writer stream can be viewed in both ways.
For example, the events stream is treated as multiple single-writer streams (option 1) by the sync handler, so that events are sent to clients as soon as possible.
But the background process that works through events treats them as a single linear stream.
Another useful example is the cache invalidation stream.
The facts this stream holds are instructions to "you should now invalidate these cache entries".
We only ever treat this as a multiple single-writer streams as there is no important ordering between cache invalidations.
(Invalidations are self-contained facts; and the invalidations commute/are idempotent).
### Writing to streams
Writers need to track:
- track their current position (i.e. its own per-writer stream ID).
- their facts currently awaiting completion.
At startup,
- the current position of that writer can be found by querying the database (which suggests that facts need to be written to the database atomically, in a transaction); and
- there are no facts awaiting completion.
To reserve a stream ID, call [`nextval`](https://www.postgresql.org/docs/current/functions-sequence.html) on the appropriate postgres sequence.
To write a fact to the stream: insert the appropriate rows to the appropriate backing table.
To complete a fact, first remove it from your map of facts currently awaiting completion.
Then, if no earlier fact is awaiting completion, the writer can advance its current position in that stream.
Upon doing so it should emit an `RDATA` message[^3], once for every fact between the old and the new stream ID.
### Subscribing to streams
Readers need to track the current position of every writer.
At startup, they can find this by contacting each writer with a `REPLICATE` message,
requesting that all writers reply describing their current position in their streams.
Writers reply with a `POSITION` message.
To learn about new facts, readers should listen for `RDATA` messages and process them to respond to the new fact.
The `RDATA` itself is not a self-contained representation of the fact;
readers will have to query the stream tables for the full details.
Readers must also advance their record of the writer's current position for that stream.
# Summary
In a nutshell: we have an append-only log with a "buffer/scratchpad" at the end where we have to wait for the sequence to be linear and contiguous.
---
[^1]: we use the word _fact_ here for two reasons.
Firstly, the word "event" is already heavily overloaded (PDUs, EDUs, account data, ...) and we don't need to make that worse.
Secondly, "fact" emphasises that the things we append to a stream cannot change after the fact.
[^2]: A fact might be expressed with 0 rows, e.g. if we opened a transaction to persist an event, but failed and rolled the transaction back before marking the fact as completed.
In principle a fact might be expressed with 2 or more rows; if so, each of those rows should share the fact's stream ID.
[^3]: This communication used to happen directly with the writers [over TCP](../../tcp_replication.md);
nowadays it's done via Redis's Pubsub.

View File

@ -86,7 +86,7 @@ So we have stopped processing the request (and will probably go on to
start processing the next), without clearing the logcontext.
To circumvent this problem, synapse code assumes that, wherever you have
an awaitable, you will want to `await` it. To that end, whereever
an awaitable, you will want to `await` it. To that end, wherever
functions return awaitables, we adopt the following conventions:
**Rules for functions returning awaitables:**

View File

@ -8,8 +8,7 @@ and allow server and room admins to configure how long messages should
be kept in a homeserver's database before being purged from it.
**Please note that, as this feature isn't part of the Matrix
specification yet, this implementation is to be considered as
experimental. There are known bugs which may cause database corruption.
Proceed with caution.**
experimental.**
A message retention policy is mainly defined by its `max_lifetime`
parameter, which defines how long a message can be kept around after

View File

@ -42,3 +42,16 @@ operations to keep track of them. (e.g. add them to a database table). The user
represented by their Matrix user ID.
If multiple modules implement this callback, Synapse runs them all in order.
### `on_user_login`
_First introduced in Synapse v1.98.0_
```python
async def on_user_login(user_id: str, auth_provider_type: str, auth_provider_id: str) -> None
```
Called after successfully login or registration of a user for cases when module needs to perform extra operations after auth.
represented by their Matrix user ID.
If multiple modules implement this callback, Synapse runs them all in order.

View File

@ -0,0 +1,32 @@
# Add extra fields to client events unsigned section callbacks
_First introduced in Synapse v1.96.0_
This callback allows modules to add extra fields to the unsigned section of
events when they get sent down to clients.
These get called *every* time an event is to be sent to clients, so care should
be taken to ensure with respect to performance.
### API
To register the callback, use
`register_add_extra_fields_to_unsigned_client_event_callbacks` on the
`ModuleApi`.
The callback should be of the form
```python
async def add_field_to_unsigned(
event: EventBase,
) -> JsonDict:
```
where the extra fields to add to the event's unsigned section is returned.
(Modules must not attempt to modify the `event` directly).
This cannot be used to alter the "core" fields in the unsigned section emitted
by Synapse itself.
If multiple such callbacks try to add the same field to an event's unsigned
section, the last-registered callback wins.

View File

@ -1,8 +1,16 @@
# Presence router callbacks
Presence router callbacks allow module developers to specify additional users (local or remote)
to receive certain presence updates from local users. Presence router callbacks can be
registered using the module API's `register_presence_router_callbacks` method.
Presence router callbacks allow module developers to define additional users
which receive presence updates from local users. The additional users
can be local or remote.
For example, it could be used to direct all of `@alice:example.com` (a local user)'s
presence updates to `@bob:matrix.org` (a remote user), even though they don't share a
room. (Note that those presence updates might not make it to `@bob:matrix.org`'s client
unless a similar presence router is running on that homeserver.)
Presence router callbacks can be registered using the module API's
`register_presence_router_callbacks` method.
## Callbacks

View File

@ -348,6 +348,42 @@ callback returns `False`, Synapse falls through to the next one. The value of th
callback that does not return `False` will be used. If this happens, Synapse will not call
any of the subsequent implementations of this callback.
### `check_login_for_spam`
_First introduced in Synapse v1.87.0_
```python
async def check_login_for_spam(
user_id: str,
device_id: Optional[str],
initial_display_name: Optional[str],
request_info: Collection[Tuple[Optional[str], str]],
auth_provider_id: Optional[str] = None,
) -> Union["synapse.module_api.NOT_SPAM", "synapse.module_api.errors.Codes"]
```
Called when a user logs in.
The arguments passed to this callback are:
* `user_id`: The user ID the user is logging in with
* `device_id`: The device ID the user is re-logging into.
* `initial_display_name`: The device display name, if any.
* `request_info`: A collection of tuples, which first item is a user agent, and which
second item is an IP address. These user agents and IP addresses are the ones that were
used during the login process.
* `auth_provider_id`: The identifier of the SSO authentication provider, if any.
If multiple modules implement this callback, they will be considered in order. If a
callback returns `synapse.module_api.NOT_SPAM`, Synapse falls through to the next one.
The value of the first callback that does not return `synapse.module_api.NOT_SPAM` will
be used. If this happens, Synapse will not call any of the subsequent implementations of
this callback.
*Note:* This will not be called when a user registers.
## Example
The example below is a module that implements the spam checker callback

View File

@ -51,6 +51,11 @@ docker run -d --name jaeger \
jaegertracing/all-in-one:1
```
By default, Synapse will publish traces to Jaeger on localhost.
If Jaeger is hosted elsewhere, point Synapse to the correct host by setting
`opentracing.jaeger_config.local_agent.reporting_host` [in the Synapse configuration](usage/configuration/config_documentation.md#opentracing-1)
or by setting the `JAEGER_AGENT_HOST` environment variable to the desired address.
Latest documentation is probably at
https://www.jaegertracing.io/docs/latest/getting-started.

View File

@ -66,7 +66,7 @@ database:
args:
user: <user>
password: <pass>
database: <db>
dbname: <db>
host: <host>
cp_min: 5
cp_max: 10
@ -249,7 +249,7 @@ of `COLLATE` and `CTYPE` unless the config flag `allow_unsafe_locale`, found in
underneath the database, or if a different version of the locale is used on any
replicas.
If you have a databse with an unsafe locale, the safest way to fix the issue is to dump the database and recreate it with
If you have a database with an unsafe locale, the safest way to fix the issue is to dump the database and recreate it with
the correct locale parameter (as shown above). It is also possible to change the
parameters on a live database and run a `REINDEX` on the entire database,
however extreme care must be taken to avoid database corruption.

View File

@ -95,7 +95,7 @@ matrix.example.com {
}
example.com:8448 {
reverse_proxy localhost:8008
reverse_proxy /_matrix/* localhost:8008
}
```
@ -181,7 +181,11 @@ frontend matrix-federation
backend matrix
server matrix 127.0.0.1:8008
```
Example configuration, if using a UNIX socket. The configuration lines regarding the frontends do not need to be modified.
```
backend matrix
server matrix unix@/run/synapse/main_public.sock
```
[Delegation](delegate.md) example:
```

View File

@ -46,6 +46,7 @@ server_notices:
system_mxid_display_name: "Server Notices"
system_mxid_avatar_url: "mxc://server.com/oumMVlgDnLYFaPVkExemNVVZ"
room_name: "Server Notices"
auto_join: true
```
The only compulsory setting is `system_mxid_localpart`, which defines the user
@ -55,6 +56,8 @@ room which will be created.
`system_mxid_display_name` and `system_mxid_avatar_url` can be used to set the
displayname and avatar of the Server Notices user.
`auto_join` will autojoin users to the notices room instead of sending an invite.
## Sending notices
To send server notices to users you can use the

View File

@ -37,7 +37,7 @@ Dockerfile to automate a synapse server in a single Docker image, at
<https://hub.docker.com/r/avhost/docker-matrix/tags/>
Slavi Pantaleev has created an Ansible playbook,
which installs the offical Docker image of Matrix Synapse
which installs the official Docker image of Matrix Synapse
along with many other Matrix-related services (Postgres database, Element, coturn,
ma1sd, SSL support, etc.).
For more details, see
@ -93,7 +93,7 @@ For `bookworm` and `sid`, it can be installed simply with:
sudo apt install matrix-synapse
```
Synapse is also avaliable in `bullseye-backports`. Please
Synapse is also available in `bullseye-backports`. Please
see the [Debian documentation](https://backports.debian.org/Instructions/)
for information on how to use backports.
@ -135,8 +135,8 @@ Unofficial package are built for SLES 15 in the openSUSE:Backports:SLE-15 reposi
#### ArchLinux
The quickest way to get up and running with ArchLinux is probably with the community package
<https://archlinux.org/packages/community/x86_64/matrix-synapse/>, which should pull in most of
The quickest way to get up and running with ArchLinux is probably with the package provided by ArchLinux
<https://archlinux.org/packages/extra/x86_64/matrix-synapse/>, which should pull in most of
the necessary dependencies.
pip may be outdated (6.0.7-1 and needs to be upgraded to 6.0.8-1 ):
@ -155,6 +155,14 @@ sudo pip uninstall py-bcrypt
sudo pip install py-bcrypt
```
#### Alpine Linux
6543 maintains [Synapse packages for Alpine Linux](https://pkgs.alpinelinux.org/packages?name=synapse&branch=edge) in the community repository. Install with:
```sh
sudo apk add synapse
```
#### Void Linux
Synapse can be found in the void repositories as
@ -200,7 +208,7 @@ When following this route please make sure that the [Platform-specific prerequis
System requirements:
- POSIX-compliant system (tested on Linux & OS X)
- Python 3.7 or later, up to Python 3.11.
- Python 3.8 or later, up to Python 3.11.
- At least 1GB of free RAM if you want to join large public rooms like #matrix:matrix.org
If building on an uncommon architecture for which pre-built wheels are

View File

@ -3,7 +3,7 @@
A structured logging system can be useful when your logs are destined for a
machine to parse and process. By maintaining its machine-readable characteristics,
it enables more efficient searching and aggregations when consumed by software
such as the "ELK stack".
such as the [ELK stack](https://opensource.com/article/18/9/open-source-log-aggregation-tools).
Synapse's structured logging system is configured via the file that Synapse's
`log_config` config option points to. The file should include a formatter which

View File

@ -1,8 +1,4 @@
worker_app: synapse.app.generic_worker
worker_name: background_worker
# The replication listener on the main synapse process.
worker_replication_host: 127.0.0.1
worker_replication_http_port: 9093
worker_log_config: /etc/matrix-synapse/background-worker-log.yaml

View File

@ -1,9 +1,5 @@
worker_app: synapse.app.generic_worker
worker_name: event_persister1
# The replication listener on the main synapse process.
worker_replication_host: 127.0.0.1
worker_replication_http_port: 9093
worker_name: event_persister1
worker_listeners:
- type: http

View File

@ -1,8 +1,4 @@
worker_app: synapse.app.federation_sender
worker_name: federation_sender1
# The replication listener on the main synapse process.
worker_replication_host: 127.0.0.1
worker_replication_http_port: 9093
worker_log_config: /etc/matrix-synapse/federation-sender-log.yaml

View File

@ -1,10 +1,6 @@
worker_app: synapse.app.media_repository
worker_name: media_worker
# The replication listener on the main synapse process.
worker_replication_host: 127.0.0.1
worker_replication_http_port: 9093
worker_listeners:
- type: http
port: 8085

View File

@ -1,8 +1,4 @@
worker_app: synapse.app.pusher
worker_name: pusher_worker1
# The replication listener on the main synapse process.
worker_replication_host: 127.0.0.1
worker_replication_http_port: 9093
worker_log_config: /etc/matrix-synapse/pusher-worker-log.yaml

View File

@ -38,7 +38,7 @@ noted when manually using the protocol:
been disabled on the main process.
- The server will only time connections out that have sent a `PING`
command. If a ping is sent then the connection will be closed if no
further commands are receieved within 15s. Both the client and
further commands are received within 15s. Both the client and
server protocol implementations will send an initial PING on
connection and ensure at least one command every 5s is sent (not
necessarily `PING`).
@ -128,7 +128,7 @@ batching. See `RdataCommand` for more details.
### Example
An example iteraction is shown below. Each line is prefixed with '>'
An example interaction is shown below. Each line is prefixed with '>'
or '<' to indicate which side is sending, these are *not* included on
the wire:

View File

@ -18,7 +18,7 @@ This documentation provides two TURN server configuration examples:
For TURN relaying to work, the TURN service must be hosted on a server/endpoint with a public IP.
Hosting TURN behind NAT requires port forwaring and for the NAT gateway to have a public IP.
Hosting TURN behind NAT requires port forwarding and for the NAT gateway to have a public IP.
However, even with appropriate configuration, NAT is known to cause issues and to often not work.
Afterwards, the homeserver needs some further configuration.

View File

@ -87,6 +87,74 @@ process, for example:
wget https://packages.matrix.org/debian/pool/main/m/matrix-synapse-py3/matrix-synapse-py3_1.3.0+stretch1_amd64.deb
dpkg -i matrix-synapse-py3_1.3.0+stretch1_amd64.deb
```
Generally Synapse database schemas are compatible across multiple versions, once
a version of Synapse is deployed you may not be able to rollback automatically.
The following table gives the version ranges and the earliest version they can
be rolled back to. E.g. Synapse versions v1.58.0 through v1.61.1 can be rolled
back safely to v1.57.0, but starting with v1.62.0 it is only safe to rollback to
v1.61.0.
<!-- REPLACE_WITH_SCHEMA_VERSIONS -->
# Upgrading to v1.93.0
## Minimum supported Rust version
The minimum supported Rust version has been increased from v1.60.0 to v1.61.0.
Users building from source will need to ensure their `rustc` version is up to
date.
# Upgrading to v1.90.0
## App service query parameter authorization is now a configuration option
Synapse v1.81.0 deprecated application service authorization via query parameters as this is
considered insecure - and from Synapse v1.71.0 forwards the application service token has also been sent via
[the `Authorization` header](https://spec.matrix.org/v1.6/application-service-api/#authorization)], making the insecure
query parameter authorization redundant. Since removing the ability to continue to use query parameters could break
backwards compatibility it has now been put behind a configuration option, `use_appservice_legacy_authorization`.
This option defaults to false, but can be activated by adding
```yaml
use_appservice_legacy_authorization: true
```
to your configuration.
# Upgrading to v1.89.0
## Removal of unspecced `user` property for `/register`
Application services can no longer call `/register` with a `user` property to create new users.
The standard `username` property should be used instead. See the
[Application Service specification](https://spec.matrix.org/v1.7/application-service-api/#server-admin-style-permissions)
for more information.
# Upgrading to v1.88.0
## Minimum supported Python version
The minimum supported Python version has been increased from v3.7 to v3.8.
You will need Python 3.8 to run Synapse v1.88.0 (due out July 18th, 2023).
If you use current versions of the Matrix.org-distributed Debian
packages or Docker images, no action is required.
## Removal of `worker_replication_*` settings
As mentioned previously in [Upgrading to v1.84.0](#upgrading-to-v1840), the following deprecated settings
are being removed in this release of Synapse:
* [`worker_replication_host`](https://matrix-org.github.io/synapse/v1.86/usage/configuration/config_documentation.html#worker_replication_host)
* [`worker_replication_http_port`](https://matrix-org.github.io/synapse/v1.86/usage/configuration/config_documentation.html#worker_replication_http_port)
* [`worker_replication_http_tls`](https://matrix-org.github.io/synapse/v1.86/usage/configuration/config_documentation.html#worker_replication_http_tls)
Please ensure that you have migrated to using `main` on your shared configuration's `instance_map`
(or create one if necessary). This is required if you have ***any*** workers at all;
administrators of single-process (monolith) installations don't need to do anything.
For an illustrative example, please see [Upgrading to v1.84.0](#upgrading-to-v1840) below.
# Upgrading to v1.86.0
## Minimum supported Rust version
@ -1293,7 +1361,7 @@ In line with our [deprecation policy](deprecation_policy.md),
we've dropped support for Python 3.5 and PostgreSQL 9.5, as they are no
longer supported upstream.
This release of Synapse requires Python 3.6+ and PostgresSQL 9.6+ or
This release of Synapse requires Python 3.6+ and PostgreSQL 9.6+ or
SQLite 3.22+.
## Removal of old List Accounts Admin API
@ -2253,7 +2321,7 @@ for details.
# Upgrading to v0.11.0
This release includes the option to send anonymous usage stats to
matrix.org, and requires that administrators explictly opt in or out by
matrix.org, and requires that administrators explicitly opt in or out by
setting the `report_stats` option to either `true` or `false`.
We would really appreciate it if you could help our project out by
@ -2357,7 +2425,7 @@ latest module, please run:
# Upgrading to v0.5.0
The webclient has been split out into a seperate repository/pacakage in
The webclient has been split out into a separate repository/package in
this release. Before you restart your homeserver you will need to pull
in the webclient package by running:

View File

@ -77,7 +77,7 @@ The following fields are returned in the JSON response body:
remote server, in ms. This is `0` if the last attempt to communicate with the
remote server was successful.
- `retry_interval` - integer - How long since the last time Synapse tried to reach
the remote server before trying again, in ms. This is `0` if no further retrying occuring.
the remote server before trying again, in ms. This is `0` if no further retrying occurring.
- `failure_ts` - nullable integer - The first time Synapse tried and failed to reach the
remote server, in ms. This is `null` if communication with the remote server has never failed.
- `last_successful_stream_ordering` - nullable integer - The stream ordering of the most

View File

@ -1,5 +1,7 @@
# Registration Tokens
**Note:** This API is disabled when MSC3861 is enabled. [See #15582](https://github.com/matrix-org/synapse/pull/15582)
This API allows you to manage tokens which can be used to authenticate
registration requests, as proposed in
[MSC3231](https://github.com/matrix-org/matrix-doc/blob/main/proposals/3231-token-authenticated-registration.md)

View File

@ -193,7 +193,7 @@ SELECT rss.room_id, rss.name, rss.canonical_alias, rss.topic, rss.encryption,
rsc.joined_members, rsc.local_users_in_room, rss.join_rules
FROM room_stats_state rss
LEFT JOIN room_stats_current rsc USING (room_id)
WHERE room_id IN ( WHERE room_id IN (
WHERE room_id IN (
'!OGEhHVWSdvArJzumhm:matrix.org',
'!YTvKGNlinIzlkMTVRl:matrix.org'
);

View File

@ -25,12 +25,31 @@ messages from the database after 5 minutes, rather than 5 months.
In addition, configuration options referring to size use the following suffixes:
* `M` = MiB, or 1,048,576 bytes
* `K` = KiB, or 1024 bytes
* `M` = MiB, or 1,048,576 bytes
* `G` = GiB, or 1,073,741,824 bytes
* `T` = TiB, or 1,099,511,627,776 bytes
For example, setting `max_avatar_size: 10M` means that Synapse will not accept files larger than 10,485,760 bytes
for a user avatar.
## Config Validation
The configuration file can be validated with the following command:
```bash
python -m synapse.config read <config key to print> -c <path to config>
```
To validate the entire file, omit `read <config key to print>`:
```bash
python -m synapse.config -c <path to config>
```
To see how to set other options, check the help reference:
```bash
python -m synapse.config --help
```
### YAML
The configuration file is a [YAML](https://yaml.org/) file, which means that certain syntax rules
apply if you want your config file to be read properly. A few helpful things to know:
@ -228,6 +247,13 @@ Example configuration:
presence:
enabled: false
```
`enabled` can also be set to a special value of "untracked" which ignores updates
received via clients and federation, while still accepting updates from the
[module API](../../modules/index.md).
*The "untracked" option was added in Synapse 1.96.0.*
---
### `require_auth_for_profile_requests`
@ -462,6 +488,20 @@ See the docs [request log format](../administration/request_log.md).
* `additional_resources`: Only valid for an 'http' listener. A map of
additional endpoints which should be loaded via dynamic modules.
Unix socket support (_Added in Synapse 1.89.0_):
* `path`: A path and filename for a Unix socket. Make sure it is located in a
directory with read and write permissions, and that it already exists (the directory
will not be created). Defaults to `None`.
* **Note**: The use of both `path` and `port` options for the same `listener` is not
compatible.
* The `x_forwarded` option defaults to true when using Unix sockets and can be omitted.
* Other options that would not make sense to use with a UNIX socket, such as
`bind_addresses` and `tls` will be ignored and can be removed.
* `mode`: The file permissions to set on the UNIX socket. Defaults to `666`
* **Note:** Must be set as `type: http` (does not support `metrics` and `manhole`).
Also make sure that `metrics` is not included in `resources` -> `names`
Valid resource names are:
* `client`: the client-server API (/_matrix/client), and the synapse admin API (/_synapse/admin). Also implies `media` and `static`.
@ -474,7 +514,7 @@ Valid resource names are:
* `media`: the media API (/_matrix/media).
* `metrics`: the metrics interface. See [here](../../metrics-howto.md).
* `metrics`: the metrics interface. See [here](../../metrics-howto.md). (Not compatible with Unix sockets)
* `openid`: OpenID authentication. See [here](../../openid.md).
@ -505,7 +545,7 @@ listeners:
Example configuration #2:
```yaml
listeners:
# Unsecure HTTP listener: for when matrix traffic passes through a reverse proxy
# Insecure HTTP listener: for when matrix traffic passes through a reverse proxy
# that unwraps TLS.
#
# If you plan to use a reverse proxy, please see
@ -533,6 +573,22 @@ listeners:
bind_addresses: ['::1', '127.0.0.1']
type: manhole
```
Example configuration #3:
```yaml
listeners:
# Unix socket listener: Ideal for Synapse deployments behind a reverse proxy, offering
# lightweight interprocess communication without TCP/IP overhead, avoid port
# conflicts, and providing enhanced security through system file permissions.
#
# Note that x_forwarded will default to true, when using a UNIX socket. Please see
# https://matrix-org.github.io/synapse/latest/reverse_proxy.html.
#
- path: /run/synapse/main_public.sock
type: http
resources:
- names: [client, federation]
```
---
### `manhole_settings`
@ -904,6 +960,17 @@ Example configuration:
redaction_retention_period: 28d
```
---
### `forgotten_room_retention_period`
How long to keep locally forgotten rooms before purging them from the DB.
Defaults to `null`, meaning it's disabled.
Example configuration:
```yaml
forgotten_room_retention_period: 28d
```
---
### `user_ips_max_age`
How long to track users' last seen time and IPs in the database.
@ -983,11 +1050,8 @@ which are older than the room's maximum retention period. Synapse will also
filter events received over federation so that events that should have been
purged are ignored and not stored again.
The message retention policies feature is disabled by default. Please be advised
that enabling this feature carries some risk. There are known bugs with the implementation
which can cause database corruption. Setting retention to delete older history
is less risky than deleting newer history but in general caution is advised when enabling this
experimental feature. You can read more about this feature [here](../../message_retention_policies.md).
The message retention policies feature is disabled by default. You can read more
about this feature [here](../../message_retention_policies.md).
This setting has the following sub-options:
* `default_policy`: Default retention policy. If set, Synapse will apply it to rooms that lack the
@ -1090,14 +1154,14 @@ federation_verify_certificates: false
The minimum TLS version that will be used for outbound federation requests.
Defaults to `1`. Configurable to `1`, `1.1`, `1.2`, or `1.3`. Note
that setting this value higher than `1.2` will prevent federation to most
of the public Matrix network: only configure it to `1.3` if you have an
Defaults to `"1"`. Configurable to `"1"`, `"1.1"`, `"1.2"`, or `"1.3"`. Note
that setting this value higher than `"1.2"` will prevent federation to most
of the public Matrix network: only configure it to `"1.3"` if you have an
entirely private federation setup and you can ensure TLS 1.3 support.
Example configuration:
```yaml
federation_client_minimum_tls_version: 1.2
federation_client_minimum_tls_version: "1.2"
```
---
### `federation_certificate_verification_whitelist`
@ -1150,6 +1214,11 @@ inbound federation traffic as early as possible, rather than relying
purely on this application-layer restriction. If not specified, the
default is to whitelist everything.
Note: this does not stop a server from joining rooms that servers not on the
whitelist are in. As such, this option is really only useful to establish a
"private federation", where a group of servers all whitelist each other and have
the same whitelist.
Example configuration:
```yaml
federation_domain_whitelist:
@ -1196,6 +1265,43 @@ Example configuration:
allow_device_name_lookup_over_federation: true
```
---
### `federation`
The federation section defines some sub-options related to federation.
The following options are related to configuring timeout and retry logic for one request,
independently of the others.
Short retry algorithm is used when something or someone will wait for the request to have an
answer, while long retry is used for requests that happen in the background,
like sending a federation transaction.
* `client_timeout`: timeout for the federation requests. Default to 60s.
* `max_short_retry_delay`: maximum delay to be used for the short retry algo. Default to 2s.
* `max_long_retry_delay`: maximum delay to be used for the short retry algo. Default to 60s.
* `max_short_retries`: maximum number of retries for the short retry algo. Default to 3 attempts.
* `max_long_retries`: maximum number of retries for the long retry algo. Default to 10 attempts.
The following options control the retry logic when communicating with a specific homeserver destination.
Unlike the previous configuration options, these values apply across all requests
for a given destination and the state of the backoff is stored in the database.
* `destination_min_retry_interval`: the initial backoff, after the first request fails. Defaults to 10m.
* `destination_retry_multiplier`: how much we multiply the backoff by after each subsequent fail. Defaults to 2.
* `destination_max_retry_interval`: a cap on the backoff. Defaults to a week.
Example configuration:
```yaml
federation:
client_timeout: 180s
max_short_retry_delay: 7s
max_long_retry_delay: 100s
max_short_retries: 5
max_long_retries: 20
destination_min_retry_interval: 30s
destination_retry_multiplier: 5
destination_max_retry_interval: 12h
```
---
## Caching
Options related to caching.
@ -1358,7 +1464,7 @@ database:
args:
user: synapse_user
password: secretpassword
database: synapse
dbname: synapse
host: localhost
port: 5432
cp_min: 5
@ -1437,7 +1543,7 @@ databases:
args:
user: synapse_user
password: secretpassword
database: synapse_main
dbname: synapse_main
host: localhost
port: 5432
cp_min: 5
@ -1450,7 +1556,7 @@ databases:
args:
user: synapse_user
password: secretpassword
database: synapse_state
dbname: synapse_state
host: localhost
port: 5432
cp_min: 5
@ -1664,6 +1770,19 @@ rc_third_party_invite:
burst_count: 10
```
---
### `rc_media_create`
This option ratelimits creation of MXC URIs via the `/_matrix/media/v1/create`
endpoint based on the account that's creating the media. Defaults to
`per_second: 10`, `burst_count: 50`.
Example configuration:
```yaml
rc_media_create:
per_second: 10
burst_count: 50
```
---
### `rc_federation`
Defines limits on federation requests.
@ -1725,6 +1844,27 @@ Example configuration:
media_store_path: "DATADIR/media_store"
```
---
### `max_pending_media_uploads`
How many *pending media uploads* can a given user have? A pending media upload
is a created MXC URI that (a) is not expired (the `unused_expires_at` timestamp
has not passed) and (b) the media has not yet been uploaded for. Defaults to 5.
Example configuration:
```yaml
max_pending_media_uploads: 5
```
---
### `unused_expiration_time`
How long to wait in milliseconds before expiring created media IDs. Defaults to
"24h"
Example configuration:
```yaml
unused_expiration_time: "1h"
```
---
### `media_storage_providers`
Media storage providers allow media to be stored in different
@ -2781,6 +2921,20 @@ Example configuration:
```yaml
track_appservice_user_ips: true
```
---
### `use_appservice_legacy_authorization`
Whether to send the application service access tokens via the `access_token` query parameter
per older versions of the Matrix specification. Defaults to false. Set to true to enable sending
access tokens via a query parameter.
**Enabling this option is considered insecure and is not recommended. **
Example configuration:
```yaml
use_appservice_legacy_authorization: true
```
---
### `macaroon_secret_key`
@ -2862,7 +3016,7 @@ Normally, the connection to the key server is validated via TLS certificates.
Additional security can be provided by configuring a `verify key`, which
will make synapse check that the response is signed by that key.
This setting supercedes an older setting named `perspectives`. The old format
This setting supersedes an older setting named `perspectives`. The old format
is still supported for backwards-compatibility, but it is deprecated.
`trusted_key_servers` defaults to matrix.org, but using it will generate a
@ -2944,6 +3098,16 @@ enable SAML login. You can either put your entire pysaml config inline using the
option, or you can specify a path to a psyaml config file with the sub-option `config_path`.
This setting has the following sub-options:
* `idp_name`: A user-facing name for this identity provider, which is used to
offer the user a choice of login mechanisms.
* `idp_icon`: An optional icon for this identity provider, which is presented
by clients and Synapse's own IdP picker page. If given, must be an
MXC URI of the format `mxc://<server-name>/<media-id>`. (An easy way to
obtain such an MXC URI is to upload an image to an (unencrypted) room
and then copy the "url" from the source of the event.)
* `idp_brand`: An optional brand for this identity provider, allowing clients
to style the login flow according to the identity provider in question.
See the [spec](https://spec.matrix.org/latest/) for possible options here.
* `sp_config`: the configuration for the pysaml2 Service Provider. See pysaml2 docs for format of config.
Default values will be used for the `entityid` and `service` settings,
so it is not normally necessary to specify them unless you need to
@ -3095,7 +3259,7 @@ Options for each entry include:
* `idp_icon`: An optional icon for this identity provider, which is presented
by clients and Synapse's own IdP picker page. If given, must be an
MXC URI of the format mxc://<server-name>/<media-id>. (An easy way to
MXC URI of the format `mxc://<server-name>/<media-id>`. (An easy way to
obtain such an MXC URI is to upload an image to an (unencrypted) room
and then copy the "url" from the source of the event.)
@ -3113,6 +3277,14 @@ Options for each entry include:
* `client_secret`: oauth2 client secret to use. May be omitted if
`client_secret_jwt_key` is given, or if `client_auth_method` is 'none'.
Must be omitted if `client_secret_path` is specified.
* `client_secret_path`: path to the oauth2 client secret to use. With that
it's not necessary to leak secrets into the config file itself.
Mutually exclusive with `client_secret`. Can be omitted if
`client_secret_jwt_key` is specified.
*Added in Synapse 1.91.0.*
* `client_secret_jwt_key`: Alternative to client_secret: details of a key used
to create a JSON Web Token to be used as an OAuth2 client secret. If
@ -3310,7 +3482,18 @@ Enable Central Authentication Service (CAS) for registration and login.
Has the following sub-options:
* `enabled`: Set this to true to enable authorization against a CAS server.
Defaults to false.
* `idp_name`: A user-facing name for this identity provider, which is used to
offer the user a choice of login mechanisms.
* `idp_icon`: An optional icon for this identity provider, which is presented
by clients and Synapse's own IdP picker page. If given, must be an
MXC URI of the format `mxc://<server-name>/<media-id>`. (An easy way to
obtain such an MXC URI is to upload an image to an (unencrypted) room
and then copy the "url" from the source of the event.)
* `idp_brand`: An optional brand for this identity provider, allowing clients
to style the login flow according to the identity provider in question.
See the [spec](https://spec.matrix.org/latest/) for possible options here.
* `server_url`: The URL of the CAS authorization endpoint.
* `protocol_version`: The CAS protocol version, defaults to none (version 3 is required if you want to use "required_attributes").
* `displayname_attribute`: The attribute of the CAS response to use as the display name.
If no name is given here, no displayname will be set.
* `required_attributes`: It is possible to configure Synapse to only allow logins if CAS attributes
@ -3318,16 +3501,24 @@ Has the following sub-options:
and the values must match the given value. Alternately if the given value
is `None` then any value is allowed (the attribute just must exist).
All of the listed attributes must match for the login to be permitted.
* `enable_registration`: set to 'false' to disable automatic registration of new
users. This allows the CAS SSO flow to be limited to sign in only, rather than
automatically registering users that have a valid SSO login but do not have
a pre-registered account. Defaults to true.
*Added in Synapse 1.93.0.*
Example configuration:
```yaml
cas_config:
enabled: true
server_url: "https://cas-server.com"
protocol_version: 3
displayname_attribute: name
required_attributes:
userGroup: "staff"
department: None
enable_registration: true
```
---
### `sso`
@ -3550,6 +3741,7 @@ This option has the following sub-options:
* `prefer_local_users`: Defines whether to prefer local users in search query results.
If set to true, local users are more likely to appear above remote users when searching the
user directory. Defaults to false.
* `show_locked_users`: Defines whether to show locked users in search query results. Defaults to false.
Example configuration:
```yaml
@ -3557,6 +3749,7 @@ user_directory:
enabled: false
search_all_users: true
prefer_local_users: true
show_locked_users: true
```
---
### `user_consent`
@ -3639,6 +3832,8 @@ Sub-options for this setting include:
* `system_mxid_display_name`: set the display name of the "notices" user
* `system_mxid_avatar_url`: set the avatar for the "notices" user
* `room_name`: set the room name of the server notices room
* `auto_join`: boolean. If true, the user will be automatically joined to the room instead of being invited.
Defaults to false. _Added in Synapse 1.98.0._
Example configuration:
```yaml
@ -3647,6 +3842,7 @@ server_notices:
system_mxid_display_name: "Server Notices"
system_mxid_avatar_url: "mxc://server.com/oumMVlgDnLYFaPVkExemNVVZ"
room_name: "Server Notices"
auto_join: true
```
---
### `enable_room_list_search`
@ -3662,62 +3858,160 @@ enable_room_list_search: false
---
### `alias_creation_rules`
The `alias_creation_rules` option controls who is allowed to create aliases
on this server.
The `alias_creation_rules` option allows server admins to prevent unwanted
alias creation on this server.
The format of this option is a list of rules that contain globs that
match against user_id, room_id and the new alias (fully qualified with
server name). The action in the first rule that matches is taken,
which can currently either be "allow" or "deny".
This setting is an optional list of 0 or more rules. By default, no list is
provided, meaning that all alias creations are permitted.
Missing user_id/room_id/alias fields default to "*".
Otherwise, requests to create aliases are matched against each rule in order.
The first rule that matches decides if the request is allowed or denied. If no
rule matches, the request is denied. In particular, this means that configuring
an empty list of rules will deny every alias creation request.
If no rules match the request is denied. An empty list means no one
can create aliases.
Each rule is a YAML object containing four fields, each of which is an optional string:
Options for the rules include:
* `user_id`: Matches against the creator of the alias. Defaults to "*".
* `alias`: Matches against the alias being created. Defaults to "*".
* `room_id`: Matches against the room ID the alias is being pointed at. Defaults to "*"
* `action`: Whether to "allow" or "deny" the request if the rule matches. Defaults to allow.
* `user_id`: a glob pattern that matches against the creator of the alias.
* `alias`: a glob pattern that matches against the alias being created.
* `room_id`: a glob pattern that matches against the room ID the alias is being pointed at.
* `action`: either `allow` or `deny`. What to do with the request if the rule matches. Defaults to `allow`.
Each of the glob patterns is optional, defaulting to `*` ("match anything").
Note that the patterns match against fully qualified IDs, e.g. against
`@alice:example.com`, `#room:example.com` and `!abcdefghijk:example.com` instead
of `alice`, `room` and `abcedgghijk`.
Example configuration:
```yaml
# No rule list specified. All alias creations are allowed.
# This is the default behaviour.
alias_creation_rules:
- user_id: "bad_user"
alias: "spammy_alias"
room_id: "*"
action: deny
```
```yaml
# A list of one rule which allows everything.
# This has the same effect as the previous example.
alias_creation_rules:
- "action": "allow"
```
```yaml
# An empty list of rules. All alias creations are denied.
alias_creation_rules: []
```
```yaml
# A list of one rule which denies everything.
# This has the same effect as the previous example.
alias_creation_rules:
- "action": "deny"
```
```yaml
# Prevent a specific user from creating aliases.
# Allow other users to create any alias
alias_creation_rules:
- user_id: "@bad_user:example.com"
action: deny
- action: allow
```
```yaml
# Prevent aliases being created which point to a specific room.
alias_creation_rules:
- room_id: "!forbiddenRoom:example.com"
action: deny
- action: allow
```
---
### `room_list_publication_rules`
The `room_list_publication_rules` option controls who can publish and
which rooms can be published in the public room list.
The `room_list_publication_rules` option allows server admins to prevent
unwanted entries from being published in the public room list.
The format of this option is the same as that for
`alias_creation_rules`.
[`alias_creation_rules`](#alias_creation_rules): an optional list of 0 or more
rules. By default, no list is provided, meaning that all rooms may be
published to the room list.
If the room has one or more aliases associated with it, only one of
the aliases needs to match the alias rule. If there are no aliases
then only rules with `alias: *` match.
Otherwise, requests to publish a room are matched against each rule in order.
The first rule that matches decides if the request is allowed or denied. If no
rule matches, the request is denied. In particular, this means that configuring
an empty list of rules will deny every alias creation request.
If no rules match the request is denied. An empty list means no one
can publish rooms.
Each rule is a YAML object containing four fields, each of which is an optional string:
* `user_id`: a glob pattern that matches against the user publishing the room.
* `alias`: a glob pattern that matches against one of published room's aliases.
- If the room has no aliases, the alias match fails unless `alias` is unspecified or `*`.
- If the room has exactly one alias, the alias match succeeds if the `alias` pattern matches that alias.
- If the room has two or more aliases, the alias match succeeds if the pattern matches at least one of the aliases.
* `room_id`: a glob pattern that matches against the room ID of the room being published.
* `action`: either `allow` or `deny`. What to do with the request if the rule matches. Defaults to `allow`.
Each of the glob patterns is optional, defaulting to `*` ("match anything").
Note that the patterns match against fully qualified IDs, e.g. against
`@alice:example.com`, `#room:example.com` and `!abcdefghijk:example.com` instead
of `alice`, `room` and `abcedgghijk`.
Options for the rules include:
* `user_id`: Matches against the creator of the alias. Defaults to "*".
* `alias`: Matches against any current local or canonical aliases associated with the room. Defaults to "*".
* `room_id`: Matches against the room ID being published. Defaults to "*".
* `action`: Whether to "allow" or "deny" the request if the rule matches. Defaults to allow.
Example configuration:
```yaml
# No rule list specified. Anyone may publish any room to the public list.
# This is the default behaviour.
room_list_publication_rules:
- user_id: "*"
alias: "*"
room_id: "*"
action: allow
```
```yaml
# A list of one rule which allows everything.
# This has the same effect as the previous example.
room_list_publication_rules:
- "action": "allow"
```
```yaml
# An empty list of rules. No-one may publish to the room list.
room_list_publication_rules: []
```
```yaml
# A list of one rule which denies everything.
# This has the same effect as the previous example.
room_list_publication_rules:
- "action": "deny"
```
```yaml
# Prevent a specific user from publishing rooms.
# Allow other users to publish anything.
room_list_publication_rules:
- user_id: "@bad_user:example.com"
action: deny
- action: allow
```
```yaml
# Prevent publication of a specific room.
room_list_publication_rules:
- room_id: "!forbiddenRoom:example.com"
action: deny
- action: allow
```
```yaml
# Prevent publication of rooms with at least one alias containing the word "potato".
room_list_publication_rules:
- alias: "#*potato*:example.com"
action: deny
- action: allow
```
---
@ -3754,6 +4048,19 @@ Example configuration:
```yaml
forget_rooms_on_leave: false
```
---
### `exclude_rooms_from_sync`
A list of rooms to exclude from sync responses. This is useful for server
administrators wishing to group users into a room without these users being able
to see it from their client.
By default, no room is excluded.
Example configuration:
```yaml
exclude_rooms_from_sync:
- !foo:example.com
```
---
## Opentracing
@ -3904,13 +4211,14 @@ federation_sender_instances:
---
### `instance_map`
When using workers this should be a map from [`worker_name`](#worker_name) to the
HTTP replication listener of the worker, if configured, and to the main process.
Each worker declared under [`stream_writers`](../../workers.md#stream-writers) needs
a HTTP replication listener, and that listener should be included in the `instance_map`.
The main process also needs an entry on the `instance_map`, and it should be listed under
`main` **if even one other worker exists**. Ensure the port matches with what is declared
inside the `listener` block for a `replication` listener.
When using workers this should be a map from [`worker_name`](#worker_name) to the HTTP
replication listener of the worker, if configured, and to the main process. Each worker
declared under [`stream_writers`](../../workers.md#stream-writers) and
[`outbound_federation_restricted_to`](#outbound_federation_restricted_to) needs a HTTP
replication listener, and that listener should be included in the `instance_map`. The
main process also needs an entry on the `instance_map`, and it should be listed under
`main` **if even one other worker exists**. Ensure the port matches with what is
declared inside the `listener` block for a `replication` listener.
Example configuration:
@ -3923,6 +4231,14 @@ instance_map:
host: localhost
port: 8034
```
Example configuration(#2, for UNIX sockets):
```yaml
instance_map:
main:
path: /run/synapse/main_replication.sock
worker1:
path: /run/synapse/worker1_replication.sock
```
---
### `stream_writers`
@ -3939,6 +4255,27 @@ stream_writers:
events: worker1
typing: worker1
```
---
### `outbound_federation_restricted_to`
When using workers, you can restrict outbound federation traffic to only go through a
specific subset of workers. Any worker specified here must also be in the
[`instance_map`](#instance_map).
[`worker_replication_secret`](#worker_replication_secret) must also be configured to
authorize inter-worker communication.
```yaml
outbound_federation_restricted_to:
- federation_sender1
- federation_sender2
```
Also see the [worker
documentation](../../workers.md#restrict-outbound-federation-traffic-to-a-specific-set-of-workers)
for more info.
_Added in Synapse 1.89.0._
---
### `run_background_tasks_on`
@ -4064,51 +4401,6 @@ Example configuration:
worker_name: generic_worker1
```
---
### `worker_replication_host`
*Deprecated as of version 1.84.0. Place `host` under `main` entry on the [`instance_map`](#instance_map) in your shared yaml configuration instead.*
The HTTP replication endpoint that it should talk to on the main Synapse process.
The main Synapse process defines this with a `replication` resource in
[`listeners` option](#listeners).
Example configuration:
```yaml
worker_replication_host: 127.0.0.1
```
---
### `worker_replication_http_port`
*Deprecated as of version 1.84.0. Place `port` under `main` entry on the [`instance_map`](#instance_map) in your shared yaml configuration instead.*
The HTTP replication port that it should talk to on the main Synapse process.
The main Synapse process defines this with a `replication` resource in
[`listeners` option](#listeners).
Example configuration:
```yaml
worker_replication_http_port: 9093
```
---
### `worker_replication_http_tls`
*Deprecated as of version 1.84.0. Place `tls` under `main` entry on the [`instance_map`](#instance_map) in your shared yaml configuration instead.*
Whether TLS should be used for talking to the HTTP replication port on the main
Synapse process.
The main Synapse process defines this with the `tls` option on its [listener](#listeners) that
has the `replication` resource enabled.
**Please note:** by default, it is not safe to expose replication ports to the
public Internet, even with TLS enabled.
See [`worker_replication_secret`](#worker_replication_secret).
Defaults to `false`.
*Added in Synapse 1.72.0.*
Example configuration:
```yaml
worker_replication_http_tls: true
```
---
### `worker_listeners`
A worker can handle HTTP requests. To do so, a `worker_listeners` option
@ -4127,6 +4419,18 @@ worker_listeners:
resources:
- names: [client, federation]
```
Example configuration(#2, using UNIX sockets with a `replication` listener):
```yaml
worker_listeners:
- type: http
path: /run/synapse/worker_replication.sock
resources:
- names: [replication]
- type: http
path: /run/synapse/worker_public.sock
resources:
- names: [client, federation]
```
---
### `worker_manhole`

View File

@ -1,49 +1,133 @@
User Directory API Implementation
=================================
# User Directory API Implementation
The user directory is currently maintained based on the 'visible' users
on this particular server - i.e. ones which your account shares a room with, or
who are present in a publicly viewable room present on the server.
The user directory is maintained based on users that are 'visible' to the homeserver -
i.e. ones which are local to the server and ones which any local user shares a
room with.
The directory info is stored in various tables, which can (typically after
DB corruption) get stale or out of sync. If this happens, for now the
The directory info is stored in various tables, which can sometimes get out of
sync (although this is considered a bug). If this happens, for now the
solution to fix it is to use the [admin API](usage/administration/admin_api/background_updates.md#run)
and execute the job `regenerate_directory`. This should then start a background task to
flush the current tables and regenerate the directory.
flush the current tables and regenerate the directory. Depending on the size
of your homeserver (number of users and rooms) this can take a while.
Data model
----------
## Data model
There are five relevant tables that collectively form the "user directory".
Three of them track a master list of all the users we could search for.
The last two (collectively called the "search tables") track who can
see who.
Three of them track a list of all known users. The last two (collectively called
the "search tables") track which users are visible to each other.
From all of these tables we exclude three types of local user:
- support users
- appservice users
- deactivated users
* `user_directory`. This contains the user_id, display name and avatar we'll
return when you search the directory.
- Because there's only one directory entry per user, it's important that we only
ever put publicly visible names here. Otherwise we might leak a private
- support users
- appservice users
- deactivated users
A description of each table follows:
* `user_directory`. This contains the user ID, display name and avatar of each user.
- Because there is only one directory entry per user, it is important that it
only contain publicly visible information. Otherwise, this will leak the
nickname or avatar used in a private room.
- Indexed on rooms. Indexed on users.
* `user_directory_search`. To be joined to `user_directory`. It contains an extra
column that enables full text search based on user ids and display names.
Different schemas for SQLite and Postgres with different code paths to match.
column that enables full text search based on user IDs and display names.
Different schemas for SQLite and Postgres are used.
- Indexed on the full text search data. Indexed on users.
* `user_directory_stream_pos`. When the initial background update to populate
the directory is complete, we record a stream position here. This indicates
that synapse should now listen for room changes and incrementally update
the directory where necessary.
the directory where necessary. (See [stream positions](development/synapse_architecture/streams.html).)
* `users_in_public_rooms`. Contains associations between users and the public rooms they're in.
Used to determine which users are in public rooms and should be publicly visible in the directory.
* `users_in_public_rooms`. Contains associations between users and the public
rooms they're in. Used to determine which users are in public rooms and should
be publicly visible in the directory. Both local and remote users are tracked.
* `users_who_share_private_rooms`. Rows are triples `(L, M, room id)` where `L`
is a local user and `M` is a local or remote user. `L` and `M` should be
different, but this isn't enforced by a constraint.
Note that if two local users share a room then there will be two entries:
`(user1, user2, !room_id)` and `(user2, user1, !room_id)`.
## Configuration options
The exact way user search works can be tweaked via some server-level
[configuration options](usage/configuration/config_documentation.md#user_directory).
The information is not repeated here, but the options are mentioned below.
## Search algorithm
If `search_all_users` is `false`, then results are limited to users who:
1. Are found in the `users_in_public_rooms` table, or
2. Are found in the `users_who_share_private_rooms` where `L` is the requesting
user and `M` is the search result.
Otherwise, if `search_all_users` is `true`, no such limits are placed and all
users known to the server (matching the search query) will be returned.
By default, locked users are not returned. If `show_locked_users` is `true` then
no filtering on the locked status of a user is done.
The user provided search term is lowercased and normalized using [NFKC](https://en.wikipedia.org/wiki/Unicode_equivalence#Normalization),
this treats the string as case-insensitive, canonicalizes different forms of the
same text, and maps some "roughly equivalent" characters together.
The search term is then split into words:
* If [ICU](https://en.wikipedia.org/wiki/International_Components_for_Unicode) is
available, then the system's [default locale](https://unicode-org.github.io/icu/userguide/locale/#default-locales)
will be used to break the search term into words. (See the
[installation instructions](setup/installation.md) for how to install ICU.)
* If unavailable, then runs of ASCII characters, numbers, underscores, and hyphens
are considered words.
The queries for PostgreSQL and SQLite are detailed below, by their overall goal
is to find matching users, preferring users who are "real" (e.g. not bots,
not deactivated). It is assumed that real users will have an display name and
avatar set.
### PostgreSQL
The above words are then transformed into two queries:
1. "exact" which matches the parsed words exactly (using [`to_tsquery`](https://www.postgresql.org/docs/current/textsearch-controls.html#TEXTSEARCH-PARSING-QUERIES));
2. "prefix" which matches the parsed words as prefixes (using `to_tsquery`).
Results are composed of all rows in the `user_directory_search` table whose information
matches one (or both) of these queries. Results are ordered by calculating a weighted
score for each result, higher scores are returned first:
* 4x if a user ID exists.
* 1.2x if the user has a display name set.
* 1.2x if the user has an avatar set.
* 0x-3x by the full text search results using the [`ts_rank_cd` function](https://www.postgresql.org/docs/current/textsearch-controls.html#TEXTSEARCH-RANKING)
against the "exact" search query; this has four variables with the following weightings:
* `D`: 0.1 for the user ID's domain
* `C`: 0.1 for unused
* `B`: 0.9 for the user's display name (or an empty string if it is not set)
* `A`: 0.1 for the user ID's localpart
* 0x-1x by the full text search results using the `ts_rank_cd` function against the
"prefix" search query. (Using the same weightings as above.)
* If `prefer_local_users` is `true`, then 2x if the user is local to the homeserver.
Note that `ts_rank_cd` returns a weight between 0 and 1. The initial weighting of
all results is 1.
### SQLite
Results are composed of all rows in the `user_directory_search` whose information
matches the query. Results are ordered by the following information, with each
subsequent column used as a tiebreaker, for each result:
1. By the [`rank`](https://www.sqlite.org/windowfunctions.html#built_in_window_functions)
of the full text search results using the [`matchinfo` function](https://www.sqlite.org/fts3.html#matchinfo). Higher
ranks are returned first.
2. If `prefer_local_users` is `true`, then users local to the homeserver are
returned first.
3. Users with a display name set are returned first.
4. Users with an avatar set are returned first.

View File

@ -24,6 +24,11 @@ Finally, we also stylise the chapter titles in the left sidebar by indenting the
slightly so that they are more visually distinguishable from the section headers
(the bold titles). This is done through the `indent-section-headers.css` file.
In addition to these modifications, we have added a version picker to the documentation.
Users can switch between documentations for different versions of Synapse.
This functionality was implemented through the `version-picker.js` and
`version-picker.css` files.
More information can be found in mdbook's official documentation for
[injecting page JS/CSS](https://rust-lang.github.io/mdBook/format/config.html)
and

View File

@ -131,6 +131,18 @@
<i class="fa fa-search"></i>
</button>
{{/if}}
<div class="version-picker">
<div class="dropdown">
<div class="select">
<span></span>
<i class="fa fa-chevron-down"></i>
</div>
<input type="hidden" name="version">
<ul class="dropdown-menu">
<!-- Versions will be added dynamically in version-picker.js -->
</ul>
</div>
</div>
</div>
<h1 class="menu-title">{{ book_title }}</h1>
@ -309,4 +321,4 @@
{{/if}}
</body>
</html>
</html>

View File

@ -0,0 +1,78 @@
.version-picker {
display: flex;
align-items: center;
}
.version-picker .dropdown {
width: 130px;
max-height: 29px;
margin-left: 10px;
display: inline-block;
border-radius: 4px;
border: 1px solid var(--theme-popup-border);
position: relative;
font-size: 13px;
color: var(--fg);
height: 100%;
text-align: left;
}
.version-picker .dropdown .select {
cursor: pointer;
display: block;
padding: 5px 2px 5px 15px;
}
.version-picker .dropdown .select > i {
font-size: 10px;
color: var(--fg);
cursor: pointer;
float: right;
line-height: 20px !important;
}
.version-picker .dropdown:hover {
border: 1px solid var(--theme-popup-border);
}
.version-picker .dropdown:active {
background-color: var(--theme-popup-bg);
}
.version-picker .dropdown.active:hover,
.version-picker .dropdown.active {
border: 1px solid var(--theme-popup-border);
border-radius: 2px 2px 0 0;
background-color: var(--theme-popup-bg);
}
.version-picker .dropdown.active .select > i {
transform: rotate(-180deg);
}
.version-picker .dropdown .dropdown-menu {
position: absolute;
background-color: var(--theme-popup-bg);
width: 100%;
left: -1px;
right: 1px;
margin-top: 1px;
border: 1px solid var(--theme-popup-border);
border-radius: 0 0 4px 4px;
overflow: hidden;
display: none;
max-height: 300px;
overflow-y: auto;
z-index: 9;
}
.version-picker .dropdown .dropdown-menu li {
font-size: 12px;
padding: 6px 20px;
cursor: pointer;
}
.version-picker .dropdown .dropdown-menu {
padding: 0;
list-style: none;
}
.version-picker .dropdown .dropdown-menu li:hover {
background-color: var(--theme-hover);
}
.version-picker .dropdown .dropdown-menu li.active::before {
display: inline-block;
content: "✓";
margin-inline-start: -14px;
width: 14px;
}

View File

@ -0,0 +1,127 @@
const dropdown = document.querySelector('.version-picker .dropdown');
const dropdownMenu = dropdown.querySelector('.dropdown-menu');
fetchVersions(dropdown, dropdownMenu).then(() => {
initializeVersionDropdown(dropdown, dropdownMenu);
});
/**
* Initialize the dropdown functionality for version selection.
*
* @param {Element} dropdown - The dropdown element.
* @param {Element} dropdownMenu - The dropdown menu element.
*/
function initializeVersionDropdown(dropdown, dropdownMenu) {
// Toggle the dropdown menu on click
dropdown.addEventListener('click', function () {
this.setAttribute('tabindex', 1);
this.classList.toggle('active');
dropdownMenu.style.display = (dropdownMenu.style.display === 'block') ? 'none' : 'block';
});
// Remove the 'active' class and hide the dropdown menu on focusout
dropdown.addEventListener('focusout', function () {
this.classList.remove('active');
dropdownMenu.style.display = 'none';
});
// Handle item selection within the dropdown menu
const dropdownMenuItems = dropdownMenu.querySelectorAll('li');
dropdownMenuItems.forEach(function (item) {
item.addEventListener('click', function () {
dropdownMenuItems.forEach(function (item) {
item.classList.remove('active');
});
this.classList.add('active');
dropdown.querySelector('span').textContent = this.textContent;
dropdown.querySelector('input').value = this.getAttribute('id');
window.location.href = changeVersion(window.location.href, this.textContent);
});
});
};
/**
* This function fetches the available versions from a GitHub repository
* and inserts them into the version picker.
*
* @param {Element} dropdown - The dropdown element.
* @param {Element} dropdownMenu - The dropdown menu element.
* @returns {Promise<Array<string>>} A promise that resolves with an array of available versions.
*/
function fetchVersions(dropdown, dropdownMenu) {
return new Promise((resolve, reject) => {
window.addEventListener("load", () => {
fetch("https://api.github.com/repos/matrix-org/synapse/git/trees/gh-pages", {
cache: "force-cache",
}).then(res =>
res.json()
).then(resObject => {
const excluded = ['dev-docs', 'v1.91.0', 'v1.80.0', 'v1.69.0'];
const tree = resObject.tree.filter(item => item.type === "tree" && !excluded.includes(item.path));
const versions = tree.map(item => item.path).sort(sortVersions);
// Create a list of <li> items for versions
versions.forEach((version) => {
const li = document.createElement("li");
li.textContent = version;
li.id = version;
if (window.SYNAPSE_VERSION === version) {
li.classList.add('active');
dropdown.querySelector('span').textContent = version;
dropdown.querySelector('input').value = version;
}
dropdownMenu.appendChild(li);
});
resolve(versions);
}).catch(ex => {
console.error("Failed to fetch version data", ex);
reject(ex);
})
});
});
}
/**
* Custom sorting function to sort an array of version strings.
*
* @param {string} a - The first version string to compare.
* @param {string} b - The second version string to compare.
* @returns {number} - A negative number if a should come before b, a positive number if b should come before a, or 0 if they are equal.
*/
function sortVersions(a, b) {
// Put 'develop' and 'latest' at the top
if (a === 'develop' || a === 'latest') return -1;
if (b === 'develop' || b === 'latest') return 1;
const versionA = (a.match(/v\d+(\.\d+)+/) || [])[0];
const versionB = (b.match(/v\d+(\.\d+)+/) || [])[0];
return versionB.localeCompare(versionA);
}
/**
* Change the version in a URL path.
*
* @param {string} url - The original URL to be modified.
* @param {string} newVersion - The new version to replace the existing version in the URL.
* @returns {string} The updated URL with the new version.
*/
function changeVersion(url, newVersion) {
const parsedURL = new URL(url);
const pathSegments = parsedURL.pathname.split('/');
// Modify the version
pathSegments[2] = newVersion;
// Reconstruct the URL
parsedURL.pathname = pathSegments.join('/');
return parsedURL.href;
}

View File

@ -0,0 +1 @@
window.SYNAPSE_VERSION = 'v1.98';

View File

@ -95,9 +95,12 @@ for the main process
* Secondly, you need to enable
[redis-based replication](usage/configuration/config_documentation.md#redis)
* You will need to add an [`instance_map`](usage/configuration/config_documentation.md#instance_map)
with the `main` process defined, as well as the relevant connection information from
it's HTTP `replication` listener (defined in step 1 above). Note that the `host` defined
is the address the worker needs to look for the `main` process at, not necessarily the same address that is bound to.
with the `main` process defined, as well as the relevant connection information from
it's HTTP `replication` listener (defined in step 1 above).
* Note that the `host` defined is the address the worker needs to look for the `main`
process at, not necessarily the same address that is bound to.
* If you are using Unix sockets for the `replication` resource, make sure to
use a `path` to the socket file instead of a `port`.
* Optionally, a [shared secret](usage/configuration/config_documentation.md#worker_replication_secret)
can be used to authenticate HTTP traffic between workers. For example:
@ -145,9 +148,6 @@ In the config file for each worker, you must specify:
with an `http` listener.
* **Synapse 1.72 and older:** if handling the `^/_matrix/client/v3/keys/upload` endpoint, the HTTP URI for
the main process (`worker_main_http_uri`). This config option is no longer required and is ignored when running Synapse 1.73 and newer.
* **Synapse 1.83 and older:** The HTTP replication endpoint that the worker should talk to on the main synapse process
([`worker_replication_host`](usage/configuration/config_documentation.md#worker_replication_host) and
[`worker_replication_http_port`](usage/configuration/config_documentation.md#worker_replication_http_port)). If using Synapse 1.84 and newer, these are not needed if `main` is defined on the [shared configuration](#shared-configuration) `instance_map`
For example:
@ -177,11 +177,11 @@ The following applies to Synapse installations that have been installed from sou
You can start the main Synapse process with Poetry by running the following command:
```console
poetry run synapse_homeserver -c [your homeserver.yaml]
poetry run synapse_homeserver --config-file [your homeserver.yaml]
```
For worker setups, you can run the following command
```console
poetry run synapse_worker -c [your worker.yaml]
poetry run synapse_worker --config-file [your homeserver.yaml] --config-file [your worker.yaml]
```
## Available worker applications
@ -232,7 +232,6 @@ information.
^/_matrix/client/v1/rooms/.*/hierarchy$
^/_matrix/client/(v1|unstable)/rooms/.*/relations/
^/_matrix/client/v1/rooms/.*/threads$
^/_matrix/client/unstable/org.matrix.msc2716/rooms/.*/batch_send$
^/_matrix/client/unstable/im.nheko.summary/rooms/.*/summary$
^/_matrix/client/(r0|v3|unstable)/account/3pid$
^/_matrix/client/(r0|v3|unstable)/account/whoami$
@ -247,6 +246,7 @@ information.
^/_matrix/client/(r0|v3|unstable)/user/.*/filter(/|$)
^/_matrix/client/(api/v1|r0|v3|unstable)/directory/room/.*$
^/_matrix/client/(r0|v3|unstable)/capabilities$
^/_matrix/client/(r0|v3|unstable)/notifications$
# Encryption requests
^/_matrix/client/(r0|v3|unstable)/keys/query$
@ -532,6 +532,30 @@ the stream writer for the `presence` stream:
^/_matrix/client/(api/v1|r0|v3|unstable)/presence/
#### Restrict outbound federation traffic to a specific set of workers
The
[`outbound_federation_restricted_to`](usage/configuration/config_documentation.md#outbound_federation_restricted_to)
configuration is useful to make sure outbound federation traffic only goes through a
specified subset of workers. This allows you to set more strict access controls (like a
firewall) for all workers and only allow the `federation_sender`'s to contact the
outside world.
```yaml
instance_map:
main:
host: localhost
port: 8030
federation_sender1:
host: localhost
port: 8034
outbound_federation_restricted_to:
- federation_sender1
worker_replication_secret: "secret_secret"
```
#### Background tasks
There is also support for moving background tasks to a separate

View File

@ -8,41 +8,20 @@
"pre-commit-hooks": "pre-commit-hooks"
},
"locked": {
"lastModified": 1683102061,
"narHash": "sha256-kOphT6V0uQUlFNBP3GBjs7DAU7fyZGGqCs9ue1gNY6E=",
"lastModified": 1688058187,
"narHash": "sha256-ipDcc7qrucpJ0+0eYNlwnE+ISTcq4m03qW+CWUshRXI=",
"owner": "cachix",
"repo": "devenv",
"rev": "ff1f29e41756553174d596cafe3a9fa77595100b",
"rev": "c8778e3dc30eb9043e218aaa3861d42d4992de77",
"type": "github"
},
"original": {
"owner": "cachix",
"ref": "main",
"ref": "v0.6.3",
"repo": "devenv",
"type": "github"
}
},
"fenix": {
"inputs": {
"nixpkgs": [
"nixpkgs"
],
"rust-analyzer-src": "rust-analyzer-src"
},
"locked": {
"lastModified": 1682490133,
"narHash": "sha256-tR2Qx0uuk97WySpSSk4rGS/oH7xb5LykbjATcw1vw1I=",
"owner": "nix-community",
"repo": "fenix",
"rev": "4e9412753ab75ef0e038a5fe54a062fb44c27c6a",
"type": "github"
},
"original": {
"owner": "nix-community",
"repo": "fenix",
"type": "github"
}
},
"flake-compat": {
"flake": false,
"locked": {
@ -60,12 +39,33 @@
}
},
"flake-utils": {
"inputs": {
"systems": "systems"
},
"locked": {
"lastModified": 1667395993,
"narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=",
"lastModified": 1685518550,
"narHash": "sha256-o2d0KcvaXzTrPRIo0kOLV0/QXHhDQ5DTi+OxcjO8xqY=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f",
"rev": "a1720a10a6cfe8234c0e93907ffe81be440f4cef",
"type": "github"
},
"original": {
"owner": "numtide",
"repo": "flake-utils",
"type": "github"
}
},
"flake-utils_2": {
"inputs": {
"systems": "systems_2"
},
"locked": {
"lastModified": 1681202837,
"narHash": "sha256-H+Rh19JDwRtpVPAWp64F+rlEtxUWBAQW28eAi3SRSzg=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "cfacdce06f30d2b68473a46042957675eebb3401",
"type": "github"
},
"original": {
@ -170,27 +170,27 @@
},
"nixpkgs-stable": {
"locked": {
"lastModified": 1673800717,
"narHash": "sha256-SFHraUqLSu5cC6IxTprex/nTsI81ZQAtDvlBvGDWfnA=",
"lastModified": 1685801374,
"narHash": "sha256-otaSUoFEMM+LjBI1XL/xGB5ao6IwnZOXc47qhIgJe8U=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "2f9fd351ec37f5d479556cd48be4ca340da59b8f",
"rev": "c37ca420157f4abc31e26f436c1145f8951ff373",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "nixos-22.11",
"ref": "nixos-23.05",
"repo": "nixpkgs",
"type": "github"
}
},
"nixpkgs_2": {
"locked": {
"lastModified": 1682519441,
"narHash": "sha256-Vsq/8NOtvW1AoC6shCBxRxZyMQ+LhvPuJT6ltbzuv+Y=",
"lastModified": 1690535733,
"narHash": "sha256-WgjUPscQOw3cB8yySDGlyzo6cZNihnRzUwE9kadv/5I=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "7a32a141db568abde9bc389845949dc2a454dfd3",
"rev": "8cacc05fbfffeaab910e8c2c9e2a7c6b32ce881a",
"type": "github"
},
"original": {
@ -200,6 +200,22 @@
"type": "github"
}
},
"nixpkgs_3": {
"locked": {
"lastModified": 1681358109,
"narHash": "sha256-eKyxW4OohHQx9Urxi7TQlFBTDWII+F+x2hklDOQPB50=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "96ba1c52e54e74c3197f4d43026b3f3d92e83ff9",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "nixpkgs-unstable",
"repo": "nixpkgs",
"type": "github"
}
},
"pre-commit-hooks": {
"inputs": {
"flake-compat": [
@ -215,11 +231,11 @@
"nixpkgs-stable": "nixpkgs-stable"
},
"locked": {
"lastModified": 1678376203,
"narHash": "sha256-3tyYGyC8h7fBwncLZy5nCUjTJPrHbmNwp47LlNLOHSM=",
"lastModified": 1688056373,
"narHash": "sha256-2+SDlNRTKsgo3LBRiMUcoEUb6sDViRNQhzJquZ4koOI=",
"owner": "cachix",
"repo": "pre-commit-hooks.nix",
"rev": "1a20b9708962096ec2481eeb2ddca29ed747770a",
"rev": "5843cf069272d92b60c3ed9e55b7a8989c01d4c7",
"type": "github"
},
"original": {
@ -231,25 +247,27 @@
"root": {
"inputs": {
"devenv": "devenv",
"fenix": "fenix",
"nixpkgs": "nixpkgs_2",
"systems": "systems"
"rust-overlay": "rust-overlay",
"systems": "systems_3"
}
},
"rust-analyzer-src": {
"flake": false,
"rust-overlay": {
"inputs": {
"flake-utils": "flake-utils_2",
"nixpkgs": "nixpkgs_3"
},
"locked": {
"lastModified": 1682426789,
"narHash": "sha256-UqnLmJESRZE0tTEaGbRAw05Hm19TWIPA+R3meqi5I4w=",
"owner": "rust-lang",
"repo": "rust-analyzer",
"rev": "943d2a8a1ca15e8b28a1f51f5a5c135e3728da04",
"lastModified": 1693966243,
"narHash": "sha256-a2CA1aMIPE67JWSVIGoGtD3EGlFdK9+OlJQs0FOWCKY=",
"owner": "oxalica",
"repo": "rust-overlay",
"rev": "a8b4bb4cbb744baaabc3e69099f352f99164e2c1",
"type": "github"
},
"original": {
"owner": "rust-lang",
"ref": "nightly",
"repo": "rust-analyzer",
"owner": "oxalica",
"repo": "rust-overlay",
"type": "github"
}
},
@ -267,6 +285,36 @@
"repo": "default",
"type": "github"
}
},
"systems_2": {
"locked": {
"lastModified": 1681028828,
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
"owner": "nix-systems",
"repo": "default",
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
"type": "github"
},
"original": {
"owner": "nix-systems",
"repo": "default",
"type": "github"
}
},
"systems_3": {
"locked": {
"lastModified": 1681028828,
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
"owner": "nix-systems",
"repo": "default",
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
"type": "github"
},
"original": {
"owner": "nix-systems",
"repo": "default",
"type": "github"
}
}
},
"root": "root",

View File

@ -39,27 +39,27 @@
{
inputs = {
# Use the master/unstable branch of nixpkgs. The latest stable, 22.11,
# does not contain 'perl536Packages.NetAsyncHTTP', needed by Sytest.
# Use the master/unstable branch of nixpkgs. Used to fetch the latest
# available versions of packages.
nixpkgs.url = "github:NixOS/nixpkgs/master";
# Output a development shell for x86_64/aarch64 Linux/Darwin (MacOS).
systems.url = "github:nix-systems/default";
# A development environment manager built on Nix. See https://devenv.sh.
devenv.url = "github:cachix/devenv/main";
# Rust toolchains and rust-analyzer nightly.
fenix = {
url = "github:nix-community/fenix";
inputs.nixpkgs.follows = "nixpkgs";
};
devenv.url = "github:cachix/devenv/v0.6.3";
# Rust toolchain.
rust-overlay.url = "github:oxalica/rust-overlay";
};
outputs = { self, nixpkgs, devenv, systems, ... } @ inputs:
outputs = { self, nixpkgs, devenv, systems, rust-overlay, ... } @ inputs:
let
forEachSystem = nixpkgs.lib.genAttrs (import systems);
in {
devShells = forEachSystem (system:
let
pkgs = nixpkgs.legacyPackages.${system};
overlays = [ (import rust-overlay) ];
pkgs = import nixpkgs {
inherit system overlays;
};
in {
# Everything is configured via devenv - a Nix module for creating declarative
# developer environments. See https://devenv.sh/reference/options/ for a list
@ -76,6 +76,25 @@
# Configure packages to install.
# Search for package names at https://search.nixos.org/packages?channel=unstable
packages = with pkgs; [
# The rust toolchain and related tools.
# This will install the "default" profile of rust components.
# https://rust-lang.github.io/rustup/concepts/profiles.html
#
# NOTE: We currently need to set the Rust version unnecessarily high
# in order to work around https://github.com/matrix-org/synapse/issues/15939
(rust-bin.stable."1.71.1".default.override {
# Additionally install the "rust-src" extension to allow diving into the
# Rust source code in an IDE (rust-analyzer will also make use of it).
extensions = [ "rust-src" ];
})
# The rust-analyzer language server implementation.
rust-analyzer
# GCC includes a linker; needed for building `ruff`
gcc
# Needed for building `ruff`
gnumake
# Native dependencies for running Synapse.
icu
libffi
@ -124,12 +143,11 @@
# Install dependencies for the additional programming languages
# involved with Synapse development.
#
# * Rust is used for developing and running Synapse.
# * Golang is needed to run the Complement test suite.
# * Perl is needed to run the SyTest test suite.
# * Rust is used for developing and running Synapse.
# It is installed manually with `packages` above.
languages.go.enable = true;
languages.rust.enable = true;
languages.rust.version = "stable";
languages.perl.enable = true;
# Postgres is needed to run Synapse with postgres support and
@ -178,7 +196,7 @@
EOF
'';
# Start synapse when `devenv up` is run.
processes.synapse.exec = "poetry run python -m synapse.app.homeserver -c homeserver.yaml --config-directory homeserver-config-overrides.d";
processes.synapse.exec = "poetry run python -m synapse.app.homeserver -c homeserver.yaml -c homeserver-config-overrides.d";
# Define the perl modules we require to run SyTest.
#
@ -223,6 +241,19 @@
URI
YAMLLibYAML
]}";
# Clear the LD_LIBRARY_PATH environment variable on shell init.
#
# By default, devenv will set LD_LIBRARY_PATH to point to .devenv/profile/lib. This causes
# issues when we include `gcc` as a dependency to build C libraries, as the version of glibc
# that the development environment's cc compiler uses may differ from that of the system.
#
# When LD_LIBRARY_PATH is set, system tools will attempt to use the development environment's
# libraries. Which, when built against a different glibc version lead, to "version 'GLIBC_X.YY'
# not found" errors.
enterShell = ''
unset LD_LIBRARY_PATH
'';
}
];
};

View File

@ -23,7 +23,6 @@ warn_unused_ignores = True
# warn_return_any = True
# no_implicit_reexport = True
strict_equality = True
strict_concatenate = True
# Run mypy type checking with the minimum supported Python version to catch new usage
# that isn't backwards-compatible (types, overloads, etc).
@ -33,18 +32,26 @@ files =
docker/,
scripts-dev/,
synapse/,
synmark/,
tests/,
build_rust.py
[mypy-synapse.metrics._reactor_metrics]
# This module imports select.epoll. That exists on Linux, but doesn't on macOS.
# See https://github.com/matrix-org/synapse/pull/11771.
# This module pokes at the internals of OS-specific classes, to appease mypy
# on different systems we add additional ignores.
warn_unused_ignores = False
[mypy-synapse.util.caches.treecache]
disallow_untyped_defs = False
disallow_incomplete_defs = False
[mypy-synapse.util.manhole]
# This module imports something from Twisted which has a bad annotation in Twisted trunk,
# but is unannotated in Twisted's latest release. We want to type-ignore the problem
# in the twisted trunk job, even though it has no effect on normal mypy runs.
warn_unused_ignores = False
;; Dependencies without annotations
;; Before ignoring a module, check to see if type stubs are available.
;; The `typeshed` project maintains stubs here:
@ -74,24 +81,18 @@ ignore_missing_imports = True
[mypy-pympler.*]
ignore_missing_imports = True
[mypy-pyperf.*]
ignore_missing_imports = True
[mypy-rust_python_jaeger_reporter.*]
ignore_missing_imports = True
[mypy-saml2.*]
ignore_missing_imports = True
[mypy-service_identity.*]
ignore_missing_imports = True
[mypy-srvlookup.*]
ignore_missing_imports = True
# https://github.com/twisted/treq/pull/366
[mypy-treq.*]
ignore_missing_imports = True
[mypy-incremental.*]
ignore_missing_imports = True
[mypy-setuptools_rust.*]
ignore_missing_imports = True

1805
poetry.lock generated

File diff suppressed because it is too large Load Diff

View File

@ -35,7 +35,7 @@
showcontent = true
[tool.black]
target-version = ['py37', 'py38', 'py39', 'py310']
target-version = ['py38', 'py39', 'py310', 'py311']
# black ignores everything in .gitignore by default, see
# https://black.readthedocs.io/en/stable/usage_and_configuration/file_collection_and_discovery.html#gitignore
# Use `extend-exclude` if you want to exclude something in addition to this.
@ -43,33 +43,39 @@ target-version = ['py37', 'py38', 'py39', 'py310']
[tool.ruff]
line-length = 88
# See https://github.com/charliermarsh/ruff/#pycodestyle
# See https://beta.ruff.rs/docs/rules/#error-e
# for error codes. The ones we ignore are:
# E731: do not assign a lambda expression, use a def
# E501: Line too long (black enforces this for us)
# E731: do not assign a lambda expression, use a def
#
# flake8-bugbear compatible checks. Its error codes are described at
# https://github.com/charliermarsh/ruff/#flake8-bugbear
# B019: Use of functools.lru_cache or functools.cache on methods can lead to memory leaks
# https://beta.ruff.rs/docs/rules/#flake8-bugbear-b
# B023: Functions defined inside a loop must not use variables redefined in the loop
# B024: Abstract base class with no abstract method.
ignore = [
"B019",
"B023",
"B024",
"E501",
"E731",
]
select = [
# pycodestyle checks.
# pycodestyle
"E",
"W",
# pyflakes checks.
# pyflakes
"F",
# flake8-bugbear checks.
# flake8-bugbear
"B0",
# flake8-comprehensions checks.
# flake8-comprehensions
"C4",
# flake8-2020
"YTT",
# flake8-slots
"SLOT",
# flake8-debugger
"T10",
# flake8-pie
"PIE",
# flake8-executable
"EXE",
]
[tool.isort]
@ -86,10 +92,11 @@ skip_gitignore = true
[tool.maturin]
manifest-path = "rust/Cargo.toml"
module-name = "synapse.synapse_rust"
[tool.poetry]
name = "matrix-synapse"
version = "1.86.0rc2"
version = "1.98.0"
description = "Homeserver for the Matrix decentralised comms protocol"
authors = ["Matrix.org Team and Contributors <packages@matrix.org>"]
license = "Apache-2.0"
@ -147,7 +154,7 @@ synapse_review_recent_signups = "synapse._scripts.review_recent_signups:main"
update_synapse_database = "synapse._scripts.update_synapse_database:main"
[tool.poetry.dependencies]
python = "^3.7.1"
python = "^3.8.0"
# Mandatory Dependencies
# ----------------------
@ -174,7 +181,9 @@ PyYAML = ">=3.13"
pyasn1 = ">=0.1.9"
pyasn1-modules = ">=0.0.7"
bcrypt = ">=3.1.7"
Pillow = ">=5.4.0"
# 10.0.1 minimum is mandatory here because of libwebp CVE-2023-4863.
# Packagers that already took care of libwebp can lower that down to 5.4.0.
Pillow = ">=10.0.1"
# We use SortedDict.peekitem(), which was added in sortedcontainers 1.5.2.
sortedcontainers = ">=1.5.2"
pymacaroons = ">=0.13.0"
@ -183,7 +192,7 @@ phonenumbers = ">=8.2.0"
# we use GaugeHistogramMetric, which was added in prom-client 0.4.0.
prometheus-client = ">=0.4.0"
# we use `order`, which arrived in attrs 19.2.0.
# Note: 21.1.0 broke `/sync`, see #9936
# Note: 21.1.0 broke `/sync`, see https://github.com/matrix-org/synapse/issues/9936
attrs = ">=19.2.0,!=21.1.0"
netaddr = ">=0.7.18"
# Jinja 2.x is incompatible with MarkupSafe>=2.1. To ensure that admins do not
@ -201,13 +210,11 @@ cryptography = ">=3.4.7"
# ijson 3.1.4 fixes a bug with "." in property names
ijson = ">=3.1.4"
matrix-common = "^1.3.0"
# We need packaging.requirements.Requirement, added in 16.1.
packaging = ">=16.1"
# At the time of writing, we only use functions from the version `importlib.metadata`
# which shipped in Python 3.8. This corresponds to version 1.4 of the backport.
importlib_metadata = { version = ">=1.4", python = "<3.8" }
# This is the most recent version of Pydantic with available on common distros.
pydantic = ">=1.7.4"
# We need packaging.verison.Version(...).major added in 20.0.
packaging = ">=20.0"
# We support pydantic v1 and pydantic v2 via the pydantic.v1 compat module.
# See https://github.com/matrix-org/synapse/issues/15858
pydantic = ">=1.7.4, <3"
# This is for building the rust components during "poetry install", which
# currently ignores the `build-system.requires` directive (c.f.
@ -308,10 +315,15 @@ all = [
]
[tool.poetry.dev-dependencies]
# We pin black so that our tests don't start failing on new releases.
# We pin development dependencies in poetry.lock so that our tests don't start
# failing on new releases. Keeping lower bounds loose here means that dependabot
# can bump versions without having to update the content-hash in the lockfile.
# This helps prevents merge conflicts when running a batch of dependabot updates.
isort = ">=5.10.1"
black = ">=22.3.0"
ruff = "0.0.265"
black = ">=22.7.0"
ruff = "0.1.6"
# Type checking only works with the pydantic.v1 compat module from pydantic v2
pydantic = "^2"
# Typechecking
lxml-stubs = ">=0.4.0"
@ -345,7 +357,7 @@ commonmark = ">=0.9.1"
pygithub = ">=1.55"
# The following are executed as commands by the release script.
twine = "*"
# Towncrier min version comes from #3425. Rationale unclear.
# Towncrier min version comes from https://github.com/matrix-org/synapse/pull/3425. Rationale unclear.
towncrier = ">=18.6.0rc1"
# Used for checking the Poetry lockfile
@ -358,24 +370,33 @@ optional = true
[tool.poetry.group.dev-docs.dependencies]
sphinx = {version = "^6.1", python = "^3.8"}
sphinx-autodoc2 = {version = "^0.4.2", python = "^3.8"}
sphinx-autodoc2 = {version = ">=0.4.2,<0.6.0", python = "^3.8"}
myst-parser = {version = "^1.0.0", python = "^3.8"}
furo = ">=2022.12.7,<2024.0.0"
[build-system]
# The upper bounds here are defensive, intended to prevent situations like
# #13849 and #14079 where we see buildtime or runtime errors caused by build
# system changes.
# https://github.com/matrix-org/synapse/issues/13849 and
# https://github.com/matrix-org/synapse/issues/14079 where we see buildtime or
# runtime errors caused by build system changes.
# We are happy to raise these upper bounds upon request,
# provided we check that it's safe to do so (i.e. that CI passes).
requires = ["poetry-core>=1.1.0,<=1.6.0", "setuptools_rust>=1.3,<=1.6.0"]
requires = ["poetry-core>=1.1.0,<=1.8.1", "setuptools_rust>=1.3,<=1.8.1"]
build-backend = "poetry.core.masonry.api"
[tool.cibuildwheel]
# Skip unsupported platforms (by us or by Rust).
skip = "cp36* *-musllinux_i686 pp*aarch64 *-musllinux_aarch64"
# See https://cibuildwheel.readthedocs.io/en/stable/options/#build-skip for the list of build targets.
# We skip:
# - CPython 3.6 and 3.7: EOLed
# - PyPy 3.7: we only support Python 3.8+
# - musllinux i686: excluded to reduce number of wheels we build.
# c.f. https://github.com/matrix-org/synapse/pull/12595#discussion_r963107677
# - PyPy on Aarch64 and musllinux on aarch64: too slow to build.
# c.f. https://github.com/matrix-org/synapse/pull/14259
skip = "cp36* cp37* pp37* *-musllinux_i686 pp*aarch64 *-musllinux_aarch64"
# We need a rust compiler
before-all = "curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain stable -y --profile minimal"

View File

@ -7,7 +7,7 @@ name = "synapse"
version = "0.1.0"
edition = "2021"
rust-version = "1.60.0"
rust-version = "1.61.0"
[lib]
name = "synapse"
@ -15,6 +15,8 @@ name = "synapse"
# tests/benchmarks.
crate-type = ["lib", "cdylib"]
# This is deprecated, see tool.maturin in pyproject.toml.
# It is left here for compatibilty with maturin < 0.15.
[package.metadata.maturin]
# This is where we tell maturin where to place the built library.
name = "synapse.synapse_rust"
@ -23,9 +25,14 @@ name = "synapse.synapse_rust"
anyhow = "1.0.63"
lazy_static = "1.4.0"
log = "0.4.17"
pyo3 = { version = "0.17.1", features = ["macros", "anyhow", "abi3", "abi3-py37"] }
pyo3-log = "0.8.1"
pythonize = "0.17.0"
pyo3 = { version = "0.20.0", features = [
"macros",
"anyhow",
"abi3",
"abi3-py38",
] }
pyo3-log = "0.9.0"
pythonize = "0.20.0"
regex = "1.6.0"
serde = { version = "1.0.144", features = ["derive"] }
serde_json = "1.0.85"

View File

@ -13,6 +13,9 @@
// limitations under the License.
#![feature(test)]
use std::borrow::Cow;
use synapse::push::{
evaluator::PushRuleEvaluator, Condition, EventMatchCondition, FilteredPushRules, JsonValue,
PushRules, SimpleJsonValue,
@ -26,15 +29,15 @@ fn bench_match_exact(b: &mut Bencher) {
let flattened_keys = [
(
"type".to_string(),
JsonValue::Value(SimpleJsonValue::Str("m.text".to_string())),
JsonValue::Value(SimpleJsonValue::Str(Cow::Borrowed("m.text"))),
),
(
"room_id".to_string(),
JsonValue::Value(SimpleJsonValue::Str("!room:server".to_string())),
JsonValue::Value(SimpleJsonValue::Str(Cow::Borrowed("!room:server"))),
),
(
"content.body".to_string(),
JsonValue::Value(SimpleJsonValue::Str("test message".to_string())),
JsonValue::Value(SimpleJsonValue::Str(Cow::Borrowed("test message"))),
),
]
.into_iter()
@ -71,15 +74,15 @@ fn bench_match_word(b: &mut Bencher) {
let flattened_keys = [
(
"type".to_string(),
JsonValue::Value(SimpleJsonValue::Str("m.text".to_string())),
JsonValue::Value(SimpleJsonValue::Str(Cow::Borrowed("m.text"))),
),
(
"room_id".to_string(),
JsonValue::Value(SimpleJsonValue::Str("!room:server".to_string())),
JsonValue::Value(SimpleJsonValue::Str(Cow::Borrowed("!room:server"))),
),
(
"content.body".to_string(),
JsonValue::Value(SimpleJsonValue::Str("test message".to_string())),
JsonValue::Value(SimpleJsonValue::Str(Cow::Borrowed("test message"))),
),
]
.into_iter()
@ -116,15 +119,15 @@ fn bench_match_word_miss(b: &mut Bencher) {
let flattened_keys = [
(
"type".to_string(),
JsonValue::Value(SimpleJsonValue::Str("m.text".to_string())),
JsonValue::Value(SimpleJsonValue::Str(Cow::Borrowed("m.text"))),
),
(
"room_id".to_string(),
JsonValue::Value(SimpleJsonValue::Str("!room:server".to_string())),
JsonValue::Value(SimpleJsonValue::Str(Cow::Borrowed("!room:server"))),
),
(
"content.body".to_string(),
JsonValue::Value(SimpleJsonValue::Str("test message".to_string())),
JsonValue::Value(SimpleJsonValue::Str(Cow::Borrowed("test message"))),
),
]
.into_iter()
@ -161,15 +164,15 @@ fn bench_eval_message(b: &mut Bencher) {
let flattened_keys = [
(
"type".to_string(),
JsonValue::Value(SimpleJsonValue::Str("m.text".to_string())),
JsonValue::Value(SimpleJsonValue::Str(Cow::Borrowed("m.text"))),
),
(
"room_id".to_string(),
JsonValue::Value(SimpleJsonValue::Str("!room:server".to_string())),
JsonValue::Value(SimpleJsonValue::Str(Cow::Borrowed("!room:server"))),
),
(
"content.body".to_string(),
JsonValue::Value(SimpleJsonValue::Str("test message".to_string())),
JsonValue::Value(SimpleJsonValue::Str(Cow::Borrowed("test message"))),
),
]
.into_iter()

102
rust/src/acl/mod.rs Normal file
View File

@ -0,0 +1,102 @@
// Copyright 2023 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! An implementation of Matrix server ACL rules.
use std::net::Ipv4Addr;
use std::str::FromStr;
use anyhow::Error;
use pyo3::prelude::*;
use regex::Regex;
use crate::push::utils::{glob_to_regex, GlobMatchType};
/// Called when registering modules with python.
pub fn register_module(py: Python<'_>, m: &PyModule) -> PyResult<()> {
let child_module = PyModule::new(py, "acl")?;
child_module.add_class::<ServerAclEvaluator>()?;
m.add_submodule(child_module)?;
// We need to manually add the module to sys.modules to make `from
// synapse.synapse_rust import acl` work.
py.import("sys")?
.getattr("modules")?
.set_item("synapse.synapse_rust.acl", child_module)?;
Ok(())
}
#[derive(Debug, Clone)]
#[pyclass(frozen)]
pub struct ServerAclEvaluator {
allow_ip_literals: bool,
allow: Vec<Regex>,
deny: Vec<Regex>,
}
#[pymethods]
impl ServerAclEvaluator {
#[new]
pub fn py_new(
allow_ip_literals: bool,
allow: Vec<&str>,
deny: Vec<&str>,
) -> Result<Self, Error> {
let allow = allow
.iter()
.map(|s| glob_to_regex(s, GlobMatchType::Whole))
.collect::<Result<_, _>>()?;
let deny = deny
.iter()
.map(|s| glob_to_regex(s, GlobMatchType::Whole))
.collect::<Result<_, _>>()?;
Ok(ServerAclEvaluator {
allow_ip_literals,
allow,
deny,
})
}
pub fn server_matches_acl_event(&self, server_name: &str) -> bool {
// first of all, check if literal IPs are blocked, and if so, whether the
// server name is a literal IP
if !self.allow_ip_literals {
// check for ipv6 literals. These start with '['.
if server_name.starts_with('[') {
return false;
}
// check for ipv4 literals. We can just lift the routine from std::net.
if Ipv4Addr::from_str(server_name).is_ok() {
return false;
}
}
// next, check the deny list
if self.deny.iter().any(|e| e.is_match(server_name)) {
return false;
}
// then the allow list.
if self.allow.iter().any(|e| e.is_match(server_name)) {
return true;
}
// everything else should be rejected.
false
}
}

View File

@ -2,6 +2,7 @@ use lazy_static::lazy_static;
use pyo3::prelude::*;
use pyo3_log::ResetHandle;
pub mod acl;
pub mod push;
lazy_static! {
@ -38,6 +39,7 @@ fn synapse_rust(py: Python<'_>, m: &PyModule) -> PyResult<()> {
m.add_function(wrap_pyfunction!(get_rust_file_digest, m)?)?;
m.add_function(wrap_pyfunction!(reset_logging_config, m)?)?;
acl::register_module(py, m)?;
push::register_module(py, m)?;
Ok(())

View File

@ -63,21 +63,18 @@ pub const BASE_PREPEND_OVERRIDE_RULES: &[PushRule] = &[PushRule {
}];
pub const BASE_APPEND_OVERRIDE_RULES: &[PushRule] = &[
// We don't want to notify on edits. Not only can this be confusing in real
// time (2 notifications, one message) but it's especially confusing
// if a bridge needs to edit a previously backfilled message.
PushRule {
rule_id: Cow::Borrowed("global/override/.com.beeper.suppress_edits"),
rule_id: Cow::Borrowed("global/override/.org.matrix.msc4028.encrypted_event"),
priority_class: 5,
conditions: Cow::Borrowed(&[Condition::Known(KnownCondition::EventMatch(
EventMatchCondition {
key: Cow::Borrowed("content.m\\.relates_to.rel_type"),
pattern: Cow::Borrowed("m.replace"),
key: Cow::Borrowed("type"),
pattern: Cow::Borrowed("m.room.encrypted"),
},
))]),
actions: Cow::Borrowed(&[]),
actions: Cow::Borrowed(&[Action::Notify]),
default: true,
default_enabled: true,
default_enabled: false,
},
PushRule {
rule_id: Cow::Borrowed("global/override/.m.rule.suppress_notices"),
@ -142,11 +139,11 @@ pub const BASE_APPEND_OVERRIDE_RULES: &[PushRule] = &[
default_enabled: true,
},
PushRule {
rule_id: Cow::Borrowed("global/override/.m.is_user_mention"),
rule_id: Cow::Borrowed("global/override/.m.rule.is_user_mention"),
priority_class: 5,
conditions: Cow::Borrowed(&[Condition::Known(
KnownCondition::ExactEventPropertyContainsType(EventPropertyIsTypeCondition {
key: Cow::Borrowed("content.m\\.mentions.user_ids"),
key: Cow::Borrowed(r"content.m\.mentions.user_ids"),
value_type: Cow::Borrowed(&EventMatchPatternType::UserId),
}),
)]),
@ -163,12 +160,12 @@ pub const BASE_APPEND_OVERRIDE_RULES: &[PushRule] = &[
default_enabled: true,
},
PushRule {
rule_id: Cow::Borrowed("global/override/.m.is_room_mention"),
rule_id: Cow::Borrowed("global/override/.m.rule.is_room_mention"),
priority_class: 5,
conditions: Cow::Borrowed(&[
Condition::Known(KnownCondition::EventPropertyIs(EventPropertyIsCondition {
key: Cow::Borrowed("content.m\\.mentions.room"),
value: Cow::Borrowed(&SimpleJsonValue::Bool(true)),
key: Cow::Borrowed(r"content.m\.mentions.room"),
value: Cow::Owned(SimpleJsonValue::Bool(true)),
})),
Condition::Known(KnownCondition::SenderNotificationPermission {
key: Cow::Borrowed("room"),
@ -241,6 +238,21 @@ pub const BASE_APPEND_OVERRIDE_RULES: &[PushRule] = &[
default: true,
default_enabled: true,
},
// We don't want to notify on edits *unless* the edit directly mentions a
// user, which is handled above.
PushRule {
rule_id: Cow::Borrowed("global/override/.m.rule.suppress_edits"),
priority_class: 5,
conditions: Cow::Borrowed(&[Condition::Known(KnownCondition::EventPropertyIs(
EventPropertyIsCondition {
key: Cow::Borrowed(r"content.m\.relates_to.rel_type"),
value: Cow::Owned(SimpleJsonValue::Str(Cow::Borrowed("m.replace"))),
},
))]),
actions: Cow::Borrowed(&[]),
default: true,
default_enabled: true,
},
PushRule {
rule_id: Cow::Borrowed("global/override/.org.matrix.msc3930.rule.poll_response"),
priority_class: 5,

View File

@ -105,6 +105,17 @@ impl PushRuleEvaluator {
/// Create a new `PushRuleEvaluator`. See struct docstring for details.
#[allow(clippy::too_many_arguments)]
#[new]
#[pyo3(signature = (
flattened_keys,
has_mentions,
room_member_count,
sender_power_level,
notification_power_levels,
related_events_flattened,
related_event_match_enabled,
room_version_feature_flags,
msc3931_enabled,
))]
pub fn py_new(
flattened_keys: BTreeMap<String, JsonValue>,
has_mentions: bool,
@ -117,7 +128,7 @@ impl PushRuleEvaluator {
msc3931_enabled: bool,
) -> Result<Self, Error> {
let body = match flattened_keys.get("content.body") {
Some(JsonValue::Value(SimpleJsonValue::Str(s))) => s.clone(),
Some(JsonValue::Value(SimpleJsonValue::Str(s))) => s.clone().into_owned(),
_ => String::new(),
};
@ -313,13 +324,15 @@ impl PushRuleEvaluator {
};
let pattern = match &*exact_event_match.value_type {
EventMatchPatternType::UserId => user_id,
EventMatchPatternType::UserLocalpart => get_localpart_from_id(user_id)?,
EventMatchPatternType::UserId => user_id.to_owned(),
EventMatchPatternType::UserLocalpart => {
get_localpart_from_id(user_id)?.to_owned()
}
};
self.match_event_property_contains(
exact_event_match.key.clone(),
Cow::Borrowed(&SimpleJsonValue::Str(pattern.to_string())),
Cow::Borrowed(&SimpleJsonValue::Str(Cow::Owned(pattern))),
)?
}
KnownCondition::ContainsDisplayName => {
@ -494,7 +507,7 @@ fn push_rule_evaluator() {
let mut flattened_keys = BTreeMap::new();
flattened_keys.insert(
"content.body".to_string(),
JsonValue::Value(SimpleJsonValue::Str("foo bar bob hello".to_string())),
JsonValue::Value(SimpleJsonValue::Str(Cow::Borrowed("foo bar bob hello"))),
);
let evaluator = PushRuleEvaluator::py_new(
flattened_keys,
@ -522,7 +535,7 @@ fn test_requires_room_version_supports_condition() {
let mut flattened_keys = BTreeMap::new();
flattened_keys.insert(
"content.body".to_string(),
JsonValue::Value(SimpleJsonValue::Str("foo bar bob hello".to_string())),
JsonValue::Value(SimpleJsonValue::Str(Cow::Borrowed("foo bar bob hello"))),
);
let flags = vec![RoomVersionFeatures::ExtensibleEvents.as_str().to_string()];
let evaluator = PushRuleEvaluator::py_new(

View File

@ -256,7 +256,7 @@ impl<'de> Deserialize<'de> for Action {
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
#[serde(untagged)]
pub enum SimpleJsonValue {
Str(String),
Str(Cow<'static, str>),
Int(i64),
Bool(bool),
Null,
@ -265,7 +265,7 @@ pub enum SimpleJsonValue {
impl<'source> FromPyObject<'source> for SimpleJsonValue {
fn extract(ob: &'source PyAny) -> PyResult<Self> {
if let Ok(s) = <PyString as pyo3::PyTryFrom>::try_from(ob) {
Ok(SimpleJsonValue::Str(s.to_string()))
Ok(SimpleJsonValue::Str(Cow::Owned(s.to_string())))
// A bool *is* an int, ensure we try bool first.
} else if let Ok(b) = <PyBool as pyo3::PyTryFrom>::try_from(ob) {
Ok(SimpleJsonValue::Bool(b.extract()?))
@ -296,8 +296,7 @@ impl<'source> FromPyObject<'source> for JsonValue {
match l.iter().map(SimpleJsonValue::extract).collect() {
Ok(a) => Ok(JsonValue::Array(a)),
Err(e) => Err(PyTypeError::new_err(format!(
"Can't convert to JsonValue::Array: {}",
e
"Can't convert to JsonValue::Array: {e}"
))),
}
} else if let Ok(v) = SimpleJsonValue::extract(ob) {
@ -527,7 +526,7 @@ pub struct FilteredPushRules {
msc1767_enabled: bool,
msc3381_polls_enabled: bool,
msc3664_enabled: bool,
msc3958_suppress_edits_enabled: bool,
msc4028_push_encrypted_events: bool,
}
#[pymethods]
@ -539,7 +538,7 @@ impl FilteredPushRules {
msc1767_enabled: bool,
msc3381_polls_enabled: bool,
msc3664_enabled: bool,
msc3958_suppress_edits_enabled: bool,
msc4028_push_encrypted_events: bool,
) -> Self {
Self {
push_rules,
@ -547,7 +546,7 @@ impl FilteredPushRules {
msc1767_enabled,
msc3381_polls_enabled,
msc3664_enabled,
msc3958_suppress_edits_enabled,
msc4028_push_encrypted_events,
}
}
@ -584,8 +583,8 @@ impl FilteredPushRules {
return false;
}
if !self.msc3958_suppress_edits_enabled
&& rule.rule_id == "global/override/.com.beeper.suppress_edits"
if !self.msc4028_push_encrypted_events
&& rule.rule_id == "global/override/.org.matrix.msc4028.encrypted_event"
{
return false;
}

Some files were not shown because too many files have changed in this diff Show More