From 81818c0eeed2751d7eef94803f8ed0162e0b710b Mon Sep 17 00:00:00 2001 From: reivilibre Date: Tue, 15 Aug 2023 11:15:27 +0000 Subject: [PATCH] deploy: 29638220ab31aea16cd3da073025e9cb734628d5 --- latest/404.html | 2 +- latest/CAPTCHA_SETUP.html | 2 +- latest/admin_api/account_validity.html | 2 +- latest/admin_api/event_reports.html | 2 +- latest/admin_api/experimental_features.html | 2 +- latest/admin_api/media_admin_api.html | 2 +- latest/admin_api/purge_history_api.html | 2 +- latest/admin_api/register_api.html | 2 +- latest/admin_api/room_membership.html | 2 +- latest/admin_api/rooms.html | 2 +- latest/admin_api/server_notices.html | 2 +- latest/admin_api/statistics.html | 2 +- latest/admin_api/user_admin_api.html | 2 +- latest/admin_api/version_api.html | 2 +- latest/application_services.html | 2 +- latest/auth_chain_difference_algorithm.html | 2 +- latest/code_style.html | 2 +- latest/consent_tracking.html | 2 +- latest/delegate.html | 2 +- latest/deprecation_policy.html | 2 +- latest/development/cas.html | 2 +- latest/development/contributing_guide.html | 2 +- latest/development/database_schema.html | 2 +- latest/development/demo.html | 2 +- latest/development/dependencies.html | 2 +- latest/development/experimental_features.html | 2 +- latest/development/git.html | 2 +- .../internal_documentation/index.html | 2 +- latest/development/releases.html | 2 +- latest/development/reviews.html | 2 +- latest/development/room-dag-concepts.html | 2 +- latest/development/saml.html | 2 +- .../synapse_architecture/cancellation.html | 2 +- .../synapse_architecture/faster_joins.html | 2 +- .../synapse_architecture/streams.html | 316 ++++++++++++++++++ latest/federate.html | 2 +- latest/index.html | 2 +- latest/jwt.html | 2 +- latest/log_contexts.html | 2 +- latest/manhole.html | 2 +- latest/media_repository.html | 2 +- latest/message_retention_policies.html | 2 +- latest/metrics-howto.html | 2 +- latest/modules/account_data_callbacks.html | 2 +- .../modules/account_validity_callbacks.html | 2 +- ...ackground_update_controller_callbacks.html | 2 +- latest/modules/index.html | 2 +- .../password_auth_provider_callbacks.html | 2 +- latest/modules/porting_legacy_module.html | 2 +- latest/modules/presence_router_callbacks.html | 2 +- latest/modules/spam_checker_callbacks.html | 2 +- .../modules/third_party_rules_callbacks.html | 2 +- latest/modules/writing_a_module.html | 2 +- latest/openid.html | 2 +- latest/opentracing.html | 2 +- ...ing_synapse_on_single_board_computers.html | 2 +- latest/password_auth_providers.html | 2 +- latest/postgres.html | 2 +- latest/print.html | 162 ++++++++- latest/replication.html | 6 +- latest/reverse_proxy.html | 4 +- latest/room_and_user_statistics.html | 2 +- latest/searchindex.js | 2 +- latest/searchindex.json | 2 +- latest/server_notices.html | 2 +- latest/setup/forward_proxy.html | 2 +- latest/setup/installation.html | 2 +- latest/setup/turn/coturn.html | 2 +- latest/setup/turn/eturnal.html | 2 +- latest/sso_mapping_providers.html | 2 +- latest/structured_logging.html | 2 +- latest/synctl_workers.html | 2 +- latest/systemd-with-workers/index.html | 2 +- latest/tcp_replication.html | 6 +- latest/templates.html | 2 +- latest/turn-howto.html | 2 +- latest/upgrade.html | 13 +- .../admin_api/background_updates.html | 2 +- .../administration/admin_api/federation.html | 2 +- .../usage/administration/admin_api/index.html | 2 +- .../admin_api/registration_tokens.html | 2 +- latest/usage/administration/admin_faq.html | 2 +- .../database_maintenance_tools.html | 2 +- latest/usage/administration/index.html | 2 +- ...reporting_homeserver_usage_statistics.html | 2 +- .../administration/monthly_active_users.html | 2 +- latest/usage/administration/request_log.html | 2 +- latest/usage/administration/state_groups.html | 2 +- ...anding_synapse_through_grafana_graphs.html | 2 +- .../administration/useful_sql_for_admins.html | 2 +- .../configuration/config_documentation.html | 22 +- .../homeserver_sample_config.html | 2 +- latest/usage/configuration/index.html | 2 +- .../configuration/logging_sample_config.html | 2 +- .../user_authentication/index.html | 2 +- .../user_authentication/refresh_tokens.html | 2 +- .../single_sign_on/cas.html | 2 +- .../single_sign_on/index.html | 2 +- .../single_sign_on/saml.html | 2 +- latest/user_directory.html | 2 +- latest/welcome_and_overview.html | 2 +- latest/workers.html | 2 +- 102 files changed, 612 insertions(+), 107 deletions(-) create mode 100644 latest/development/synapse_architecture/streams.html diff --git a/latest/404.html b/latest/404.html index e4cdd912a2..399c3a5829 100644 --- a/latest/404.html +++ b/latest/404.html @@ -77,7 +77,7 @@ diff --git a/latest/CAPTCHA_SETUP.html b/latest/CAPTCHA_SETUP.html index ed7e8c4ffc..a0050f43fb 100644 --- a/latest/CAPTCHA_SETUP.html +++ b/latest/CAPTCHA_SETUP.html @@ -76,7 +76,7 @@ diff --git a/latest/admin_api/account_validity.html b/latest/admin_api/account_validity.html index 8c73efd2bf..cafea79756 100644 --- a/latest/admin_api/account_validity.html +++ b/latest/admin_api/account_validity.html @@ -76,7 +76,7 @@ diff --git a/latest/admin_api/event_reports.html b/latest/admin_api/event_reports.html index 79bf44a935..aabec67f60 100644 --- a/latest/admin_api/event_reports.html +++ b/latest/admin_api/event_reports.html @@ -76,7 +76,7 @@ diff --git a/latest/admin_api/experimental_features.html b/latest/admin_api/experimental_features.html index 046c6874da..88b3e9aa0e 100644 --- a/latest/admin_api/experimental_features.html +++ b/latest/admin_api/experimental_features.html @@ -76,7 +76,7 @@ diff --git a/latest/admin_api/media_admin_api.html b/latest/admin_api/media_admin_api.html index e26846a1f3..89249d8c2c 100644 --- a/latest/admin_api/media_admin_api.html +++ b/latest/admin_api/media_admin_api.html @@ -76,7 +76,7 @@ diff --git a/latest/admin_api/purge_history_api.html b/latest/admin_api/purge_history_api.html index 28f693ec4f..9b5e258b90 100644 --- a/latest/admin_api/purge_history_api.html +++ b/latest/admin_api/purge_history_api.html @@ -76,7 +76,7 @@ diff --git a/latest/admin_api/register_api.html b/latest/admin_api/register_api.html index f1407aa8d6..3a6d44811f 100644 --- a/latest/admin_api/register_api.html +++ b/latest/admin_api/register_api.html @@ -76,7 +76,7 @@ diff --git a/latest/admin_api/room_membership.html b/latest/admin_api/room_membership.html index e79f987e7d..b5b3e76b61 100644 --- a/latest/admin_api/room_membership.html +++ b/latest/admin_api/room_membership.html @@ -76,7 +76,7 @@ diff --git a/latest/admin_api/rooms.html b/latest/admin_api/rooms.html index 0e06e4f4cc..fe246b4041 100644 --- a/latest/admin_api/rooms.html +++ b/latest/admin_api/rooms.html @@ -76,7 +76,7 @@ diff --git a/latest/admin_api/server_notices.html b/latest/admin_api/server_notices.html index c895b5f65d..fdb9641d8c 100644 --- a/latest/admin_api/server_notices.html +++ b/latest/admin_api/server_notices.html @@ -76,7 +76,7 @@ diff --git a/latest/admin_api/statistics.html b/latest/admin_api/statistics.html index 4bf39652a8..323a1165e1 100644 --- a/latest/admin_api/statistics.html +++ b/latest/admin_api/statistics.html @@ -76,7 +76,7 @@ diff --git a/latest/admin_api/user_admin_api.html b/latest/admin_api/user_admin_api.html index ef51693edc..0acc361803 100644 --- a/latest/admin_api/user_admin_api.html +++ b/latest/admin_api/user_admin_api.html @@ -76,7 +76,7 @@ diff --git a/latest/admin_api/version_api.html b/latest/admin_api/version_api.html index 5b708befc3..00a04cf442 100644 --- a/latest/admin_api/version_api.html +++ b/latest/admin_api/version_api.html @@ -76,7 +76,7 @@ diff --git a/latest/application_services.html b/latest/application_services.html index d710b93369..3d7dd407d8 100644 --- a/latest/application_services.html +++ b/latest/application_services.html @@ -76,7 +76,7 @@ diff --git a/latest/auth_chain_difference_algorithm.html b/latest/auth_chain_difference_algorithm.html index c7fe5ab185..d7d451fcfd 100644 --- a/latest/auth_chain_difference_algorithm.html +++ b/latest/auth_chain_difference_algorithm.html @@ -76,7 +76,7 @@ diff --git a/latest/code_style.html b/latest/code_style.html index 004a191579..02d716dbf6 100644 --- a/latest/code_style.html +++ b/latest/code_style.html @@ -76,7 +76,7 @@ diff --git a/latest/consent_tracking.html b/latest/consent_tracking.html index 573b5603ad..34cfff1162 100644 --- a/latest/consent_tracking.html +++ b/latest/consent_tracking.html @@ -76,7 +76,7 @@ diff --git a/latest/delegate.html b/latest/delegate.html index bc8d358910..fa99c564b2 100644 --- a/latest/delegate.html +++ b/latest/delegate.html @@ -76,7 +76,7 @@ diff --git a/latest/deprecation_policy.html b/latest/deprecation_policy.html index ceed20d222..95742bacae 100644 --- a/latest/deprecation_policy.html +++ b/latest/deprecation_policy.html @@ -76,7 +76,7 @@ diff --git a/latest/development/cas.html b/latest/development/cas.html index 193205b1d6..ae35250574 100644 --- a/latest/development/cas.html +++ b/latest/development/cas.html @@ -76,7 +76,7 @@ diff --git a/latest/development/contributing_guide.html b/latest/development/contributing_guide.html index cbfd13c9a7..b328bba158 100644 --- a/latest/development/contributing_guide.html +++ b/latest/development/contributing_guide.html @@ -76,7 +76,7 @@ diff --git a/latest/development/database_schema.html b/latest/development/database_schema.html index fe1e178fb5..3ba3b06439 100644 --- a/latest/development/database_schema.html +++ b/latest/development/database_schema.html @@ -76,7 +76,7 @@ diff --git a/latest/development/demo.html b/latest/development/demo.html index 11d7469f70..2d4754ff33 100644 --- a/latest/development/demo.html +++ b/latest/development/demo.html @@ -76,7 +76,7 @@ diff --git a/latest/development/dependencies.html b/latest/development/dependencies.html index d64f15fc41..f44f2db144 100644 --- a/latest/development/dependencies.html +++ b/latest/development/dependencies.html @@ -76,7 +76,7 @@ diff --git a/latest/development/experimental_features.html b/latest/development/experimental_features.html index f077a2a1af..98974dd4cd 100644 --- a/latest/development/experimental_features.html +++ b/latest/development/experimental_features.html @@ -76,7 +76,7 @@ diff --git a/latest/development/git.html b/latest/development/git.html index dfbb987e67..a6a9a322e6 100644 --- a/latest/development/git.html +++ b/latest/development/git.html @@ -76,7 +76,7 @@ diff --git a/latest/development/internal_documentation/index.html b/latest/development/internal_documentation/index.html index 1f1015dbed..10f4a067e1 100644 --- a/latest/development/internal_documentation/index.html +++ b/latest/development/internal_documentation/index.html @@ -76,7 +76,7 @@ diff --git a/latest/development/releases.html b/latest/development/releases.html index 83330e341d..5cea2ff5ec 100644 --- a/latest/development/releases.html +++ b/latest/development/releases.html @@ -76,7 +76,7 @@ diff --git a/latest/development/reviews.html b/latest/development/reviews.html index 26e589db32..ee932ce705 100644 --- a/latest/development/reviews.html +++ b/latest/development/reviews.html @@ -76,7 +76,7 @@ diff --git a/latest/development/room-dag-concepts.html b/latest/development/room-dag-concepts.html index b61d197b36..eaa06b291c 100644 --- a/latest/development/room-dag-concepts.html +++ b/latest/development/room-dag-concepts.html @@ -76,7 +76,7 @@ diff --git a/latest/development/saml.html b/latest/development/saml.html index 1a989d6454..3a2e8ac490 100644 --- a/latest/development/saml.html +++ b/latest/development/saml.html @@ -76,7 +76,7 @@ diff --git a/latest/development/synapse_architecture/cancellation.html b/latest/development/synapse_architecture/cancellation.html index b2c1680ed5..8f7d25bb88 100644 --- a/latest/development/synapse_architecture/cancellation.html +++ b/latest/development/synapse_architecture/cancellation.html @@ -76,7 +76,7 @@ diff --git a/latest/development/synapse_architecture/faster_joins.html b/latest/development/synapse_architecture/faster_joins.html index 1c77bbc91f..d60c159d11 100644 --- a/latest/development/synapse_architecture/faster_joins.html +++ b/latest/development/synapse_architecture/faster_joins.html @@ -76,7 +76,7 @@ diff --git a/latest/development/synapse_architecture/streams.html b/latest/development/synapse_architecture/streams.html new file mode 100644 index 0000000000..a76d3e8020 --- /dev/null +++ b/latest/development/synapse_architecture/streams.html @@ -0,0 +1,316 @@ + + + + + + Streams - Synapse + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + + + + +
+
+ +
+ +
+ +

Streams

+

Synapse has a concept of "streams", which are roughly described in id_generators.py. +Generally speaking, streams are a series of notifications that something in Synapse's database has changed that the application might need to respond to. +For example:

+
    +
  • The events stream reports new events (PDUs) that Synapse creates, or that Synapse accepts from another homeserver.
    • +
  • The account data stream reports changes to users' account data.
    • +
  • The to-device stream reports when a device has a new to-device message.
    • +
+

See synapse.replication.tcp.streams for the full list of streams.

+

It is very helpful to understand the streams mechanism when working on any part of Synapse that needs to respond to changes—especially if those changes are made by different workers. +To that end, let's describe streams formally, paraphrasing from the docstring of AbstractStreamIdGenerator.

+

Definition

+

A stream is an append-only log T1, T2, ..., Tn, ... of facts1 which grows over time. +Only "writers" can add facts to a stream, and there may be multiple writers.

+

Each fact has an ID, called its "stream ID". +Readers should only process facts in ascending stream ID order.

+

Roughly speaking, each stream is backed by a database table. +It should have a stream_id (or similar) bigint column holding stream IDs, plus additional columns as necessary to describe the fact. +Typically, a fact is expressed with a single row in its backing table.2 +Within a stream, no two facts may have the same stream_id.

+
+

Aside. Some additional notes on streams' backing tables.

+
    +
  1. Rich would like to ditch the backing tables.
    2. +
  2. The backing tables may have other uses. +> For example, the events table serves backs the events stream, and is read when processing new events. +> But old rows are read from the table all the time, whenever Synapse needs to lookup some facts about an event.
    3. +
  3. Rich suspects that sometimes the stream is backed by multiple tables, so the stream proper is the union of those tables.
    4. +
+
+

Stream writers can "reserve" a stream ID, and then later mark it as having being completed. +Stream writers need to track the completion of each stream fact. +In the happy case, completion means a fact has been written to the stream table. +But unhappy cases (e.g. transaction rollback due to an error) also count as completion. +Once completed, the rows written with that stream ID are fixed, and no new rows +will be inserted with that ID.

+

Current stream ID

+

For any given stream reader (including writers themselves), we may define a per-writer current stream ID:

+
+

The current stream ID for a writer W is the largest stream ID such that +all transactions added by W with equal or smaller ID have completed.

+
+

Similarly, there is a "linear" notion of current stream ID:

+
+

The "linear" current stream ID is the largest stream ID such that +all facts (added by any writer) with equal or smaller ID have completed.

+
+

Because different stream readers A and B learn about new facts at different times, A and B may disagree about current stream IDs. +Put differently: we should think of stream readers as being independent of each other, proceeding through a stream of facts at different rates.

+

NB. For both senses of "current", that if a writer opens a transaction that never completes, the current stream ID will never advance beyond that writer's last written stream ID.

+

For single-writer streams, the per-writer current ID and the linear current ID are the same. +Both senses of current ID are monotonic, but they may "skip" or jump over IDs because facts complete out of order.

+

Example. +Consider a single-writer stream which is initially at ID 1.

+ + + + + + + + + + + + +
ActionCurrent stream IDNotes
1
Reserve 21
Reserve 31
Complete 31current ID unchanged, waiting for 2 to complete
Complete 23current ID jumps from 1 -> 3
Reserve 43
Reserve 53
Reserve 63
Complete 53
Complete 45current ID jumps 3->5, even though 6 is pending
Complete 66
+

Multi-writer streams

+

There are two ways to view a multi-writer stream.

+
    +
  1. Treat it as a collection of distinct single-writer streams, one +for each writer.
    2. +
  2. Treat it as a single stream.
    3. +
+

The single stream (option 2) is conceptually simpler, and easier to represent (a single stream id). +However, it requires each reader to know about the entire set of writers, to ensures that readers don't erroneously advance their current stream position too early and miss a fact from an unknown writer. +In contrast, multiple parallel streams (option 1) are more complex, requiring more state to represent (map from writer to stream id). +The payoff for doing so is that readers can "peek" ahead to facts that completed on one writer no matter the state of the others, reducing latency.

+

Note that a multi-writer stream can be viewed in both ways. +For example, the events stream is treated as multiple single-writer streams (option 1) by the sync handler, so that events are sent to clients as soon as possible. +But the background process that works through events treats them as a single linear stream.

+

Another useful example is the cache invalidation stream. +The facts this stream holds are instructions to "you should now invalidate these cache entries". +We only ever treat this as a multiple single-writer streams as there is no important ordering between cache invalidations. +(Invalidations are self-contained facts; and the invalidations commute/are idempotent).

+

Writing to streams

+

Writers need to track:

+
    +
  • track their current position (i.e. its own per-writer stream ID).
    • +
  • their facts currently awaiting completion.
    • +
+

At startup,

+
    +
  • the current position of that writer can be found by querying the database (which suggests that facts need to be written to the database atomically, in a transaction); and
    • +
  • there are no facts awaiting completion.
    • +
+

To reserve a stream ID, call nextval on the appropriate postgres sequence.

+

To write a fact to the stream: insert the appropriate rows to the appropriate backing table.

+

To complete a fact, first remove it from your map of facts currently awaiting completion. +Then, if no earlier fact is awaiting completion, the writer can advance its current position in that stream. +Upon doing so it should emit an RDATA message3, once for every fact between the old and the new stream ID.

+

Subscribing to streams

+

Readers need to track the current position of every writer.

+

At startup, they can find this by contacting each writer with a REPLICATE message, +requesting that all writers reply describing their current position in their streams. +Writers reply with a POSITION message.

+

To learn about new facts, readers should listen for RDATA messages and process them to respond to the new fact. +The RDATA itself is not a self-contained representation of the fact; +readers will have to query the stream tables for the full details. +Readers must also advance their record of the writer's current position for that stream.

+

Summary

+

In a nutshell: we have an append-only log with a "buffer/scratchpad" at the end where we have to wait for the sequence to be linear and contiguous.

+
+
1 +

we use the word fact here for two reasons. +Firstly, the word "event" is already heavily overloaded (PDUs, EDUs, account data, ...) and we don't need to make that worse. +Secondly, "fact" emphasises that the things we append to a stream cannot change after the fact.

+
+
2 +

A fact might be expressed with 0 rows, e.g. if we opened a transaction to persist an event, but failed and rolled the transaction back before marking the fact as completed. +In principle a fact might be expressed with 2 or more rows; if so, each of those rows should share the fact's stream ID.

+
+
3 +

This communication used to happen directly with the writers over TCP; +nowadays it's done via Redis's Pubsub.

+
+ +
+ + +
+
+ + + +
+ + + + + + + + + + + + + \ No newline at end of file diff --git a/latest/federate.html b/latest/federate.html index f96c47cbcf..d49f9a2993 100644 --- a/latest/federate.html +++ b/latest/federate.html @@ -76,7 +76,7 @@ diff --git a/latest/index.html b/latest/index.html index a52dd12d95..3afa750990 100644 --- a/latest/index.html +++ b/latest/index.html @@ -76,7 +76,7 @@ diff --git a/latest/jwt.html b/latest/jwt.html index d739c35e4d..4e922b3892 100644 --- a/latest/jwt.html +++ b/latest/jwt.html @@ -76,7 +76,7 @@ diff --git a/latest/log_contexts.html b/latest/log_contexts.html index 9f5ac779b1..928a92eca2 100644 --- a/latest/log_contexts.html +++ b/latest/log_contexts.html @@ -76,7 +76,7 @@ diff --git a/latest/manhole.html b/latest/manhole.html index 2b44f9d8f9..60bc9432b6 100644 --- a/latest/manhole.html +++ b/latest/manhole.html @@ -76,7 +76,7 @@ diff --git a/latest/media_repository.html b/latest/media_repository.html index 7e26d76bab..d0e87c9069 100644 --- a/latest/media_repository.html +++ b/latest/media_repository.html @@ -76,7 +76,7 @@ diff --git a/latest/message_retention_policies.html b/latest/message_retention_policies.html index e8fbed4611..57f1b48a9a 100644 --- a/latest/message_retention_policies.html +++ b/latest/message_retention_policies.html @@ -76,7 +76,7 @@ diff --git a/latest/metrics-howto.html b/latest/metrics-howto.html index b53377c476..c43ce93d88 100644 --- a/latest/metrics-howto.html +++ b/latest/metrics-howto.html @@ -76,7 +76,7 @@ diff --git a/latest/modules/account_data_callbacks.html b/latest/modules/account_data_callbacks.html index 6e45ffcf38..c50b180e92 100644 --- a/latest/modules/account_data_callbacks.html +++ b/latest/modules/account_data_callbacks.html @@ -76,7 +76,7 @@ diff --git a/latest/modules/account_validity_callbacks.html b/latest/modules/account_validity_callbacks.html index a61463342f..543e8d6634 100644 --- a/latest/modules/account_validity_callbacks.html +++ b/latest/modules/account_validity_callbacks.html @@ -76,7 +76,7 @@ diff --git a/latest/modules/background_update_controller_callbacks.html b/latest/modules/background_update_controller_callbacks.html index e954537ec7..4b30f6bc9c 100644 --- a/latest/modules/background_update_controller_callbacks.html +++ b/latest/modules/background_update_controller_callbacks.html @@ -76,7 +76,7 @@ diff --git a/latest/modules/index.html b/latest/modules/index.html index b6c3f98dc5..af253fedf7 100644 --- a/latest/modules/index.html +++ b/latest/modules/index.html @@ -76,7 +76,7 @@ diff --git a/latest/modules/password_auth_provider_callbacks.html b/latest/modules/password_auth_provider_callbacks.html index 0a767725e2..bf3acec439 100644 --- a/latest/modules/password_auth_provider_callbacks.html +++ b/latest/modules/password_auth_provider_callbacks.html @@ -76,7 +76,7 @@ diff --git a/latest/modules/porting_legacy_module.html b/latest/modules/porting_legacy_module.html index 85f3c55082..d803d76acc 100644 --- a/latest/modules/porting_legacy_module.html +++ b/latest/modules/porting_legacy_module.html @@ -76,7 +76,7 @@ diff --git a/latest/modules/presence_router_callbacks.html b/latest/modules/presence_router_callbacks.html index 2747fbebe3..c8d26631a1 100644 --- a/latest/modules/presence_router_callbacks.html +++ b/latest/modules/presence_router_callbacks.html @@ -76,7 +76,7 @@ diff --git a/latest/modules/spam_checker_callbacks.html b/latest/modules/spam_checker_callbacks.html index 3522dc4334..d35145ad4d 100644 --- a/latest/modules/spam_checker_callbacks.html +++ b/latest/modules/spam_checker_callbacks.html @@ -76,7 +76,7 @@ diff --git a/latest/modules/third_party_rules_callbacks.html b/latest/modules/third_party_rules_callbacks.html index 495af9c1b7..bb89d8aa87 100644 --- a/latest/modules/third_party_rules_callbacks.html +++ b/latest/modules/third_party_rules_callbacks.html @@ -76,7 +76,7 @@ diff --git a/latest/modules/writing_a_module.html b/latest/modules/writing_a_module.html index e731074390..8a64cb764f 100644 --- a/latest/modules/writing_a_module.html +++ b/latest/modules/writing_a_module.html @@ -76,7 +76,7 @@ diff --git a/latest/openid.html b/latest/openid.html index 23e95a8eb0..f22812e707 100644 --- a/latest/openid.html +++ b/latest/openid.html @@ -76,7 +76,7 @@ diff --git a/latest/opentracing.html b/latest/opentracing.html index b1184ef398..d7a8a401f1 100644 --- a/latest/opentracing.html +++ b/latest/opentracing.html @@ -76,7 +76,7 @@ diff --git a/latest/other/running_synapse_on_single_board_computers.html b/latest/other/running_synapse_on_single_board_computers.html index 809ebb811f..26f7e9af90 100644 --- a/latest/other/running_synapse_on_single_board_computers.html +++ b/latest/other/running_synapse_on_single_board_computers.html @@ -76,7 +76,7 @@ diff --git a/latest/password_auth_providers.html b/latest/password_auth_providers.html index 4c8014a4b3..33eed556f9 100644 --- a/latest/password_auth_providers.html +++ b/latest/password_auth_providers.html @@ -76,7 +76,7 @@ diff --git a/latest/postgres.html b/latest/postgres.html index 8066c1cb14..f3e98fda6e 100644 --- a/latest/postgres.html +++ b/latest/postgres.html @@ -76,7 +76,7 @@ diff --git a/latest/print.html b/latest/print.html index 1d43f1242e..f5e9a2ec47 100644 --- a/latest/print.html +++ b/latest/print.html @@ -77,7 +77,7 @@ @@ -942,7 +942,7 @@ reverse proxy is using.

} example.com:8448 { - reverse_proxy localhost:8008 + reverse_proxy /_matrix/* localhost:8008 }

Delegation example:

@@ -1779,6 +1779,17 @@ dpkg -i matrix-synapse-py3_1.3.0+stretch1_amd64.deb +

Upgrading to v1.90.0

+

App service query parameter authorization is now a configuration option

+

Synapse v1.81.0 deprecated application service authorization via query parameters as this is +considered insecure - and from Synapse v1.71.0 forwards the application service token has also been sent via +the Authorization header], making the insecure +query parameter authorization redundant. Since removing the ability to continue to use query parameters could break +backwards compatibility it has now been put behind a configuration option, use_appservice_legacy_authorization.
+This option defaults to false, but can be activated by adding

+
use_appservice_legacy_authorization: true 
+
+

to your configuration.

Upgrading to v1.89.0

Removal of unspecced user property for /register

Application services can no longer call /register with a user property to create new users. @@ -4732,6 +4743,14 @@ like sending a federation transaction.

  • max_short_retries: maximum number of retries for the short retry algo. Default to 3 attempts.
  • max_long_retries: maximum number of retries for the long retry algo. Default to 10 attempts.
    • +

    The following options control the retry logic when communicating with a specific homeserver destination. +Unlike the previous configuration options, these values apply across all requests +for a given destination and the state of the backoff is stored in the database.

    +

    Example configuration:

    federation:
   client_timeout: 180s
@@ -4739,6 +4758,9 @@ like sending a federation transaction.
    
   max_long_retry_delay: 100s
   max_short_retries: 5
   max_long_retries: 20
+  destination_min_retry_interval: 30s
+  destination_retry_multiplier: 5
+  destination_max_retry_interval: 12h

    Caching

    @@ -6019,6 +6041,15 @@ Implicitly enables MAU tracking for application service users.

    track_appservice_user_ips: true
    +

    use_appservice_legacy_authorization

    +

    Whether to send the application service access tokens via the access_token query parameter +per older versions of the Matrix specification. Defaults to false. Set to true to enable sending +access tokens via a query parameter.

    +

    **Enabling this option is considered insecure and is not recommended. **

    +

    Example configuration:

    +
    use_appservice_legacy_authorization: true 
+
    +

    macaroon_secret_key

    A secret which is used to sign

    +

    Upgrading to v1.90.0

    +

    App service query parameter authorization is now a configuration option

    +

    Synapse v1.81.0 deprecated application service authorization via query parameters as this is +considered insecure - and from Synapse v1.71.0 forwards the application service token has also been sent via +the Authorization header], making the insecure +query parameter authorization redundant. Since removing the ability to continue to use query parameters could break +backwards compatibility it has now been put behind a configuration option, use_appservice_legacy_authorization.
    +This option defaults to false, but can be activated by adding

    +
    use_appservice_legacy_authorization: true 
+
    +

    to your configuration.

    Upgrading to v1.89.0

    Removal of unspecced user property for /register

    Application services can no longer call /register with a user property to create new users. diff --git a/latest/usage/administration/admin_api/background_updates.html b/latest/usage/administration/admin_api/background_updates.html index 7bc4efe403..7b7b5dc646 100644 --- a/latest/usage/administration/admin_api/background_updates.html +++ b/latest/usage/administration/admin_api/background_updates.html @@ -76,7 +76,7 @@

    diff --git a/latest/usage/administration/admin_api/federation.html b/latest/usage/administration/admin_api/federation.html index 1902d4a142..5eaa154eac 100644 --- a/latest/usage/administration/admin_api/federation.html +++ b/latest/usage/administration/admin_api/federation.html @@ -76,7 +76,7 @@ diff --git a/latest/usage/administration/admin_api/index.html b/latest/usage/administration/admin_api/index.html index 0ac2e7fcd8..9fddd06fda 100644 --- a/latest/usage/administration/admin_api/index.html +++ b/latest/usage/administration/admin_api/index.html @@ -76,7 +76,7 @@ diff --git a/latest/usage/administration/admin_api/registration_tokens.html b/latest/usage/administration/admin_api/registration_tokens.html index 84fb365570..d7f1fbe22b 100644 --- a/latest/usage/administration/admin_api/registration_tokens.html +++ b/latest/usage/administration/admin_api/registration_tokens.html @@ -76,7 +76,7 @@ diff --git a/latest/usage/administration/admin_faq.html b/latest/usage/administration/admin_faq.html index d5f891e668..0cdca99700 100644 --- a/latest/usage/administration/admin_faq.html +++ b/latest/usage/administration/admin_faq.html @@ -76,7 +76,7 @@ diff --git a/latest/usage/administration/database_maintenance_tools.html b/latest/usage/administration/database_maintenance_tools.html index f949c57ecc..b613b01a4c 100644 --- a/latest/usage/administration/database_maintenance_tools.html +++ b/latest/usage/administration/database_maintenance_tools.html @@ -76,7 +76,7 @@ diff --git a/latest/usage/administration/index.html b/latest/usage/administration/index.html index c99082bfdc..5e9d90d7c0 100644 --- a/latest/usage/administration/index.html +++ b/latest/usage/administration/index.html @@ -76,7 +76,7 @@ diff --git a/latest/usage/administration/monitoring/reporting_homeserver_usage_statistics.html b/latest/usage/administration/monitoring/reporting_homeserver_usage_statistics.html index 5c6073ae96..de414dcf85 100644 --- a/latest/usage/administration/monitoring/reporting_homeserver_usage_statistics.html +++ b/latest/usage/administration/monitoring/reporting_homeserver_usage_statistics.html @@ -76,7 +76,7 @@ diff --git a/latest/usage/administration/monthly_active_users.html b/latest/usage/administration/monthly_active_users.html index d189460792..b491490d72 100644 --- a/latest/usage/administration/monthly_active_users.html +++ b/latest/usage/administration/monthly_active_users.html @@ -76,7 +76,7 @@ diff --git a/latest/usage/administration/request_log.html b/latest/usage/administration/request_log.html index d122fc900b..91ff155bcd 100644 --- a/latest/usage/administration/request_log.html +++ b/latest/usage/administration/request_log.html @@ -76,7 +76,7 @@ diff --git a/latest/usage/administration/state_groups.html b/latest/usage/administration/state_groups.html index ee51fd673f..3c0b912629 100644 --- a/latest/usage/administration/state_groups.html +++ b/latest/usage/administration/state_groups.html @@ -76,7 +76,7 @@ diff --git a/latest/usage/administration/understanding_synapse_through_grafana_graphs.html b/latest/usage/administration/understanding_synapse_through_grafana_graphs.html index 76f188a7b4..f1b92356c0 100644 --- a/latest/usage/administration/understanding_synapse_through_grafana_graphs.html +++ b/latest/usage/administration/understanding_synapse_through_grafana_graphs.html @@ -76,7 +76,7 @@ diff --git a/latest/usage/administration/useful_sql_for_admins.html b/latest/usage/administration/useful_sql_for_admins.html index 69434ed404..2453388d11 100644 --- a/latest/usage/administration/useful_sql_for_admins.html +++ b/latest/usage/administration/useful_sql_for_admins.html @@ -76,7 +76,7 @@ diff --git a/latest/usage/configuration/config_documentation.html b/latest/usage/configuration/config_documentation.html index 0b8eddf339..1719e87a65 100644 --- a/latest/usage/configuration/config_documentation.html +++ b/latest/usage/configuration/config_documentation.html @@ -76,7 +76,7 @@ @@ -1206,6 +1206,14 @@ like sending a federation transaction.

  • max_short_retries: maximum number of retries for the short retry algo. Default to 3 attempts.
  • max_long_retries: maximum number of retries for the long retry algo. Default to 10 attempts.
    • +

    The following options control the retry logic when communicating with a specific homeserver destination. +Unlike the previous configuration options, these values apply across all requests +for a given destination and the state of the backoff is stored in the database.

    +

    Example configuration:

    federation:
   client_timeout: 180s
@@ -1213,6 +1221,9 @@ like sending a federation transaction.
    
   max_long_retry_delay: 100s
   max_short_retries: 5
   max_long_retries: 20
+  destination_min_retry_interval: 30s
+  destination_retry_multiplier: 5
+  destination_max_retry_interval: 12h

    Caching

    @@ -2493,6 +2504,15 @@ Implicitly enables MAU tracking for application service users.

    track_appservice_user_ips: true
    +

    use_appservice_legacy_authorization

    +

    Whether to send the application service access tokens via the access_token query parameter +per older versions of the Matrix specification. Defaults to false. Set to true to enable sending +access tokens via a query parameter.

    +

    **Enabling this option is considered insecure and is not recommended. **

    +

    Example configuration:

    +
    use_appservice_legacy_authorization: true 
+
    +

    macaroon_secret_key

    A secret which is used to sign