From 8a87b4435a736cd42454cad7e57b65ec911f01fa Mon Sep 17 00:00:00 2001
From: Sean Quah <8349537+squahtx@users.noreply.github.com>
Date: Mon, 25 Apr 2022 19:39:17 +0100
Subject: [PATCH 01/12] Handle cancellation in
 `EventsWorkerStore._get_events_from_cache_or_db` (#12529)

Multiple calls to `EventsWorkerStore._get_events_from_cache_or_db` can
reuse the same database fetch, which is initiated by the first call.
Ensure that cancelling the first call doesn't cancel the other calls
sharing the same database fetch.

Signed-off-by: Sean Quah <seanq@element.io>
---
 changelog.d/12529.misc                        |   1 +
 .../storage/databases/main/events_worker.py   |  79 +++++++-----
 .../databases/main/test_events_worker.py      | 121 +++++++++++++++++-
 3 files changed, 167 insertions(+), 34 deletions(-)
 create mode 100644 changelog.d/12529.misc

diff --git a/changelog.d/12529.misc b/changelog.d/12529.misc
new file mode 100644
index 0000000000..5427108742
--- /dev/null
+++ b/changelog.d/12529.misc
@@ -0,0 +1 @@
+Handle cancellation in `EventsWorkerStore._get_events_from_cache_or_db`.
diff --git a/synapse/storage/databases/main/events_worker.py b/synapse/storage/databases/main/events_worker.py
index 6d6e146ff1..c31fc00eaa 100644
--- a/synapse/storage/databases/main/events_worker.py
+++ b/synapse/storage/databases/main/events_worker.py
@@ -75,7 +75,7 @@ from synapse.storage.util.id_generators import (
 from synapse.storage.util.sequence import build_sequence_generator
 from synapse.types import JsonDict, get_domain_from_id
 from synapse.util import unwrapFirstError
-from synapse.util.async_helpers import ObservableDeferred
+from synapse.util.async_helpers import ObservableDeferred, delay_cancellation
 from synapse.util.caches.descriptors import cached, cachedList
 from synapse.util.caches.lrucache import LruCache
 from synapse.util.iterutils import batch_iter
@@ -640,42 +640,57 @@ class EventsWorkerStore(SQLBaseStore):
         missing_events_ids.difference_update(already_fetching_ids)
 
         if missing_events_ids:
-            log_ctx = current_context()
-            log_ctx.record_event_fetch(len(missing_events_ids))
 
-            # Add entries to `self._current_event_fetches` for each event we're
-            # going to pull from the DB. We use a single deferred that resolves
-            # to all the events we pulled from the DB (this will result in this
-            # function returning more events than requested, but that can happen
-            # already due to `_get_events_from_db`).
-            fetching_deferred: ObservableDeferred[
-                Dict[str, EventCacheEntry]
-            ] = ObservableDeferred(defer.Deferred(), consumeErrors=True)
-            for event_id in missing_events_ids:
-                self._current_event_fetches[event_id] = fetching_deferred
+            async def get_missing_events_from_db() -> Dict[str, EventCacheEntry]:
+                """Fetches the events in `missing_event_ids` from the database.
 
-            # Note that _get_events_from_db is also responsible for turning db rows
-            # into FrozenEvents (via _get_event_from_row), which involves seeing if
-            # the events have been redacted, and if so pulling the redaction event out
-            # of the database to check it.
-            #
-            try:
-                missing_events = await self._get_events_from_db(
-                    missing_events_ids,
-                )
+                Also creates entries in `self._current_event_fetches` to allow
+                concurrent `_get_events_from_cache_or_db` calls to reuse the same fetch.
+                """
+                log_ctx = current_context()
+                log_ctx.record_event_fetch(len(missing_events_ids))
 
-                event_entry_map.update(missing_events)
-            except Exception as e:
-                with PreserveLoggingContext():
-                    fetching_deferred.errback(e)
-                raise e
-            finally:
-                # Ensure that we mark these events as no longer being fetched.
+                # Add entries to `self._current_event_fetches` for each event we're
+                # going to pull from the DB. We use a single deferred that resolves
+                # to all the events we pulled from the DB (this will result in this
+                # function returning more events than requested, but that can happen
+                # already due to `_get_events_from_db`).
+                fetching_deferred: ObservableDeferred[
+                    Dict[str, EventCacheEntry]
+                ] = ObservableDeferred(defer.Deferred(), consumeErrors=True)
                 for event_id in missing_events_ids:
-                    self._current_event_fetches.pop(event_id, None)
+                    self._current_event_fetches[event_id] = fetching_deferred
 
-            with PreserveLoggingContext():
-                fetching_deferred.callback(missing_events)
+                # Note that _get_events_from_db is also responsible for turning db rows
+                # into FrozenEvents (via _get_event_from_row), which involves seeing if
+                # the events have been redacted, and if so pulling the redaction event
+                # out of the database to check it.
+                #
+                try:
+                    missing_events = await self._get_events_from_db(
+                        missing_events_ids,
+                    )
+                except Exception as e:
+                    with PreserveLoggingContext():
+                        fetching_deferred.errback(e)
+                    raise e
+                finally:
+                    # Ensure that we mark these events as no longer being fetched.
+                    for event_id in missing_events_ids:
+                        self._current_event_fetches.pop(event_id, None)
+
+                with PreserveLoggingContext():
+                    fetching_deferred.callback(missing_events)
+
+                return missing_events
+
+            # We must allow the database fetch to complete in the presence of
+            # cancellations, since multiple `_get_events_from_cache_or_db` calls can
+            # reuse the same fetch.
+            missing_events: Dict[str, EventCacheEntry] = await delay_cancellation(
+                get_missing_events_from_db()
+            )
+            event_entry_map.update(missing_events)
 
         if already_fetching_deferreds:
             # Wait for the other event requests to finish and add their results
diff --git a/tests/storage/databases/main/test_events_worker.py b/tests/storage/databases/main/test_events_worker.py
index 1f6a9eb07b..bf6374f93d 100644
--- a/tests/storage/databases/main/test_events_worker.py
+++ b/tests/storage/databases/main/test_events_worker.py
@@ -13,10 +13,11 @@
 # limitations under the License.
 import json
 from contextlib import contextmanager
-from typing import Generator
+from typing import Generator, Tuple
+from unittest import mock
 
 from twisted.enterprise.adbapi import ConnectionPool
-from twisted.internet.defer import ensureDeferred
+from twisted.internet.defer import CancelledError, Deferred, ensureDeferred
 from twisted.test.proto_helpers import MemoryReactor
 
 from synapse.api.room_versions import EventFormatVersions, RoomVersions
@@ -281,3 +282,119 @@ class DatabaseOutageTestCase(unittest.HomeserverTestCase):
 
         # This next event fetch should succeed
         self.get_success(self.store.get_event(self.event_ids[0]))
+
+
+class GetEventCancellationTestCase(unittest.HomeserverTestCase):
+    """Test cancellation of `get_event` calls."""
+
+    servlets = [
+        admin.register_servlets,
+        room.register_servlets,
+        login.register_servlets,
+    ]
+
+    def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer):
+        self.store: EventsWorkerStore = hs.get_datastores().main
+
+        self.user = self.register_user("user", "pass")
+        self.token = self.login(self.user, "pass")
+
+        self.room = self.helper.create_room_as(self.user, tok=self.token)
+
+        res = self.helper.send(self.room, tok=self.token)
+        self.event_id = res["event_id"]
+
+        # Reset the event cache so the tests start with it empty
+        self.store._get_event_cache.clear()
+
+    @contextmanager
+    def blocking_get_event_calls(
+        self,
+    ) -> Generator[
+        Tuple["Deferred[None]", "Deferred[None]", "Deferred[None]"], None, None
+    ]:
+        """Starts two concurrent `get_event` calls for the same event.
+
+        Both `get_event` calls will use the same database fetch, which will be blocked
+        at the time this function returns.
+
+        Returns:
+            A tuple containing:
+             * A `Deferred` that unblocks the database fetch.
+             * A cancellable `Deferred` for the first `get_event` call.
+             * A cancellable `Deferred` for the second `get_event` call.
+        """
+        # Patch `DatabasePool.runWithConnection` to block.
+        unblock: "Deferred[None]" = Deferred()
+        original_runWithConnection = self.store.db_pool.runWithConnection
+
+        async def runWithConnection(*args, **kwargs):
+            await unblock
+            return await original_runWithConnection(*args, **kwargs)
+
+        with mock.patch.object(
+            self.store.db_pool,
+            "runWithConnection",
+            new=runWithConnection,
+        ):
+            ctx1 = LoggingContext("get_event1")
+            ctx2 = LoggingContext("get_event2")
+
+            async def get_event(ctx: LoggingContext) -> None:
+                with ctx:
+                    await self.store.get_event(self.event_id)
+
+            get_event1 = ensureDeferred(get_event(ctx1))
+            get_event2 = ensureDeferred(get_event(ctx2))
+
+            # Both `get_event` calls ought to be blocked.
+            self.assertNoResult(get_event1)
+            self.assertNoResult(get_event2)
+
+            yield unblock, get_event1, get_event2
+
+        # Confirm that the two `get_event` calls shared the same database fetch.
+        self.assertEqual(ctx1.get_resource_usage().evt_db_fetch_count, 1)
+        self.assertEqual(ctx2.get_resource_usage().evt_db_fetch_count, 0)
+
+    def test_first_get_event_cancelled(self):
+        """Test cancellation of the first `get_event` call sharing a database fetch.
+
+        The first `get_event` call is the one which initiates the fetch. We expect the
+        fetch to complete despite the cancellation. Furthermore, the first `get_event`
+        call must not abort before the fetch is complete, otherwise the fetch will be
+        using a finished logging context.
+        """
+        with self.blocking_get_event_calls() as (unblock, get_event1, get_event2):
+            # Cancel the first `get_event` call.
+            get_event1.cancel()
+            # The first `get_event` call must not abort immediately, otherwise its
+            # logging context will be finished while it is still in use by the database
+            # fetch.
+            self.assertNoResult(get_event1)
+            # The second `get_event` call must not be cancelled.
+            self.assertNoResult(get_event2)
+
+            # Unblock the database fetch.
+            unblock.callback(None)
+            # A `CancelledError` should be raised out of the first `get_event` call.
+            exc = self.get_failure(get_event1, CancelledError).value
+            self.assertIsInstance(exc, CancelledError)
+            # The second `get_event` call should complete successfully.
+            self.get_success(get_event2)
+
+    def test_second_get_event_cancelled(self):
+        """Test cancellation of the second `get_event` call sharing a database fetch."""
+        with self.blocking_get_event_calls() as (unblock, get_event1, get_event2):
+            # Cancel the second `get_event` call.
+            get_event2.cancel()
+            # The first `get_event` call must not be cancelled.
+            self.assertNoResult(get_event1)
+            # The second `get_event` call gets cancelled immediately.
+            exc = self.get_failure(get_event2, CancelledError).value
+            self.assertIsInstance(exc, CancelledError)
+
+            # Unblock the database fetch.
+            unblock.callback(None)
+            # The first `get_event` call should complete successfully.
+            self.get_success(get_event1)

From e75c7e3b6ddc25b76b4eabf10eca1e923048ae43 Mon Sep 17 00:00:00 2001
From: Shay <hillerys@element.io>
Date: Mon, 25 Apr 2022 11:43:59 -0700
Subject: [PATCH 02/12] Add a table of contents to config manual  (#12527)

* Update config_documentation.md
---
 changelog.d/12527.doc                         |  2 +
 .../configuration/config_documentation.md     | 46 ++++++++++++++++++-
 2 files changed, 46 insertions(+), 2 deletions(-)
 create mode 100644 changelog.d/12527.doc

diff --git a/changelog.d/12527.doc b/changelog.d/12527.doc
new file mode 100644
index 0000000000..e6907321e7
--- /dev/null
+++ b/changelog.d/12527.doc
@@ -0,0 +1,2 @@
+Add an index to the configuration manual.
+
diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md
index 9c864af6ec..968b0fbfaf 100644
--- a/docs/usage/configuration/config_documentation.md
+++ b/docs/usage/configuration/config_documentation.md
@@ -64,7 +64,49 @@ apply if you want your config file to be read properly. A few helpful things to
   In addition, each setting has an example of its usage, with the proper indentation
   shown. 
 
-  
+## Contents
+[Modules](#modules)
+
+[Server](#server)
+
+[Homeserver Blocking](#homeserver-blocking)
+
+[TLS](#tls)
+
+[Federation](#federation)
+
+[Caching](#caching)
+
+[Database](#database)
+
+[Logging](#logging)
+
+[Ratelimiting](#ratelimiting)
+
+[Media Store](#media-store)
+
+[Captcha](#captcha)
+
+[TURN](#turn)
+
+[Registration](#registration)
+
+[API Configuration](#api-configuration)
+
+[Signing Keys](#signing-keys)
+
+[Single Sign On Integration](#single-sign-on-integration)
+
+[Push](#push)
+
+[Rooms](#rooms)
+
+[Opentracing](#opentracing)
+
+[Workers](#workers)
+
+[Background Updates](#background-updates)
+
 ## Modules
 
 Server admins can expand Synapse's functionality with external modules.
@@ -3409,4 +3451,4 @@ background_updates:
     sleep_duration_ms: 300
     min_batch_size: 10
     default_batch_size: 50
-```
\ No newline at end of file
+```

From 17d99f758a768b886842cf496ff236fbe3829236 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com>
Date: Tue, 26 Apr 2022 10:27:11 +0100
Subject: [PATCH 03/12] Optimise backfill calculation (#12522)

Try to avoid an OOM by checking fewer extremities.

Generally this is a big rewrite of _maybe_backfill, to try and fix some of the TODOs and other problems in it. It's best reviewed commit-by-commit.
---
 changelog.d/12522.bugfix                      |   1 +
 synapse/handlers/federation.py                | 234 +++++++++++-------
 synapse/handlers/room_batch.py                |   2 +-
 .../databases/main/event_federation.py        |  30 ++-
 synapse/visibility.py                         |   7 +
 5 files changed, 168 insertions(+), 106 deletions(-)
 create mode 100644 changelog.d/12522.bugfix

diff --git a/changelog.d/12522.bugfix b/changelog.d/12522.bugfix
new file mode 100644
index 0000000000..2220f05ceb
--- /dev/null
+++ b/changelog.d/12522.bugfix
@@ -0,0 +1 @@
+Fix a bug introduced in Synapse 0.99.3 which could cause Synapse to consume large amounts of RAM when back-paginating in a large room.
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index 1434e99056..d2ba70a814 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -1,4 +1,4 @@
-# Copyright 2014-2021 The Matrix.org Foundation C.I.C.
+# Copyright 2014-2022 The Matrix.org Foundation C.I.C.
 # Copyright 2020 Sorunome
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,10 +15,14 @@
 
 """Contains handlers for federation events."""
 
+import enum
+import itertools
 import logging
+from enum import Enum
 from http import HTTPStatus
 from typing import TYPE_CHECKING, Dict, Iterable, List, Optional, Tuple, Union
 
+import attr
 from signedjson.key import decode_verify_key_bytes
 from signedjson.sign import verify_signed_json
 from unpaddedbase64 import decode_base64
@@ -92,6 +96,24 @@ def get_domains_from_state(state: StateMap[EventBase]) -> List[Tuple[str, int]]:
     return sorted(joined_domains.items(), key=lambda d: d[1])
 
 
+class _BackfillPointType(Enum):
+    # a regular backwards extremity (ie, an event which we don't yet have, but which
+    # is referred to by other events in the DAG)
+    BACKWARDS_EXTREMITY = enum.auto()
+
+    # an MSC2716 "insertion event"
+    INSERTION_PONT = enum.auto()
+
+
+@attr.s(slots=True, auto_attribs=True, frozen=True)
+class _BackfillPoint:
+    """A potential point we might backfill from"""
+
+    event_id: str
+    depth: int
+    type: _BackfillPointType
+
+
 class FederationHandler:
     """Handles general incoming federation requests
 
@@ -157,89 +179,51 @@ class FederationHandler:
     async def _maybe_backfill_inner(
         self, room_id: str, current_depth: int, limit: int
     ) -> bool:
-        oldest_events_with_depth = (
-            await self.store.get_oldest_event_ids_with_depth_in_room(room_id)
-        )
+        backwards_extremities = [
+            _BackfillPoint(event_id, depth, _BackfillPointType.BACKWARDS_EXTREMITY)
+            for event_id, depth in await self.store.get_oldest_event_ids_with_depth_in_room(
+                room_id
+            )
+        ]
 
-        insertion_events_to_be_backfilled: Dict[str, int] = {}
+        insertion_events_to_be_backfilled: List[_BackfillPoint] = []
         if self.hs.config.experimental.msc2716_enabled:
-            insertion_events_to_be_backfilled = (
-                await self.store.get_insertion_event_backward_extremities_in_room(
+            insertion_events_to_be_backfilled = [
+                _BackfillPoint(event_id, depth, _BackfillPointType.INSERTION_PONT)
+                for event_id, depth in await self.store.get_insertion_event_backward_extremities_in_room(
                     room_id
                 )
-            )
+            ]
         logger.debug(
-            "_maybe_backfill_inner: extremities oldest_events_with_depth=%s insertion_events_to_be_backfilled=%s",
-            oldest_events_with_depth,
+            "_maybe_backfill_inner: backwards_extremities=%s insertion_events_to_be_backfilled=%s",
+            backwards_extremities,
             insertion_events_to_be_backfilled,
         )
 
-        if not oldest_events_with_depth and not insertion_events_to_be_backfilled:
+        if not backwards_extremities and not insertion_events_to_be_backfilled:
             logger.debug("Not backfilling as no extremeties found.")
             return False
 
-        # We only want to paginate if we can actually see the events we'll get,
-        # as otherwise we'll just spend a lot of resources to get redacted
-        # events.
-        #
-        # We do this by filtering all the backwards extremities and seeing if
-        # any remain. Given we don't have the extremity events themselves, we
-        # need to actually check the events that reference them.
-        #
-        # *Note*: the spec wants us to keep backfilling until we reach the start
-        # of the room in case we are allowed to see some of the history. However
-        # in practice that causes more issues than its worth, as a) its
-        # relatively rare for there to be any visible history and b) even when
-        # there is its often sufficiently long ago that clients would stop
-        # attempting to paginate before backfill reached the visible history.
-        #
-        # TODO: If we do do a backfill then we should filter the backwards
-        #   extremities to only include those that point to visible portions of
-        #   history.
-        #
-        # TODO: Correctly handle the case where we are allowed to see the
-        #   forward event but not the backward extremity, e.g. in the case of
-        #   initial join of the server where we are allowed to see the join
-        #   event but not anything before it. This would require looking at the
-        #   state *before* the event, ignoring the special casing certain event
-        #   types have.
-
-        forward_event_ids = await self.store.get_successor_events(
-            list(oldest_events_with_depth)
+        # we now have a list of potential places to backpaginate from. We prefer to
+        # start with the most recent (ie, max depth), so let's sort the list.
+        sorted_backfill_points: List[_BackfillPoint] = sorted(
+            itertools.chain(
+                backwards_extremities,
+                insertion_events_to_be_backfilled,
+            ),
+            key=lambda e: -int(e.depth),
         )
 
-        extremities_events = await self.store.get_events(
-            forward_event_ids,
-            redact_behaviour=EventRedactBehaviour.AS_IS,
-            get_prev_content=False,
-        )
-
-        # We set `check_history_visibility_only` as we might otherwise get false
-        # positives from users having been erased.
-        filtered_extremities = await filter_events_for_server(
-            self.storage,
-            self.server_name,
-            list(extremities_events.values()),
-            redact=False,
-            check_history_visibility_only=True,
-        )
         logger.debug(
-            "_maybe_backfill_inner: filtered_extremities %s", filtered_extremities
+            "_maybe_backfill_inner: room_id: %s: current_depth: %s, limit: %s, "
+            "backfill points (%d): %s",
+            room_id,
+            current_depth,
+            limit,
+            len(sorted_backfill_points),
+            sorted_backfill_points,
         )
 
-        if not filtered_extremities and not insertion_events_to_be_backfilled:
-            return False
-
-        extremities = {
-            **oldest_events_with_depth,
-            # TODO: insertion_events_to_be_backfilled is currently skipping the filtered_extremities checks
-            **insertion_events_to_be_backfilled,
-        }
-
-        # Check if we reached a point where we should start backfilling.
-        sorted_extremeties_tuple = sorted(extremities.items(), key=lambda e: -int(e[1]))
-        max_depth = sorted_extremeties_tuple[0][1]
-
         # If we're approaching an extremity we trigger a backfill, otherwise we
         # no-op.
         #
@@ -249,6 +233,11 @@ class FederationHandler:
         # chose more than one times the limit in case of failure, but choosing a
         # much larger factor will result in triggering a backfill request much
         # earlier than necessary.
+        #
+        # XXX: shouldn't we do this *after* the filter by depth below? Again, we don't
+        # care about events that have happened after our current position.
+        #
+        max_depth = sorted_backfill_points[0].depth
         if current_depth - 2 * limit > max_depth:
             logger.debug(
                 "Not backfilling as we don't need to. %d < %d - 2 * %d",
@@ -265,31 +254,98 @@ class FederationHandler:
         #    2. we have likely previously tried and failed to backfill from that
         #       extremity, so to avoid getting "stuck" requesting the same
         #       backfill repeatedly we drop those extremities.
-        filtered_sorted_extremeties_tuple = [
-            t for t in sorted_extremeties_tuple if int(t[1]) <= current_depth
-        ]
-
-        logger.debug(
-            "room_id: %s, backfill: current_depth: %s, limit: %s, max_depth: %s, extrems (%d): %s filtered_sorted_extremeties_tuple: %s",
-            room_id,
-            current_depth,
-            limit,
-            max_depth,
-            len(sorted_extremeties_tuple),
-            sorted_extremeties_tuple,
-            filtered_sorted_extremeties_tuple,
-        )
-
+        #
         # However, we need to check that the filtered extremities are non-empty.
         # If they are empty then either we can a) bail or b) still attempt to
         # backfill. We opt to try backfilling anyway just in case we do get
         # relevant events.
-        if filtered_sorted_extremeties_tuple:
-            sorted_extremeties_tuple = filtered_sorted_extremeties_tuple
+        #
+        filtered_sorted_backfill_points = [
+            t for t in sorted_backfill_points if t.depth <= current_depth
+        ]
+        if filtered_sorted_backfill_points:
+            logger.debug(
+                "_maybe_backfill_inner: backfill points before current depth: %s",
+                filtered_sorted_backfill_points,
+            )
+            sorted_backfill_points = filtered_sorted_backfill_points
+        else:
+            logger.debug(
+                "_maybe_backfill_inner: all backfill points are *after* current depth. Backfilling anyway."
+            )
 
-        # We don't want to specify too many extremities as it causes the backfill
-        # request URI to be too long.
-        extremities = dict(sorted_extremeties_tuple[:5])
+        # For performance's sake, we only want to paginate from a particular extremity
+        # if we can actually see the events we'll get. Otherwise, we'd just spend a lot
+        # of resources to get redacted events. We check each extremity in turn and
+        # ignore those which users on our server wouldn't be able to see.
+        #
+        # Additionally, we limit ourselves to backfilling from at most 5 extremities,
+        # for two reasons:
+        #
+        # - The check which determines if we can see an extremity's events can be
+        #   expensive (we load the full state for the room at each of the backfill
+        #   points, or (worse) their successors)
+        # - We want to avoid the server-server API request URI becoming too long.
+        #
+        # *Note*: the spec wants us to keep backfilling until we reach the start
+        # of the room in case we are allowed to see some of the history. However,
+        # in practice that causes more issues than its worth, as (a) it's
+        # relatively rare for there to be any visible history and (b) even when
+        # there is it's often sufficiently long ago that clients would stop
+        # attempting to paginate before backfill reached the visible history.
+
+        extremities_to_request: List[str] = []
+        for bp in sorted_backfill_points:
+            if len(extremities_to_request) >= 5:
+                break
+
+            # For regular backwards extremities, we don't have the extremity events
+            # themselves, so we need to actually check the events that reference them -
+            # their "successor" events.
+            #
+            # TODO: Correctly handle the case where we are allowed to see the
+            #   successor event but not the backward extremity, e.g. in the case of
+            #   initial join of the server where we are allowed to see the join
+            #   event but not anything before it. This would require looking at the
+            #   state *before* the event, ignoring the special casing certain event
+            #   types have.
+            if bp.type == _BackfillPointType.INSERTION_PONT:
+                event_ids_to_check = [bp.event_id]
+            else:
+                event_ids_to_check = await self.store.get_successor_events(bp.event_id)
+
+            events_to_check = await self.store.get_events_as_list(
+                event_ids_to_check,
+                redact_behaviour=EventRedactBehaviour.AS_IS,
+                get_prev_content=False,
+            )
+
+            # We set `check_history_visibility_only` as we might otherwise get false
+            # positives from users having been erased.
+            filtered_extremities = await filter_events_for_server(
+                self.storage,
+                self.server_name,
+                events_to_check,
+                redact=False,
+                check_history_visibility_only=True,
+            )
+            if filtered_extremities:
+                extremities_to_request.append(bp.event_id)
+            else:
+                logger.debug(
+                    "_maybe_backfill_inner: skipping extremity %s as it would not be visible",
+                    bp,
+                )
+
+        if not extremities_to_request:
+            logger.debug(
+                "_maybe_backfill_inner: found no extremities which would be visible"
+            )
+            return False
+
+        logger.debug(
+            "_maybe_backfill_inner: extremities_to_request %s", extremities_to_request
+        )
 
         # Now we need to decide which hosts to hit first.
 
@@ -309,7 +365,7 @@ class FederationHandler:
             for dom in domains:
                 try:
                     await self._federation_event_handler.backfill(
-                        dom, room_id, limit=100, extremities=extremities
+                        dom, room_id, limit=100, extremities=extremities_to_request
                     )
                     # If this succeeded then we probably already have the
                     # appropriate stuff.
diff --git a/synapse/handlers/room_batch.py b/synapse/handlers/room_batch.py
index 78e299d3a5..29de7e5bed 100644
--- a/synapse/handlers/room_batch.py
+++ b/synapse/handlers/room_batch.py
@@ -54,7 +54,7 @@ class RoomBatchHandler:
         # it has a larger `depth` but before the successor event because the `stream_ordering`
         # is negative before the successor event.
         successor_event_ids = await self.store.get_successor_events(
-            [most_recent_prev_event_id]
+            most_recent_prev_event_id
         )
 
         # If we can't find any successor events, then it's a forward extremity of
diff --git a/synapse/storage/databases/main/event_federation.py b/synapse/storage/databases/main/event_federation.py
index 634e19e035..4710224708 100644
--- a/synapse/storage/databases/main/event_federation.py
+++ b/synapse/storage/databases/main/event_federation.py
@@ -695,7 +695,9 @@ class EventFederationWorkerStore(SignatureWorkerStore, EventsWorkerStore, SQLBas
         # Return all events where not all sets can reach them.
         return {eid for eid, n in event_to_missing_sets.items() if n}
 
-    async def get_oldest_event_ids_with_depth_in_room(self, room_id) -> Dict[str, int]:
+    async def get_oldest_event_ids_with_depth_in_room(
+        self, room_id
+    ) -> List[Tuple[str, int]]:
         """Gets the oldest events(backwards extremities) in the room along with the
         aproximate depth.
 
@@ -708,7 +710,7 @@ class EventFederationWorkerStore(SignatureWorkerStore, EventsWorkerStore, SQLBas
             room_id: Room where we want to find the oldest events
 
         Returns:
-            Map from event_id to depth
+            List of (event_id, depth) tuples
         """
 
         def get_oldest_event_ids_with_depth_in_room_txn(txn, room_id):
@@ -741,7 +743,7 @@ class EventFederationWorkerStore(SignatureWorkerStore, EventsWorkerStore, SQLBas
 
             txn.execute(sql, (room_id, False))
 
-            return dict(txn)
+            return txn.fetchall()
 
         return await self.db_pool.runInteraction(
             "get_oldest_event_ids_with_depth_in_room",
@@ -751,7 +753,7 @@ class EventFederationWorkerStore(SignatureWorkerStore, EventsWorkerStore, SQLBas
 
     async def get_insertion_event_backward_extremities_in_room(
         self, room_id
-    ) -> Dict[str, int]:
+    ) -> List[Tuple[str, int]]:
         """Get the insertion events we know about that we haven't backfilled yet.
 
         We use this function so that we can compare and see if someones current
@@ -763,7 +765,7 @@ class EventFederationWorkerStore(SignatureWorkerStore, EventsWorkerStore, SQLBas
             room_id: Room where we want to find the oldest events
 
         Returns:
-            Map from event_id to depth
+            List of (event_id, depth) tuples
         """
 
         def get_insertion_event_backward_extremities_in_room_txn(txn, room_id):
@@ -778,8 +780,7 @@ class EventFederationWorkerStore(SignatureWorkerStore, EventsWorkerStore, SQLBas
             """
 
             txn.execute(sql, (room_id,))
-
-            return dict(txn)
+            return txn.fetchall()
 
         return await self.db_pool.runInteraction(
             "get_insertion_event_backward_extremities_in_room",
@@ -1295,22 +1296,19 @@ class EventFederationWorkerStore(SignatureWorkerStore, EventsWorkerStore, SQLBas
         event_results.reverse()
         return event_results
 
-    async def get_successor_events(self, event_ids: Iterable[str]) -> List[str]:
-        """Fetch all events that have the given events as a prev event
+    async def get_successor_events(self, event_id: str) -> List[str]:
+        """Fetch all events that have the given event as a prev event
 
         Args:
-            event_ids: The events to use as the previous events.
+            event_id: The event to search for as a prev_event.
         """
-        rows = await self.db_pool.simple_select_many_batch(
+        return await self.db_pool.simple_select_onecol(
             table="event_edges",
-            column="prev_event_id",
-            iterable=event_ids,
-            retcols=("event_id",),
+            keyvalues={"prev_event_id": event_id},
+            retcol="event_id",
             desc="get_successor_events",
         )
 
-        return [row["event_id"] for row in rows]
-
     @wrap_as_background_process("delete_old_forward_extrem_cache")
     async def _delete_old_forward_extrem_cache(self) -> None:
         def _delete_old_forward_extrem_cache_txn(txn):
diff --git a/synapse/visibility.py b/synapse/visibility.py
index 250f073597..de6d2ffc52 100644
--- a/synapse/visibility.py
+++ b/synapse/visibility.py
@@ -419,6 +419,13 @@ async def _event_to_memberships(
         return {}
 
     # for each event, get the event_ids of the membership state at those events.
+    #
+    # TODO: this means that we request the entire membership list. If there  are only
+    #   one or two users on this server, and the room is huge, this is very wasteful
+    #   (it means more db work, and churns the *stateGroupMembersCache*).
+    #   It might be that we could extend StateFilter to specify "give me keys matching
+    #   *:<server_name>", to avoid this.
+
     event_to_state_ids = await storage.state.get_state_ids_for_events(
         frozenset(e.event_id for e in events),
         state_filter=StateFilter.from_types(types=((EventTypes.Member, None),)),

From 99ab45423a0677ea816eb4e81c28f46dd22fc76c Mon Sep 17 00:00:00 2001
From: Shay <hillerys@element.io>
Date: Tue, 26 Apr 2022 02:34:59 -0700
Subject: [PATCH 04/12] build debian package for jammy jellyfish (#12543)

---
 changelog.d/12543.misc               | 1 +
 scripts-dev/build_debian_packages.py | 1 +
 2 files changed, 2 insertions(+)
 create mode 100644 changelog.d/12543.misc

diff --git a/changelog.d/12543.misc b/changelog.d/12543.misc
new file mode 100644
index 0000000000..eed7a6973c
--- /dev/null
+++ b/changelog.d/12543.misc
@@ -0,0 +1 @@
+Build debian packages for Ubuntu 22.04 "Jammy Jellyfish".
diff --git a/scripts-dev/build_debian_packages.py b/scripts-dev/build_debian_packages.py
index 7ff96a1ee6..e3e6878686 100755
--- a/scripts-dev/build_debian_packages.py
+++ b/scripts-dev/build_debian_packages.py
@@ -26,6 +26,7 @@ DISTS = (
     "debian:sid",
     "ubuntu:focal",  # 20.04 LTS (our EOL forced by Py38 on 2024-10-14)
     "ubuntu:impish",  # 21.10  (EOL 2022-07)
+    "ubuntu:jammy",  # 22.04 LTS (EOL 2027-04)
 )
 
 DESC = """\

From 730fcda5463144f09ff1b34198c678021469ee28 Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Tue, 26 Apr 2022 10:56:12 +0100
Subject: [PATCH 05/12] Update release script to be poetry-aware

Poetry now manages the project version in pyproject.toml.
---
 scripts-dev/release.py | 68 +++++++++++++-----------------------------
 1 file changed, 21 insertions(+), 47 deletions(-)

diff --git a/scripts-dev/release.py b/scripts-dev/release.py
index 6f7cf6888d..725e8f0479 100755
--- a/scripts-dev/release.py
+++ b/scripts-dev/release.py
@@ -25,13 +25,13 @@ import sys
 import urllib.request
 from os import path
 from tempfile import TemporaryDirectory
-from typing import List, Optional, Tuple
+from typing import List, Optional
 
 import attr
 import click
 import commonmark
 import git
-import redbaron
+
 from click.exceptions import ClickException
 from github import Github
 from packaging import version
@@ -100,7 +100,7 @@ def prepare():
     repo.remote().fetch()
 
     # Get the current version and AST from root Synapse module.
-    current_version, parsed_synapse_ast, version_node = parse_version_from_module()
+    current_version = get_package_version()
 
     # Figure out what sort of release we're doing and calcuate the new version.
     rc = click.confirm("RC", default=True)
@@ -162,7 +162,7 @@ def prepare():
         click.get_current_context().abort()
 
     # Switch to the release branch.
-    parsed_new_version = version.parse(new_version)
+    parsed_new_version: version.Version = version.parse(new_version)
 
     # We assume for debian changelogs that we only do RCs or full releases.
     assert not parsed_new_version.is_devrelease
@@ -207,17 +207,15 @@ def prepare():
         # Create the new release branch
         release_branch = repo.create_head(release_branch_name, commit=base_branch)
 
-    # Switch to the release branch and ensure its up to date.
+    # Switch to the release branch and ensure it's up to date.
     repo.git.checkout(release_branch_name)
     update_branch(repo)
 
-    # Update the `__version__` variable and write it back to the file.
-    version_node.value = '"' + new_version + '"'
-    with open("synapse/__init__.py", "w") as f:
-        f.write(parsed_synapse_ast.dumps())
+    # Update the version specified in pyproject.toml.
+    subprocess.check_output(["poetry", "version", new_version])
 
     # Generate changelogs.
-    generate_and_write_changelog(current_version)
+    generate_and_write_changelog(current_version, new_version)
 
     # Generate debian changelogs
     if parsed_new_version.pre is not None:
@@ -284,7 +282,7 @@ def tag(gh_token: Optional[str]):
     repo.remote().fetch()
 
     # Find out the version and tag name.
-    current_version, _, _ = parse_version_from_module()
+    current_version = get_package_version()
     tag_name = f"v{current_version}"
 
     # Check we haven't released this version.
@@ -362,7 +360,7 @@ def publish(gh_token: str):
     if repo.is_dirty():
         raise click.ClickException("Uncommitted changes exist.")
 
-    current_version, _, _ = parse_version_from_module()
+    current_version = get_package_version()
     tag_name = f"v{current_version}"
 
     if not click.confirm(f"Publish {tag_name}?", default=True):
@@ -396,7 +394,7 @@ def publish(gh_token: str):
 def upload():
     """Upload release to pypi."""
 
-    current_version, _, _ = parse_version_from_module()
+    current_version = get_package_version()
     tag_name = f"v{current_version}"
 
     pypi_asset_names = [
@@ -424,7 +422,7 @@ def upload():
 def announce():
     """Generate markdown to announce the release."""
 
-    current_version, _, _ = parse_version_from_module()
+    current_version = get_package_version()
     tag_name = f"v{current_version}"
 
     click.echo(
@@ -455,37 +453,11 @@ Announce the release in
         )
 
 
-def parse_version_from_module() -> Tuple[
-    version.Version, redbaron.RedBaron, redbaron.Node
-]:
-    # Parse the AST and load the `__version__` node so that we can edit it
-    # later.
-    with open("synapse/__init__.py") as f:
-        red = redbaron.RedBaron(f.read())
-
-    version_node = None
-    for node in red:
-        if node.type != "assignment":
-            continue
-
-        if node.target.type != "name":
-            continue
-
-        if node.target.value != "__version__":
-            continue
-
-        version_node = node
-        break
-
-    if not version_node:
-        print("Failed to find '__version__' definition in synapse/__init__.py")
-        sys.exit(1)
-
-    # Parse the current version.
-    current_version = version.parse(version_node.value.value.strip('"'))
-    assert isinstance(current_version, version.Version)
-
-    return current_version, red, version_node
+def get_package_version() -> version.Version:
+    version_string = subprocess.check_output(["poetry", "version", "--short"]).decode(
+        "utf-8"
+    )
+    return version.Version(version_string)
 
 
 def find_ref(repo: git.Repo, ref_name: str) -> Optional[git.HEAD]:
@@ -565,11 +537,13 @@ def get_changes_for_version(wanted_version: version.Version) -> str:
     return "\n".join(version_changelog)
 
 
-def generate_and_write_changelog(current_version: version.Version):
+def generate_and_write_changelog(current_version: version.Version, new_version: str):
     # We do this by getting a draft so that we can edit it before writing to the
     # changelog.
     result = run_until_successful(
-        "python3 -m towncrier --draft", shell=True, capture_output=True
+        f"python3 -m towncrier build --draft --version {new_version}",
+        shell=True,
+        capture_output=True,
     )
     new_changes = result.stdout.decode("utf-8")
     new_changes = new_changes.replace(

From 7c063da25c80906f0637f907d0c744cd778c682d Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Tue, 26 Apr 2022 11:14:41 +0100
Subject: [PATCH 06/12] Temporarily lower debian changelog version number

This seems to make dch happy when we prepare the release.
---
 debian/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/debian/changelog b/debian/changelog
index 05e6bd75a7..7727536a74 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,4 +1,4 @@
-matrix-synapse-py3 (1.58.0+nmu1) UNRELEASED; urgency=medium
+matrix-synapse-py3 (1.57.1+nmu1) UNRELEASED; urgency=medium
 
   * Use poetry to manage the bundled virtualenv included with this package.
 

From 30db7fdb911ead29052476db2797def29baefa1b Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Tue, 26 Apr 2022 11:15:33 +0100
Subject: [PATCH 07/12] 1.58.0rc1

---
 CHANGES.md                | 77 +++++++++++++++++++++++++++++++++++++++
 changelog.d/11398.feature |  1 -
 changelog.d/12213.bugfix  |  1 -
 changelog.d/12319.bugfix  |  1 -
 changelog.d/12337.feature |  1 -
 changelog.d/12340.doc     |  1 -
 changelog.d/12344.removal |  1 -
 changelog.d/12365.feature |  1 -
 changelog.d/12368.doc     |  1 -
 changelog.d/12382.removal |  1 -
 changelog.d/12394.misc    |  1 -
 changelog.d/12395.misc    |  1 -
 changelog.d/12399.misc    |  1 -
 changelog.d/12425.misc    |  1 -
 changelog.d/12427.feature |  1 -
 changelog.d/12434.misc    |  1 -
 changelog.d/12438.misc    |  1 -
 changelog.d/12441.misc    |  1 -
 changelog.d/12445.misc    |  1 -
 changelog.d/12449.misc    |  1 -
 changelog.d/12450.misc    |  1 -
 changelog.d/12451.doc     |  1 -
 changelog.d/12454.misc    |  1 -
 changelog.d/12455.misc    |  1 -
 changelog.d/12457.doc     |  1 -
 changelog.d/12464.misc    |  1 -
 changelog.d/12465.feature |  1 -
 changelog.d/12466.misc    |  1 -
 changelog.d/12467.misc    |  1 -
 changelog.d/12468.misc    |  1 -
 changelog.d/12472.misc    |  1 -
 changelog.d/12474.misc    |  1 -
 changelog.d/12475.doc     |  1 -
 changelog.d/12476.bugfix  |  1 -
 changelog.d/12478.misc    |  1 -
 changelog.d/12483.misc    |  1 -
 changelog.d/12492.doc     |  1 -
 changelog.d/12495.doc     |  1 -
 changelog.d/12496.bugfix  |  1 -
 changelog.d/12497.misc    |  1 -
 changelog.d/12501.doc     |  1 -
 changelog.d/12510.bugfix  |  1 -
 changelog.d/12511.misc    |  1 -
 changelog.d/12514.misc    |  1 -
 changelog.d/12519.misc    |  1 -
 changelog.d/12520.bugfix  |  1 -
 changelog.d/12522.bugfix  |  1 -
 changelog.d/12527.doc     |  2 -
 changelog.d/12528.misc    |  1 -
 changelog.d/12529.misc    |  1 -
 changelog.d/12533.doc     |  1 -
 changelog.d/12543.misc    |  1 -
 debian/changelog          |  5 ++-
 pyproject.toml            |  2 +-
 54 files changed, 81 insertions(+), 55 deletions(-)
 delete mode 100644 changelog.d/11398.feature
 delete mode 100644 changelog.d/12213.bugfix
 delete mode 100644 changelog.d/12319.bugfix
 delete mode 100644 changelog.d/12337.feature
 delete mode 100644 changelog.d/12340.doc
 delete mode 100644 changelog.d/12344.removal
 delete mode 100644 changelog.d/12365.feature
 delete mode 100644 changelog.d/12368.doc
 delete mode 100644 changelog.d/12382.removal
 delete mode 100644 changelog.d/12394.misc
 delete mode 100644 changelog.d/12395.misc
 delete mode 100644 changelog.d/12399.misc
 delete mode 100644 changelog.d/12425.misc
 delete mode 100644 changelog.d/12427.feature
 delete mode 100644 changelog.d/12434.misc
 delete mode 100644 changelog.d/12438.misc
 delete mode 100644 changelog.d/12441.misc
 delete mode 100644 changelog.d/12445.misc
 delete mode 100644 changelog.d/12449.misc
 delete mode 100644 changelog.d/12450.misc
 delete mode 100644 changelog.d/12451.doc
 delete mode 100644 changelog.d/12454.misc
 delete mode 100644 changelog.d/12455.misc
 delete mode 100644 changelog.d/12457.doc
 delete mode 100644 changelog.d/12464.misc
 delete mode 100644 changelog.d/12465.feature
 delete mode 100644 changelog.d/12466.misc
 delete mode 100644 changelog.d/12467.misc
 delete mode 100644 changelog.d/12468.misc
 delete mode 100644 changelog.d/12472.misc
 delete mode 100644 changelog.d/12474.misc
 delete mode 100644 changelog.d/12475.doc
 delete mode 100644 changelog.d/12476.bugfix
 delete mode 100644 changelog.d/12478.misc
 delete mode 100644 changelog.d/12483.misc
 delete mode 100644 changelog.d/12492.doc
 delete mode 100644 changelog.d/12495.doc
 delete mode 100644 changelog.d/12496.bugfix
 delete mode 100644 changelog.d/12497.misc
 delete mode 100644 changelog.d/12501.doc
 delete mode 100644 changelog.d/12510.bugfix
 delete mode 100644 changelog.d/12511.misc
 delete mode 100644 changelog.d/12514.misc
 delete mode 100644 changelog.d/12519.misc
 delete mode 100644 changelog.d/12520.bugfix
 delete mode 100644 changelog.d/12522.bugfix
 delete mode 100644 changelog.d/12527.doc
 delete mode 100644 changelog.d/12528.misc
 delete mode 100644 changelog.d/12529.misc
 delete mode 100644 changelog.d/12533.doc
 delete mode 100644 changelog.d/12543.misc

diff --git a/CHANGES.md b/CHANGES.md
index a7d2529b55..95f0b5ad37 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,3 +1,80 @@
+Synapse 1.58.0rc1 (2022-04-26)
+==============================
+
+Features
+--------
+
+- Implement [MSC3383](https://github.com/matrix-org/matrix-spec-proposals/pull/3383) for including the destination in server-to-server authentication headers. Contributed by @Bubu and @jcgruenhage for Famedly GmbH. ([\#11398](https://github.com/matrix-org/synapse/issues/11398))
+- Use poetry to manage Synapse's dependencies. ([\#12337](https://github.com/matrix-org/synapse/issues/12337))
+- Enable processing of device list updates asynchronously. ([\#12365](https://github.com/matrix-org/synapse/issues/12365), [\#12465](https://github.com/matrix-org/synapse/issues/12465))
+- Implement [MSC2815](https://github.com/matrix-org/matrix-spec-proposals/pull/2815) to allow room moderators to view redacted event content. Contributed by @tulir. ([\#12427](https://github.com/matrix-org/synapse/issues/12427))
+
+
+Bugfixes
+--------
+
+- Prevent a sync request from removing a user's busy presence status. ([\#12213](https://github.com/matrix-org/synapse/issues/12213))
+- Fix bug with incremental sync missing events when rejoining/backfilling. Contributed by Nick @ Beeper. ([\#12319](https://github.com/matrix-org/synapse/issues/12319))
+- Fix a long-standing bug which incorrectly caused `GET /_matrix/client/r3/rooms/{roomId}/event/{eventId}` to return edited events rather than the original. ([\#12476](https://github.com/matrix-org/synapse/issues/12476))
+- Fix bug where the admin API for [deleting forward extremities](https://github.com/matrix-org/synapse/blob/erikj/fix_delete_event_response_count/docs/admin_api/rooms.md#deleting-forward-extremities) would always return a count of 1 no matter how many extremities were deleted. Broke in v1.27.0. ([\#12496](https://github.com/matrix-org/synapse/issues/12496))
+- Fix a long-standing bug where the image thumbanils embedded into email notifications were broken. ([\#12510](https://github.com/matrix-org/synapse/issues/12510))
+- Fix a bug in the implementation of MSC3202 where Synapse would use the field name `device_unused_fallback_keys`, rather than `device_unused_fallback_key_types`. ([\#12520](https://github.com/matrix-org/synapse/issues/12520))
+- Fix a bug introduced in Synapse 0.99.3 which could cause Synapse to consume large amounts of RAM when back-paginating in a large room. ([\#12522](https://github.com/matrix-org/synapse/issues/12522))
+
+
+Improved Documentation
+----------------------
+
+- Fix rendering of the documentation site when using the 'print' feature. ([\#12340](https://github.com/matrix-org/synapse/issues/12340))
+- Add a manual documenting config file options. ([\#12368](https://github.com/matrix-org/synapse/issues/12368))
+- Update documentation to reflect that both the `run_background_tasks_on` option and the options for moving stream writers off of the main process are no longer experimental. ([\#12451](https://github.com/matrix-org/synapse/issues/12451))
+- Update worker documentation and replace old `federation_reader` with `generic_worker`. ([\#12457](https://github.com/matrix-org/synapse/issues/12457))
+- Strongly recommend `poetry` for development. ([\#12475](https://github.com/matrix-org/synapse/issues/12475))
+- Add some example configurations for workers and update architectural diagram. ([\#12492](https://github.com/matrix-org/synapse/issues/12492))
+- Fix a broken link in `README.rst`. ([\#12495](https://github.com/matrix-org/synapse/issues/12495))
+- Add HAProxy delegation example with CORS headers to docs. ([\#12501](https://github.com/matrix-org/synapse/issues/12501))
+- Add an index to the configuration manual. ([\#12527](https://github.com/matrix-org/synapse/issues/12527))
+- Remove extraneous comma in User Admin API's device deletion section so that the example JSON is actually valid and works. Contributed by @olmari. ([\#12533](https://github.com/matrix-org/synapse/issues/12533))
+
+
+Deprecations and Removals
+-------------------------
+
+- The groups/communities feature in Synapse has been disabled by default. ([\#12344](https://github.com/matrix-org/synapse/issues/12344))
+- Remove unstable identifiers from [MSC3440](https://github.com/matrix-org/matrix-doc/pull/3440). ([\#12382](https://github.com/matrix-org/synapse/issues/12382))
+
+
+Internal Changes
+----------------
+
+- Preparation for faster-room-join work: start a background process to resynchronise the room state after a room join. ([\#12394](https://github.com/matrix-org/synapse/issues/12394))
+- Remove an unstable identifier from [MSC3083](https://github.com/matrix-org/matrix-doc/pull/3083). ([\#12395](https://github.com/matrix-org/synapse/issues/12395))
+- Preparation for faster-room-join work: Implement a tracking mechanism to allow functions to wait for full room state to arrive. ([\#12399](https://github.com/matrix-org/synapse/issues/12399))
+- Run twisted trunk CI job in the locked poetry environment. ([\#12425](https://github.com/matrix-org/synapse/issues/12425))
+- Run lints under poetry in CI, and remove corresponding tox lint jobs. ([\#12434](https://github.com/matrix-org/synapse/issues/12434))
+- Run "main" trial tests under `poetry`. ([\#12438](https://github.com/matrix-org/synapse/issues/12438))
+- Bump twisted version in `poetry.lock` to work around [pip bug #9644](https://github.com/pypa/pip/issues/9644). ([\#12441](https://github.com/matrix-org/synapse/issues/12441))
+- Change Mutual Rooms' `unstable_features` flag to `uk.half-shot.msc2666.mutual_rooms` which matches the current MSC iteration. ([\#12445](https://github.com/matrix-org/synapse/issues/12445))
+- Use `poetry` to manage the virtualenv in debian packages. ([\#12449](https://github.com/matrix-org/synapse/issues/12449))
+- Fix typo in the release script help string. ([\#12450](https://github.com/matrix-org/synapse/issues/12450))
+- Limit length of device_id to less than 512 characters. ([\#12454](https://github.com/matrix-org/synapse/issues/12454))
+- Reintroduce the list of targets to the linter script, to avoid linting unwanted local-only directories during development. ([\#12455](https://github.com/matrix-org/synapse/issues/12455))
+- Dockerfile-workers: reduce the amount we install in the image. ([\#12464](https://github.com/matrix-org/synapse/issues/12464))
+- Dockerfile-workers: give the master its own log config. ([\#12466](https://github.com/matrix-org/synapse/issues/12466))
+- complement-synapse-workers: factor out separate entry point script. ([\#12467](https://github.com/matrix-org/synapse/issues/12467))
+- Update `delay_cancellation` to accept any awaitable, rather than just `Deferred`s. ([\#12468](https://github.com/matrix-org/synapse/issues/12468))
+- Add a CI job which tests Synapse against the latest version of all dependencies. ([\#12472](https://github.com/matrix-org/synapse/issues/12472))
+- Back out experimental implementation of [MSC2314](https://github.com/matrix-org/matrix-spec-proposals/pull/2314). ([\#12474](https://github.com/matrix-org/synapse/issues/12474))
+- Use poetry-core instead of setuptools to build wheels. ([\#12478](https://github.com/matrix-org/synapse/issues/12478), [\#12514](https://github.com/matrix-org/synapse/issues/12514))
+- Fix grammatical error in federation error response when the room version of a room is unknown. ([\#12483](https://github.com/matrix-org/synapse/issues/12483))
+- Fix a minor typo in the Debian changelogs generated by the release script. ([\#12497](https://github.com/matrix-org/synapse/issues/12497))
+- Remove unnecessary configuration overrides in tests. ([\#12511](https://github.com/matrix-org/synapse/issues/12511))
+- Refactor the relations code for clarity. ([\#12519](https://github.com/matrix-org/synapse/issues/12519))
+- Add type hints so `docker` and `stubs` directories pass `mypy --disallow-untyped-defs`. ([\#12528](https://github.com/matrix-org/synapse/issues/12528))
+- Handle cancellation in `EventsWorkerStore._get_events_from_cache_or_db`. ([\#12529](https://github.com/matrix-org/synapse/issues/12529))
+- Build debian packages for Ubuntu 22.04 "Jammy Jellyfish". ([\#12543](https://github.com/matrix-org/synapse/issues/12543))
+
+
 Synapse 1.57.1 (2022-04-20)
 ===========================
 
diff --git a/changelog.d/11398.feature b/changelog.d/11398.feature
deleted file mode 100644
index a910f4da14..0000000000
--- a/changelog.d/11398.feature
+++ /dev/null
@@ -1 +0,0 @@
-Implement [MSC3383](https://github.com/matrix-org/matrix-spec-proposals/pull/3383) for including the destination in server-to-server authentication headers. Contributed by @Bubu and @jcgruenhage for Famedly GmbH.
diff --git a/changelog.d/12213.bugfix b/changelog.d/12213.bugfix
deleted file mode 100644
index 9278e3a9c1..0000000000
--- a/changelog.d/12213.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Prevent a sync request from removing a user's busy presence status.
diff --git a/changelog.d/12319.bugfix b/changelog.d/12319.bugfix
deleted file mode 100644
index a50191feaa..0000000000
--- a/changelog.d/12319.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix bug with incremental sync missing events when rejoining/backfilling. Contributed by Nick @ Beeper.
diff --git a/changelog.d/12337.feature b/changelog.d/12337.feature
deleted file mode 100644
index 6c4444c707..0000000000
--- a/changelog.d/12337.feature
+++ /dev/null
@@ -1 +0,0 @@
-Use poetry to manage Synapse's dependencies.
\ No newline at end of file
diff --git a/changelog.d/12340.doc b/changelog.d/12340.doc
deleted file mode 100644
index 8354f2259e..0000000000
--- a/changelog.d/12340.doc
+++ /dev/null
@@ -1 +0,0 @@
-Fix rendering of the documentation site when using the 'print' feature.
diff --git a/changelog.d/12344.removal b/changelog.d/12344.removal
deleted file mode 100644
index ecefa76d8e..0000000000
--- a/changelog.d/12344.removal
+++ /dev/null
@@ -1 +0,0 @@
-The groups/communities feature in Synapse has been disabled by default.
diff --git a/changelog.d/12365.feature b/changelog.d/12365.feature
deleted file mode 100644
index 642dea966c..0000000000
--- a/changelog.d/12365.feature
+++ /dev/null
@@ -1 +0,0 @@
-Enable processing of device list updates asynchronously.
diff --git a/changelog.d/12368.doc b/changelog.d/12368.doc
deleted file mode 100644
index 62e4cb2c7e..0000000000
--- a/changelog.d/12368.doc
+++ /dev/null
@@ -1 +0,0 @@
-Add a manual documenting config file options.
\ No newline at end of file
diff --git a/changelog.d/12382.removal b/changelog.d/12382.removal
deleted file mode 100644
index eb91186340..0000000000
--- a/changelog.d/12382.removal
+++ /dev/null
@@ -1 +0,0 @@
-Remove unstable identifiers from [MSC3440](https://github.com/matrix-org/matrix-doc/pull/3440).
diff --git a/changelog.d/12394.misc b/changelog.d/12394.misc
deleted file mode 100644
index 69109fcc37..0000000000
--- a/changelog.d/12394.misc
+++ /dev/null
@@ -1 +0,0 @@
-Preparation for faster-room-join work: start a background process to resynchronise the room state after a room join.
diff --git a/changelog.d/12395.misc b/changelog.d/12395.misc
deleted file mode 100644
index 0a2123b294..0000000000
--- a/changelog.d/12395.misc
+++ /dev/null
@@ -1 +0,0 @@
-Remove an unstable identifier from [MSC3083](https://github.com/matrix-org/matrix-doc/pull/3083).
diff --git a/changelog.d/12399.misc b/changelog.d/12399.misc
deleted file mode 100644
index cd2e09626d..0000000000
--- a/changelog.d/12399.misc
+++ /dev/null
@@ -1 +0,0 @@
-Preparation for faster-room-join work: Implement a tracking mechanism to allow functions to wait for full room state to arrive.
diff --git a/changelog.d/12425.misc b/changelog.d/12425.misc
deleted file mode 100644
index 3b076be0bd..0000000000
--- a/changelog.d/12425.misc
+++ /dev/null
@@ -1 +0,0 @@
-Run twisted trunk CI job in the locked poetry environment.
diff --git a/changelog.d/12427.feature b/changelog.d/12427.feature
deleted file mode 100644
index e6913c8c09..0000000000
--- a/changelog.d/12427.feature
+++ /dev/null
@@ -1 +0,0 @@
-Implement [MSC2815](https://github.com/matrix-org/matrix-spec-proposals/pull/2815) to allow room moderators to view redacted event content. Contributed by @tulir.
diff --git a/changelog.d/12434.misc b/changelog.d/12434.misc
deleted file mode 100644
index 88dab428d2..0000000000
--- a/changelog.d/12434.misc
+++ /dev/null
@@ -1 +0,0 @@
-Run lints under poetry in CI, and remove corresponding tox lint jobs.
diff --git a/changelog.d/12438.misc b/changelog.d/12438.misc
deleted file mode 100644
index f2c07a56da..0000000000
--- a/changelog.d/12438.misc
+++ /dev/null
@@ -1 +0,0 @@
-Run "main" trial tests under `poetry`.
diff --git a/changelog.d/12441.misc b/changelog.d/12441.misc
deleted file mode 100644
index c2619f1654..0000000000
--- a/changelog.d/12441.misc
+++ /dev/null
@@ -1 +0,0 @@
-Bump twisted version in `poetry.lock` to work around [pip bug #9644](https://github.com/pypa/pip/issues/9644).
diff --git a/changelog.d/12445.misc b/changelog.d/12445.misc
deleted file mode 100644
index 954248115a..0000000000
--- a/changelog.d/12445.misc
+++ /dev/null
@@ -1 +0,0 @@
-Change Mutual Rooms' `unstable_features` flag to `uk.half-shot.msc2666.mutual_rooms` which matches the current MSC iteration.
\ No newline at end of file
diff --git a/changelog.d/12449.misc b/changelog.d/12449.misc
deleted file mode 100644
index 03e08aace4..0000000000
--- a/changelog.d/12449.misc
+++ /dev/null
@@ -1 +0,0 @@
-Use `poetry` to manage the virtualenv in debian packages.
diff --git a/changelog.d/12450.misc b/changelog.d/12450.misc
deleted file mode 100644
index 4b1c8cba87..0000000000
--- a/changelog.d/12450.misc
+++ /dev/null
@@ -1 +0,0 @@
-Fix typo in the release script help string.
diff --git a/changelog.d/12451.doc b/changelog.d/12451.doc
deleted file mode 100644
index c8b23c1285..0000000000
--- a/changelog.d/12451.doc
+++ /dev/null
@@ -1 +0,0 @@
-Update documentation to reflect that both the `run_background_tasks_on` option and the options for moving stream writers off of the main process are no longer experimental.
diff --git a/changelog.d/12454.misc b/changelog.d/12454.misc
deleted file mode 100644
index cb7ff74b4c..0000000000
--- a/changelog.d/12454.misc
+++ /dev/null
@@ -1 +0,0 @@
-Limit length of device_id to less than 512 characters.
diff --git a/changelog.d/12455.misc b/changelog.d/12455.misc
deleted file mode 100644
index 9b19945673..0000000000
--- a/changelog.d/12455.misc
+++ /dev/null
@@ -1 +0,0 @@
-Reintroduce the list of targets to the linter script, to avoid linting unwanted local-only directories during development.
diff --git a/changelog.d/12457.doc b/changelog.d/12457.doc
deleted file mode 100644
index a4871622cf..0000000000
--- a/changelog.d/12457.doc
+++ /dev/null
@@ -1 +0,0 @@
-Update worker documentation and replace old `federation_reader` with `generic_worker`.
\ No newline at end of file
diff --git a/changelog.d/12464.misc b/changelog.d/12464.misc
deleted file mode 100644
index 7a8cc6ba51..0000000000
--- a/changelog.d/12464.misc
+++ /dev/null
@@ -1 +0,0 @@
-Dockerfile-workers: reduce the amount we install in the image.
diff --git a/changelog.d/12465.feature b/changelog.d/12465.feature
deleted file mode 100644
index 642dea966c..0000000000
--- a/changelog.d/12465.feature
+++ /dev/null
@@ -1 +0,0 @@
-Enable processing of device list updates asynchronously.
diff --git a/changelog.d/12466.misc b/changelog.d/12466.misc
deleted file mode 100644
index b0c2c950fe..0000000000
--- a/changelog.d/12466.misc
+++ /dev/null
@@ -1 +0,0 @@
-Dockerfile-workers: give the master its own log config.
diff --git a/changelog.d/12467.misc b/changelog.d/12467.misc
deleted file mode 100644
index fbf415f707..0000000000
--- a/changelog.d/12467.misc
+++ /dev/null
@@ -1 +0,0 @@
-complement-synapse-workers: factor out separate entry point script.
diff --git a/changelog.d/12468.misc b/changelog.d/12468.misc
deleted file mode 100644
index 3d5d25247f..0000000000
--- a/changelog.d/12468.misc
+++ /dev/null
@@ -1 +0,0 @@
-Update `delay_cancellation` to accept any awaitable, rather than just `Deferred`s.
diff --git a/changelog.d/12472.misc b/changelog.d/12472.misc
deleted file mode 100644
index ed306209cc..0000000000
--- a/changelog.d/12472.misc
+++ /dev/null
@@ -1 +0,0 @@
-Add a CI job which tests Synapse against the latest version of all dependencies.
diff --git a/changelog.d/12474.misc b/changelog.d/12474.misc
deleted file mode 100644
index 5292108b39..0000000000
--- a/changelog.d/12474.misc
+++ /dev/null
@@ -1 +0,0 @@
-Back out experimental implementation of [MSC2314](https://github.com/matrix-org/matrix-spec-proposals/pull/2314).
diff --git a/changelog.d/12475.doc b/changelog.d/12475.doc
deleted file mode 100644
index f4481d0613..0000000000
--- a/changelog.d/12475.doc
+++ /dev/null
@@ -1 +0,0 @@
-Strongly recommend `poetry` for development.
diff --git a/changelog.d/12476.bugfix b/changelog.d/12476.bugfix
deleted file mode 100644
index 9ad6a71abd..0000000000
--- a/changelog.d/12476.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix a long-standing bug which incorrectly caused `GET /_matrix/client/r3/rooms/{roomId}/event/{eventId}` to return edited events rather than the original.
diff --git a/changelog.d/12478.misc b/changelog.d/12478.misc
deleted file mode 100644
index 061a604a1e..0000000000
--- a/changelog.d/12478.misc
+++ /dev/null
@@ -1 +0,0 @@
-Use poetry-core instead of setuptools to build wheels.
diff --git a/changelog.d/12483.misc b/changelog.d/12483.misc
deleted file mode 100644
index 88c6e3e465..0000000000
--- a/changelog.d/12483.misc
+++ /dev/null
@@ -1 +0,0 @@
-Fix grammatical error in federation error response when the room version of a room is unknown.
diff --git a/changelog.d/12492.doc b/changelog.d/12492.doc
deleted file mode 100644
index 4a3e2f4f06..0000000000
--- a/changelog.d/12492.doc
+++ /dev/null
@@ -1 +0,0 @@
-Add some example configurations for workers and update architectural diagram.
diff --git a/changelog.d/12495.doc b/changelog.d/12495.doc
deleted file mode 100644
index afa0111675..0000000000
--- a/changelog.d/12495.doc
+++ /dev/null
@@ -1 +0,0 @@
-Fix a broken link in `README.rst`.
diff --git a/changelog.d/12496.bugfix b/changelog.d/12496.bugfix
deleted file mode 100644
index a68df7c96a..0000000000
--- a/changelog.d/12496.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix bug where the admin API for [deleting forward extremities](https://github.com/matrix-org/synapse/blob/erikj/fix_delete_event_response_count/docs/admin_api/rooms.md#deleting-forward-extremities) would always return a count of 1 no matter how many extremities were deleted. Broke in v1.27.0.
diff --git a/changelog.d/12497.misc b/changelog.d/12497.misc
deleted file mode 100644
index 17a661ec61..0000000000
--- a/changelog.d/12497.misc
+++ /dev/null
@@ -1 +0,0 @@
-Fix a minor typo in the Debian changelogs generated by the release script.
diff --git a/changelog.d/12501.doc b/changelog.d/12501.doc
deleted file mode 100644
index 278193a69a..0000000000
--- a/changelog.d/12501.doc
+++ /dev/null
@@ -1 +0,0 @@
-Add HAProxy delegation example with CORS headers to docs.
diff --git a/changelog.d/12510.bugfix b/changelog.d/12510.bugfix
deleted file mode 100644
index d5856e982a..0000000000
--- a/changelog.d/12510.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix a long-standing bug where the image thumbanils embedded into email notifications were broken.
diff --git a/changelog.d/12511.misc b/changelog.d/12511.misc
deleted file mode 100644
index a314bedfc4..0000000000
--- a/changelog.d/12511.misc
+++ /dev/null
@@ -1 +0,0 @@
-Remove unnecessary configuration overrides in tests.
diff --git a/changelog.d/12514.misc b/changelog.d/12514.misc
deleted file mode 100644
index 061a604a1e..0000000000
--- a/changelog.d/12514.misc
+++ /dev/null
@@ -1 +0,0 @@
-Use poetry-core instead of setuptools to build wheels.
diff --git a/changelog.d/12519.misc b/changelog.d/12519.misc
deleted file mode 100644
index 9c023d8e3e..0000000000
--- a/changelog.d/12519.misc
+++ /dev/null
@@ -1 +0,0 @@
-Refactor the relations code for clarity.
diff --git a/changelog.d/12520.bugfix b/changelog.d/12520.bugfix
deleted file mode 100644
index c73005fde8..0000000000
--- a/changelog.d/12520.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix a bug in the implementation of MSC3202 where Synapse would use the field name `device_unused_fallback_keys`, rather than `device_unused_fallback_key_types`.
\ No newline at end of file
diff --git a/changelog.d/12522.bugfix b/changelog.d/12522.bugfix
deleted file mode 100644
index 2220f05ceb..0000000000
--- a/changelog.d/12522.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix a bug introduced in Synapse 0.99.3 which could cause Synapse to consume large amounts of RAM when back-paginating in a large room.
diff --git a/changelog.d/12527.doc b/changelog.d/12527.doc
deleted file mode 100644
index e6907321e7..0000000000
--- a/changelog.d/12527.doc
+++ /dev/null
@@ -1,2 +0,0 @@
-Add an index to the configuration manual.
-
diff --git a/changelog.d/12528.misc b/changelog.d/12528.misc
deleted file mode 100644
index f64b5d24b0..0000000000
--- a/changelog.d/12528.misc
+++ /dev/null
@@ -1 +0,0 @@
-Add type hints so `docker` and `stubs` directories pass `mypy --disallow-untyped-defs`.
diff --git a/changelog.d/12529.misc b/changelog.d/12529.misc
deleted file mode 100644
index 5427108742..0000000000
--- a/changelog.d/12529.misc
+++ /dev/null
@@ -1 +0,0 @@
-Handle cancellation in `EventsWorkerStore._get_events_from_cache_or_db`.
diff --git a/changelog.d/12533.doc b/changelog.d/12533.doc
deleted file mode 100644
index 2c15488111..0000000000
--- a/changelog.d/12533.doc
+++ /dev/null
@@ -1 +0,0 @@
-Remove extraneous comma in User Admin API's device deletion section so that the example JSON is actually valid and works. Contributed by @olmari.
\ No newline at end of file
diff --git a/changelog.d/12543.misc b/changelog.d/12543.misc
deleted file mode 100644
index eed7a6973c..0000000000
--- a/changelog.d/12543.misc
+++ /dev/null
@@ -1 +0,0 @@
-Build debian packages for Ubuntu 22.04 "Jammy Jellyfish".
diff --git a/debian/changelog b/debian/changelog
index 7727536a74..20e756f6db 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,8 +1,9 @@
-matrix-synapse-py3 (1.57.1+nmu1) UNRELEASED; urgency=medium
+matrix-synapse-py3 (1.58.0~rc1) stable; urgency=medium
 
   * Use poetry to manage the bundled virtualenv included with this package.
+  * New Synapse release 1.58.0rc1.
 
- -- Synapse Packaging team <packages@matrix.org>  Wed, 30 Mar 2022 12:21:43 +0100
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 26 Apr 2022 11:15:20 +0100
 
 matrix-synapse-py3 (1.57.1) stable; urgency=medium
 
diff --git a/pyproject.toml b/pyproject.toml
index c7f3e20fed..80e25cef1d 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -54,7 +54,7 @@ skip_gitignore = true
 
 [tool.poetry]
 name = "matrix-synapse"
-version = "1.57.1"
+version = "1.58.0rc1"
 description = "Homeserver for the Matrix decentralised comms protocol"
 authors = ["Matrix.org Team and Contributors <packages@matrix.org>"]
 license = "Apache-2.0"

From f987cdd80bf7c695dc977eb64614019a9ce73ff0 Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Tue, 26 Apr 2022 11:32:57 +0100
Subject: [PATCH 08/12] Changelog update

---
 CHANGES.md | 39 ++++++++++++++++-----------------------
 1 file changed, 16 insertions(+), 23 deletions(-)

diff --git a/CHANGES.md b/CHANGES.md
index 95f0b5ad37..b9d477a6eb 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -4,10 +4,11 @@ Synapse 1.58.0rc1 (2022-04-26)
 Features
 --------
 
-- Implement [MSC3383](https://github.com/matrix-org/matrix-spec-proposals/pull/3383) for including the destination in server-to-server authentication headers. Contributed by @Bubu and @jcgruenhage for Famedly GmbH. ([\#11398](https://github.com/matrix-org/synapse/issues/11398))
-- Use poetry to manage Synapse's dependencies. ([\#12337](https://github.com/matrix-org/synapse/issues/12337))
+- Implement [MSC3383](https://github.com/matrix-org/matrix-spec-proposals/pull/3383) for including the destination in server-to-server authentication headers. Contributed by @Bubu and @jcgruenhage for Famedly. ([\#11398](https://github.com/matrix-org/synapse/issues/11398))
+- Docker images and Debian packages from matrix.org now contain a locked set of Python dependencies, greatly improving reproducibility. ([Board](https://github.com/orgs/matrix-org/projects/54), [\#11537](https://github.com/matrix-org/synapse/issues/11537))
 - Enable processing of device list updates asynchronously. ([\#12365](https://github.com/matrix-org/synapse/issues/12365), [\#12465](https://github.com/matrix-org/synapse/issues/12465))
 - Implement [MSC2815](https://github.com/matrix-org/matrix-spec-proposals/pull/2815) to allow room moderators to view redacted event content. Contributed by @tulir. ([\#12427](https://github.com/matrix-org/synapse/issues/12427))
+- Build Debian packages for Ubuntu 22.04 "Jammy Jellyfish". ([\#12543](https://github.com/matrix-org/synapse/issues/12543))
 
 
 Bugfixes
@@ -15,10 +16,10 @@ Bugfixes
 
 - Prevent a sync request from removing a user's busy presence status. ([\#12213](https://github.com/matrix-org/synapse/issues/12213))
 - Fix bug with incremental sync missing events when rejoining/backfilling. Contributed by Nick @ Beeper. ([\#12319](https://github.com/matrix-org/synapse/issues/12319))
-- Fix a long-standing bug which incorrectly caused `GET /_matrix/client/r3/rooms/{roomId}/event/{eventId}` to return edited events rather than the original. ([\#12476](https://github.com/matrix-org/synapse/issues/12476))
-- Fix bug where the admin API for [deleting forward extremities](https://github.com/matrix-org/synapse/blob/erikj/fix_delete_event_response_count/docs/admin_api/rooms.md#deleting-forward-extremities) would always return a count of 1 no matter how many extremities were deleted. Broke in v1.27.0. ([\#12496](https://github.com/matrix-org/synapse/issues/12496))
-- Fix a long-standing bug where the image thumbanils embedded into email notifications were broken. ([\#12510](https://github.com/matrix-org/synapse/issues/12510))
-- Fix a bug in the implementation of MSC3202 where Synapse would use the field name `device_unused_fallback_keys`, rather than `device_unused_fallback_key_types`. ([\#12520](https://github.com/matrix-org/synapse/issues/12520))
+- Fix a long-standing bug which incorrectly caused `GET /_matrix/client/v3/rooms/{roomId}/event/{eventId}` to return edited events rather than the original. ([\#12476](https://github.com/matrix-org/synapse/issues/12476))
+- Fix a bug introduced in Synapse 1.27.0 where the admin API for [deleting forward extremities](https://github.com/matrix-org/synapse/blob/erikj/fix_delete_event_response_count/docs/admin_api/rooms.md#deleting-forward-extremities) would always return a count of 1, no matter how many extremities were deleted. ([\#12496](https://github.com/matrix-org/synapse/issues/12496))
+- Fix a long-standing bug where the image thumbnails embedded into email notifications were broken. ([\#12510](https://github.com/matrix-org/synapse/issues/12510))
+- Fix a bug in the implementation of [MSC3202](https://github.com/matrix-org/matrix-spec-proposals/pull/3202) where Synapse would use the field name `device_unused_fallback_keys`, rather than `device_unused_fallback_key_types`. ([\#12520](https://github.com/matrix-org/synapse/issues/12520))
 - Fix a bug introduced in Synapse 0.99.3 which could cause Synapse to consume large amounts of RAM when back-paginating in a large room. ([\#12522](https://github.com/matrix-org/synapse/issues/12522))
 
 
@@ -26,21 +27,20 @@ Improved Documentation
 ----------------------
 
 - Fix rendering of the documentation site when using the 'print' feature. ([\#12340](https://github.com/matrix-org/synapse/issues/12340))
-- Add a manual documenting config file options. ([\#12368](https://github.com/matrix-org/synapse/issues/12368))
+- Add a manual documenting config file options. ([\#12368](https://github.com/matrix-org/synapse/issues/12368), [\#12527](https://github.com/matrix-org/synapse/issues/12527))
 - Update documentation to reflect that both the `run_background_tasks_on` option and the options for moving stream writers off of the main process are no longer experimental. ([\#12451](https://github.com/matrix-org/synapse/issues/12451))
 - Update worker documentation and replace old `federation_reader` with `generic_worker`. ([\#12457](https://github.com/matrix-org/synapse/issues/12457))
-- Strongly recommend `poetry` for development. ([\#12475](https://github.com/matrix-org/synapse/issues/12475))
+- Strongly recommend Poetry for development. ([\#12475](https://github.com/matrix-org/synapse/issues/12475))
 - Add some example configurations for workers and update architectural diagram. ([\#12492](https://github.com/matrix-org/synapse/issues/12492))
 - Fix a broken link in `README.rst`. ([\#12495](https://github.com/matrix-org/synapse/issues/12495))
 - Add HAProxy delegation example with CORS headers to docs. ([\#12501](https://github.com/matrix-org/synapse/issues/12501))
-- Add an index to the configuration manual. ([\#12527](https://github.com/matrix-org/synapse/issues/12527))
 - Remove extraneous comma in User Admin API's device deletion section so that the example JSON is actually valid and works. Contributed by @olmari. ([\#12533](https://github.com/matrix-org/synapse/issues/12533))
 
 
 Deprecations and Removals
 -------------------------
 
-- The groups/communities feature in Synapse has been disabled by default. ([\#12344](https://github.com/matrix-org/synapse/issues/12344))
+- **The groups/communities feature in Synapse is now disabled by default. ([\#12344](https://github.com/matrix-org/synapse/issues/12344))**
 - Remove unstable identifiers from [MSC3440](https://github.com/matrix-org/matrix-doc/pull/3440). ([\#12382](https://github.com/matrix-org/synapse/issues/12382))
 
 
@@ -48,31 +48,24 @@ Internal Changes
 ----------------
 
 - Preparation for faster-room-join work: start a background process to resynchronise the room state after a room join. ([\#12394](https://github.com/matrix-org/synapse/issues/12394))
-- Remove an unstable identifier from [MSC3083](https://github.com/matrix-org/matrix-doc/pull/3083). ([\#12395](https://github.com/matrix-org/synapse/issues/12395))
 - Preparation for faster-room-join work: Implement a tracking mechanism to allow functions to wait for full room state to arrive. ([\#12399](https://github.com/matrix-org/synapse/issues/12399))
-- Run twisted trunk CI job in the locked poetry environment. ([\#12425](https://github.com/matrix-org/synapse/issues/12425))
-- Run lints under poetry in CI, and remove corresponding tox lint jobs. ([\#12434](https://github.com/matrix-org/synapse/issues/12434))
-- Run "main" trial tests under `poetry`. ([\#12438](https://github.com/matrix-org/synapse/issues/12438))
-- Bump twisted version in `poetry.lock` to work around [pip bug #9644](https://github.com/pypa/pip/issues/9644). ([\#12441](https://github.com/matrix-org/synapse/issues/12441))
-- Change Mutual Rooms' `unstable_features` flag to `uk.half-shot.msc2666.mutual_rooms` which matches the current MSC iteration. ([\#12445](https://github.com/matrix-org/synapse/issues/12445))
-- Use `poetry` to manage the virtualenv in debian packages. ([\#12449](https://github.com/matrix-org/synapse/issues/12449))
+- Remove an unstable identifier from [MSC3083](https://github.com/matrix-org/matrix-doc/pull/3083). ([\#12395](https://github.com/matrix-org/synapse/issues/12395))
+- Run CI in the locked Poetry environment, and remove corresponding `tox` jobs. ([\#12425](https://github.com/matrix-org/synapse/issues/12425)), [\#12434](https://github.com/matrix-org/synapse/issues/12434), [\#12438](https://github.com/matrix-org/synapse/issues/12438), [\#12441](https://github.com/matrix-org/synapse/issues/12441), [\#12449](https://github.com/matrix-org/synapse/issues/12449), [\#12478](https://github.com/matrix-org/synapse/issues/12478), [\#12514](https://github.com/matrix-org/synapse/issues/12514), [\#12472](https://github.com/matrix-org/synapse/issues/12472))
+- Change Mutual Rooms' `unstable_features` flag to `uk.half-shot.msc2666.mutual_rooms` which matches the current iteration of [MSC2666](https://github.com/matrix-org/matrix-spec-proposals/pull/2666). ([\#12445](https://github.com/matrix-org/synapse/issues/12445))
 - Fix typo in the release script help string. ([\#12450](https://github.com/matrix-org/synapse/issues/12450))
-- Limit length of device_id to less than 512 characters. ([\#12454](https://github.com/matrix-org/synapse/issues/12454))
+- Fix a minor typo in the Debian changelogs generated by the release script. ([\#12497](https://github.com/matrix-org/synapse/issues/12497))
 - Reintroduce the list of targets to the linter script, to avoid linting unwanted local-only directories during development. ([\#12455](https://github.com/matrix-org/synapse/issues/12455))
+- Limit length of `device_id` to less than 512 characters. ([\#12454](https://github.com/matrix-org/synapse/issues/12454))
 - Dockerfile-workers: reduce the amount we install in the image. ([\#12464](https://github.com/matrix-org/synapse/issues/12464))
 - Dockerfile-workers: give the master its own log config. ([\#12466](https://github.com/matrix-org/synapse/issues/12466))
 - complement-synapse-workers: factor out separate entry point script. ([\#12467](https://github.com/matrix-org/synapse/issues/12467))
-- Update `delay_cancellation` to accept any awaitable, rather than just `Deferred`s. ([\#12468](https://github.com/matrix-org/synapse/issues/12468))
-- Add a CI job which tests Synapse against the latest version of all dependencies. ([\#12472](https://github.com/matrix-org/synapse/issues/12472))
 - Back out experimental implementation of [MSC2314](https://github.com/matrix-org/matrix-spec-proposals/pull/2314). ([\#12474](https://github.com/matrix-org/synapse/issues/12474))
-- Use poetry-core instead of setuptools to build wheels. ([\#12478](https://github.com/matrix-org/synapse/issues/12478), [\#12514](https://github.com/matrix-org/synapse/issues/12514))
 - Fix grammatical error in federation error response when the room version of a room is unknown. ([\#12483](https://github.com/matrix-org/synapse/issues/12483))
-- Fix a minor typo in the Debian changelogs generated by the release script. ([\#12497](https://github.com/matrix-org/synapse/issues/12497))
 - Remove unnecessary configuration overrides in tests. ([\#12511](https://github.com/matrix-org/synapse/issues/12511))
 - Refactor the relations code for clarity. ([\#12519](https://github.com/matrix-org/synapse/issues/12519))
 - Add type hints so `docker` and `stubs` directories pass `mypy --disallow-untyped-defs`. ([\#12528](https://github.com/matrix-org/synapse/issues/12528))
+- Update `delay_cancellation` to accept any awaitable, rather than just `Deferred`s. ([\#12468](https://github.com/matrix-org/synapse/issues/12468))
 - Handle cancellation in `EventsWorkerStore._get_events_from_cache_or_db`. ([\#12529](https://github.com/matrix-org/synapse/issues/12529))
-- Build debian packages for Ubuntu 22.04 "Jammy Jellyfish". ([\#12543](https://github.com/matrix-org/synapse/issues/12543))
 
 
 Synapse 1.57.1 (2022-04-20)

From a54d9b0508c17a29f2f02c87054cd8990d476054 Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Tue, 26 Apr 2022 11:37:21 +0100
Subject: [PATCH 09/12] We don't require redbaron in the release script

---
 poetry.lock    | 62 +-------------------------------------------------
 pyproject.toml |  1 -
 2 files changed, 1 insertion(+), 62 deletions(-)

diff --git a/poetry.lock b/poetry.lock
index 95c1afc077..8c7af1fa1e 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -1,11 +1,3 @@
-[[package]]
-name = "appdirs"
-version = "1.4.4"
-description = "A small Python module for determining appropriate platform-specific dirs, e.g. a \"user data dir\"."
-category = "dev"
-optional = false
-python-versions = "*"
-
 [[package]]
 name = "attrs"
 version = "21.4.0"
@@ -49,17 +41,6 @@ six = "*"
 [package.extras]
 visualize = ["graphviz (>0.5.1)", "Twisted (>=16.1.1)"]
 
-[[package]]
-name = "baron"
-version = "0.10.1"
-description = "Full Syntax Tree for python to make writing refactoring code a realist task"
-category = "dev"
-optional = false
-python-versions = "*"
-
-[package.dependencies]
-rply = "*"
-
 [[package]]
 name = "bcrypt"
 version = "3.2.0"
@@ -984,20 +965,6 @@ Pygments = ">=2.5.1"
 [package.extras]
 md = ["cmarkgfm (>=0.8.0)"]
 
-[[package]]
-name = "redbaron"
-version = "0.9.2"
-description = "Abstraction on top of baron, a FST for python to make writing refactoring code a realistic task"
-category = "dev"
-optional = false
-python-versions = "*"
-
-[package.dependencies]
-baron = ">=0.7"
-
-[package.extras]
-notebook = ["pygments"]
-
 [[package]]
 name = "requests"
 version = "2.27.1"
@@ -1038,17 +1005,6 @@ python-versions = ">=3.7"
 [package.extras]
 idna2008 = ["idna"]
 
-[[package]]
-name = "rply"
-version = "0.7.8"
-description = "A pure Python Lex/Yacc that works with RPython"
-category = "dev"
-optional = false
-python-versions = "*"
-
-[package.dependencies]
-appdirs = "*"
-
 [[package]]
 name = "secretstorage"
 version = "3.3.1"
@@ -1597,13 +1553,9 @@ url_preview = ["lxml"]
 [metadata]
 lock-version = "1.1"
 python-versions = "^3.7"
-content-hash = "964ad29eaf7fd02749a4e735818f3bc0ba729c2f4b9e3213f0daa02643508b16"
+content-hash = "f482a4f594a165dfe01ce253a22510d5faf38647ab0dcebc35789350cafd9bf0"
 
 [metadata.files]
-appdirs = [
-    {file = "appdirs-1.4.4-py2.py3-none-any.whl", hash = "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128"},
-    {file = "appdirs-1.4.4.tar.gz", hash = "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41"},
-]
 attrs = [
     {file = "attrs-21.4.0-py2.py3-none-any.whl", hash = "sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4"},
     {file = "attrs-21.4.0.tar.gz", hash = "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd"},
@@ -1616,10 +1568,6 @@ automat = [
     {file = "Automat-20.2.0-py2.py3-none-any.whl", hash = "sha256:b6feb6455337df834f6c9962d6ccf771515b7d939bca142b29c20c2376bc6111"},
     {file = "Automat-20.2.0.tar.gz", hash = "sha256:7979803c74610e11ef0c0d68a2942b152df52da55336e0c9d58daf1831cbdf33"},
 ]
-baron = [
-    {file = "baron-0.10.1-py2.py3-none-any.whl", hash = "sha256:befb33f4b9e832c7cd1e3cf0eafa6dd3cb6ed4cb2544245147c019936f4e0a8a"},
-    {file = "baron-0.10.1.tar.gz", hash = "sha256:af822ad44d4eb425c8516df4239ac4fdba9fdb398ef77e4924cd7c9b4045bc2f"},
-]
 bcrypt = [
     {file = "bcrypt-3.2.0-cp36-abi3-macosx_10_10_universal2.whl", hash = "sha256:b589229207630484aefe5899122fb938a5b017b0f4349f769b8c13e78d99a8fd"},
     {file = "bcrypt-3.2.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:c95d4cbebffafcdd28bd28bb4e25b31c50f6da605c81ffd9ad8a3d1b2ab7b1b6"},
@@ -2412,10 +2360,6 @@ readme-renderer = [
     {file = "readme_renderer-33.0-py3-none-any.whl", hash = "sha256:f02cee0c4de9636b5a62b6be50c9742427ba1b956aad1d938bfb087d0d72ccdf"},
     {file = "readme_renderer-33.0.tar.gz", hash = "sha256:e3b53bc84bd6af054e4cc1fe3567dc1ae19f554134221043a3f8c674e22209db"},
 ]
-redbaron = [
-    {file = "redbaron-0.9.2-py2.py3-none-any.whl", hash = "sha256:d01032b6a848b5521a8d6ef72486315c2880f420956870cdd742e2b5a09b9bab"},
-    {file = "redbaron-0.9.2.tar.gz", hash = "sha256:472d0739ca6b2240bb2278ae428604a75472c9c12e86c6321e8c016139c0132f"},
-]
 requests = [
     {file = "requests-2.27.1-py2.py3-none-any.whl", hash = "sha256:f22fa1e554c9ddfd16e6e41ac79759e17be9e492b3587efa038054674760e72d"},
     {file = "requests-2.27.1.tar.gz", hash = "sha256:68d7c56fd5a8999887728ef304a6d12edc7be74f1cfa47714fc8b414525c9a61"},
@@ -2428,10 +2372,6 @@ rfc3986 = [
     {file = "rfc3986-2.0.0-py2.py3-none-any.whl", hash = "sha256:50b1502b60e289cb37883f3dfd34532b8873c7de9f49bb546641ce9cbd256ebd"},
     {file = "rfc3986-2.0.0.tar.gz", hash = "sha256:97aacf9dbd4bfd829baad6e6309fa6573aaf1be3f6fa735c8ab05e46cecb261c"},
 ]
-rply = [
-    {file = "rply-0.7.8-py2.py3-none-any.whl", hash = "sha256:28ffd11d656c48aeb8c508eb382acd6a0bd906662624b34388751732a27807e7"},
-    {file = "rply-0.7.8.tar.gz", hash = "sha256:2a808ac25a4580a9991fc304d64434e299a8fc75760574492f242cbb5bb301c9"},
-]
 secretstorage = [
     {file = "SecretStorage-3.3.1-py3-none-any.whl", hash = "sha256:422d82c36172d88d6a0ed5afdec956514b189ddbfb72fefab0c8a1cee4eaf71f"},
     {file = "SecretStorage-3.3.1.tar.gz", hash = "sha256:fd666c51a6bf200643495a04abb261f83229dcb6fd8472ec393df7ffc8b6f195"},
diff --git a/pyproject.toml b/pyproject.toml
index 80e25cef1d..cc4908fa23 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -270,7 +270,6 @@ idna = ">=2.5"
 
 # The following are used by the release script
 click = "==8.1.0"
-redbaron = "==0.9.2"
 GitPython = "==3.1.14"
 commonmark = "==0.9.1"
 pygithub = "==1.55"

From 416604e3bc94ce9c555b7fe4363296998fb1e945 Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Tue, 26 Apr 2022 11:51:47 +0100
Subject: [PATCH 10/12] Another set of changelog updates

---
 CHANGES.md | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/CHANGES.md b/CHANGES.md
index b9d477a6eb..58f0f0a13c 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,11 +1,13 @@
 Synapse 1.58.0rc1 (2022-04-26)
 ==============================
 
+As of this release, the groups/communities feature in Synapse is now disabled by default. See [\#11584](https://github.com/matrix-org/synapse/issues/11584) for details. As mentioned in [the upgrade notes](https://github.com/matrix-org/synapse/blob/develop/docs/upgrade.md#upgrading-to-v1580), this feature will be removed in Synapse 1.61.
+
 Features
 --------
 
 - Implement [MSC3383](https://github.com/matrix-org/matrix-spec-proposals/pull/3383) for including the destination in server-to-server authentication headers. Contributed by @Bubu and @jcgruenhage for Famedly. ([\#11398](https://github.com/matrix-org/synapse/issues/11398))
-- Docker images and Debian packages from matrix.org now contain a locked set of Python dependencies, greatly improving reproducibility. ([Board](https://github.com/orgs/matrix-org/projects/54), [\#11537](https://github.com/matrix-org/synapse/issues/11537))
+- Docker images and Debian packages from matrix.org now contain a locked set of Python dependencies, greatly improving build reproducibility. ([Board](https://github.com/orgs/matrix-org/projects/54), [\#11537](https://github.com/matrix-org/synapse/issues/11537))
 - Enable processing of device list updates asynchronously. ([\#12365](https://github.com/matrix-org/synapse/issues/12365), [\#12465](https://github.com/matrix-org/synapse/issues/12465))
 - Implement [MSC2815](https://github.com/matrix-org/matrix-spec-proposals/pull/2815) to allow room moderators to view redacted event content. Contributed by @tulir. ([\#12427](https://github.com/matrix-org/synapse/issues/12427))
 - Build Debian packages for Ubuntu 22.04 "Jammy Jellyfish". ([\#12543](https://github.com/matrix-org/synapse/issues/12543))
@@ -30,7 +32,7 @@ Improved Documentation
 - Add a manual documenting config file options. ([\#12368](https://github.com/matrix-org/synapse/issues/12368), [\#12527](https://github.com/matrix-org/synapse/issues/12527))
 - Update documentation to reflect that both the `run_background_tasks_on` option and the options for moving stream writers off of the main process are no longer experimental. ([\#12451](https://github.com/matrix-org/synapse/issues/12451))
 - Update worker documentation and replace old `federation_reader` with `generic_worker`. ([\#12457](https://github.com/matrix-org/synapse/issues/12457))
-- Strongly recommend Poetry for development. ([\#12475](https://github.com/matrix-org/synapse/issues/12475))
+- Strongly recommend [Poetry](https://python-poetry.org/) for development. ([\#12475](https://github.com/matrix-org/synapse/issues/12475))
 - Add some example configurations for workers and update architectural diagram. ([\#12492](https://github.com/matrix-org/synapse/issues/12492))
 - Fix a broken link in `README.rst`. ([\#12495](https://github.com/matrix-org/synapse/issues/12495))
 - Add HAProxy delegation example with CORS headers to docs. ([\#12501](https://github.com/matrix-org/synapse/issues/12501))
@@ -50,7 +52,7 @@ Internal Changes
 - Preparation for faster-room-join work: start a background process to resynchronise the room state after a room join. ([\#12394](https://github.com/matrix-org/synapse/issues/12394))
 - Preparation for faster-room-join work: Implement a tracking mechanism to allow functions to wait for full room state to arrive. ([\#12399](https://github.com/matrix-org/synapse/issues/12399))
 - Remove an unstable identifier from [MSC3083](https://github.com/matrix-org/matrix-doc/pull/3083). ([\#12395](https://github.com/matrix-org/synapse/issues/12395))
-- Run CI in the locked Poetry environment, and remove corresponding `tox` jobs. ([\#12425](https://github.com/matrix-org/synapse/issues/12425)), [\#12434](https://github.com/matrix-org/synapse/issues/12434), [\#12438](https://github.com/matrix-org/synapse/issues/12438), [\#12441](https://github.com/matrix-org/synapse/issues/12441), [\#12449](https://github.com/matrix-org/synapse/issues/12449), [\#12478](https://github.com/matrix-org/synapse/issues/12478), [\#12514](https://github.com/matrix-org/synapse/issues/12514), [\#12472](https://github.com/matrix-org/synapse/issues/12472))
+- Run CI in the locked [Poetry](https://python-poetry.org/) environment, and remove corresponding `tox` jobs. ([\#12425](https://github.com/matrix-org/synapse/issues/12425), [\#12434](https://github.com/matrix-org/synapse/issues/12434), [\#12438](https://github.com/matrix-org/synapse/issues/12438), [\#12441](https://github.com/matrix-org/synapse/issues/12441), [\#12449](https://github.com/matrix-org/synapse/issues/12449), [\#12478](https://github.com/matrix-org/synapse/issues/12478), [\#12514](https://github.com/matrix-org/synapse/issues/12514), [\#12472](https://github.com/matrix-org/synapse/issues/12472))
 - Change Mutual Rooms' `unstable_features` flag to `uk.half-shot.msc2666.mutual_rooms` which matches the current iteration of [MSC2666](https://github.com/matrix-org/matrix-spec-proposals/pull/2666). ([\#12445](https://github.com/matrix-org/synapse/issues/12445))
 - Fix typo in the release script help string. ([\#12450](https://github.com/matrix-org/synapse/issues/12450))
 - Fix a minor typo in the Debian changelogs generated by the release script. ([\#12497](https://github.com/matrix-org/synapse/issues/12497))

From 6b9e95015b39898011991a73051a71cb7dab3d1f Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Tue, 26 Apr 2022 11:53:37 +0100
Subject: [PATCH 11/12] Lint the release script

---
 scripts-dev/release.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scripts-dev/release.py b/scripts-dev/release.py
index 725e8f0479..9d7c7c445f 100755
--- a/scripts-dev/release.py
+++ b/scripts-dev/release.py
@@ -31,7 +31,6 @@ import attr
 import click
 import commonmark
 import git
-
 from click.exceptions import ClickException
 from github import Github
 from packaging import version

From ee1601e59d8a3de24914d4c08af4012daeb1689e Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Tue, 26 Apr 2022 11:59:10 +0100
Subject: [PATCH 12/12] Unbold deprecation: it is mentioned at the top

---
 CHANGES.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGES.md b/CHANGES.md
index 58f0f0a13c..d8ecb11e4e 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -42,7 +42,7 @@ Improved Documentation
 Deprecations and Removals
 -------------------------
 
-- **The groups/communities feature in Synapse is now disabled by default. ([\#12344](https://github.com/matrix-org/synapse/issues/12344))**
+- The groups/communities feature in Synapse is now disabled by default. ([\#12344](https://github.com/matrix-org/synapse/issues/12344))
 - Remove unstable identifiers from [MSC3440](https://github.com/matrix-org/matrix-doc/pull/3440). ([\#12382](https://github.com/matrix-org/synapse/issues/12382))