From c1ef579b63a39d8d6fb31ddc1d3cf173eaf7e5b7 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com>
Date: Thu, 1 Oct 2020 11:09:12 +0100
Subject: [PATCH 1/9] Add prometheus metrics to track federation delays (#8430)

Add a pair of federation metrics to track the delays in sending PDUs to/from
particular servers.
---
 changelog.d/8430.feature                      |  1 +
 docs/sample_config.yaml                       | 12 +++++++++
 synapse/config/_util.py                       |  6 +++--
 synapse/config/federation.py                  | 27 ++++++++++++++++++-
 synapse/config/homeserver.py                  |  1 -
 synapse/config/tls.py                         |  1 -
 synapse/federation/federation_server.py       | 24 ++++++++++++++++-
 .../federation/sender/transaction_manager.py  | 22 +++++++++++++++
 8 files changed, 88 insertions(+), 6 deletions(-)
 create mode 100644 changelog.d/8430.feature

diff --git a/changelog.d/8430.feature b/changelog.d/8430.feature
new file mode 100644
index 0000000000..1f31d42bc1
--- /dev/null
+++ b/changelog.d/8430.feature
@@ -0,0 +1 @@
+Add prometheus metrics to track federation delays.
diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml
index 066844b5a9..8a3206e845 100644
--- a/docs/sample_config.yaml
+++ b/docs/sample_config.yaml
@@ -629,6 +629,7 @@ acme:
 #tls_fingerprints: [{"sha256": "<base64_encoded_sha256_fingerprint>"}]
 
 
+## Federation ##
 
 # Restrict federation to the following whitelist of domains.
 # N.B. we recommend also firewalling your federation listener to limit
@@ -662,6 +663,17 @@ federation_ip_range_blacklist:
   - 'fe80::/64'
   - 'fc00::/7'
 
+# Report prometheus metrics on the age of PDUs being sent to and received from
+# the following domains. This can be used to give an idea of "delay" on inbound
+# and outbound federation, though be aware that any delay can be due to problems
+# at either end or with the intermediate network.
+#
+# By default, no domains are monitored in this way.
+#
+#federation_metrics_domains:
+#  - matrix.org
+#  - example.com
+
 
 ## Caching ##
 
diff --git a/synapse/config/_util.py b/synapse/config/_util.py
index cd31b1c3c9..c74969a977 100644
--- a/synapse/config/_util.py
+++ b/synapse/config/_util.py
@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from typing import Any, List
+from typing import Any, Iterable
 
 import jsonschema
 
@@ -20,7 +20,9 @@ from synapse.config._base import ConfigError
 from synapse.types import JsonDict
 
 
-def validate_config(json_schema: JsonDict, config: Any, config_path: List[str]) -> None:
+def validate_config(
+    json_schema: JsonDict, config: Any, config_path: Iterable[str]
+) -> None:
     """Validates a config setting against a JsonSchema definition
 
     This can be used to validate a section of the config file against a schema
diff --git a/synapse/config/federation.py b/synapse/config/federation.py
index 2c77d8f85b..ffd8fca54e 100644
--- a/synapse/config/federation.py
+++ b/synapse/config/federation.py
@@ -17,7 +17,8 @@ from typing import Optional
 
 from netaddr import IPSet
 
-from ._base import Config, ConfigError
+from synapse.config._base import Config, ConfigError
+from synapse.config._util import validate_config
 
 
 class FederationConfig(Config):
@@ -52,8 +53,18 @@ class FederationConfig(Config):
                 "Invalid range(s) provided in federation_ip_range_blacklist: %s" % e
             )
 
+        federation_metrics_domains = config.get("federation_metrics_domains") or []
+        validate_config(
+            _METRICS_FOR_DOMAINS_SCHEMA,
+            federation_metrics_domains,
+            ("federation_metrics_domains",),
+        )
+        self.federation_metrics_domains = set(federation_metrics_domains)
+
     def generate_config_section(self, config_dir_path, server_name, **kwargs):
         return """\
+        ## Federation ##
+
         # Restrict federation to the following whitelist of domains.
         # N.B. we recommend also firewalling your federation listener to limit
         # inbound federation traffic as early as possible, rather than relying
@@ -85,4 +96,18 @@ class FederationConfig(Config):
           - '::1/128'
           - 'fe80::/64'
           - 'fc00::/7'
+
+        # Report prometheus metrics on the age of PDUs being sent to and received from
+        # the following domains. This can be used to give an idea of "delay" on inbound
+        # and outbound federation, though be aware that any delay can be due to problems
+        # at either end or with the intermediate network.
+        #
+        # By default, no domains are monitored in this way.
+        #
+        #federation_metrics_domains:
+        #  - matrix.org
+        #  - example.com
         """
+
+
+_METRICS_FOR_DOMAINS_SCHEMA = {"type": "array", "items": {"type": "string"}}
diff --git a/synapse/config/homeserver.py b/synapse/config/homeserver.py
index 556e291495..be65554524 100644
--- a/synapse/config/homeserver.py
+++ b/synapse/config/homeserver.py
@@ -92,5 +92,4 @@ class HomeServerConfig(RootConfig):
         TracerConfig,
         WorkerConfig,
         RedisConfig,
-        FederationConfig,
     ]
diff --git a/synapse/config/tls.py b/synapse/config/tls.py
index e368ea564d..9ddb8b546b 100644
--- a/synapse/config/tls.py
+++ b/synapse/config/tls.py
@@ -471,7 +471,6 @@ class TlsConfig(Config):
         # or by checking matrix.org/federationtester/api/report?server_name=$host
         #
         #tls_fingerprints: [{"sha256": "<base64_encoded_sha256_fingerprint>"}]
-
         """
             # Lowercase the string representation of boolean values
             % {
diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py
index 2dcd081cbc..24329dd0e3 100644
--- a/synapse/federation/federation_server.py
+++ b/synapse/federation/federation_server.py
@@ -28,7 +28,7 @@ from typing import (
     Union,
 )
 
-from prometheus_client import Counter, Histogram
+from prometheus_client import Counter, Gauge, Histogram
 
 from twisted.internet import defer
 from twisted.internet.abstract import isIPAddress
@@ -88,6 +88,13 @@ pdu_process_time = Histogram(
 )
 
 
+last_pdu_age_metric = Gauge(
+    "synapse_federation_last_received_pdu_age",
+    "The age (in seconds) of the last PDU successfully received from the given domain",
+    labelnames=("server_name",),
+)
+
+
 class FederationServer(FederationBase):
     def __init__(self, hs):
         super().__init__(hs)
@@ -118,6 +125,10 @@ class FederationServer(FederationBase):
             hs, "state_ids_resp", timeout_ms=30000
         )
 
+        self._federation_metrics_domains = (
+            hs.get_config().federation.federation_metrics_domains
+        )
+
     async def on_backfill_request(
         self, origin: str, room_id: str, versions: List[str], limit: int
     ) -> Tuple[int, Dict[str, Any]]:
@@ -262,7 +273,11 @@ class FederationServer(FederationBase):
 
         pdus_by_room = {}  # type: Dict[str, List[EventBase]]
 
+        newest_pdu_ts = 0
+
         for p in transaction.pdus:  # type: ignore
+            # FIXME (richardv): I don't think this works:
+            #  https://github.com/matrix-org/synapse/issues/8429
             if "unsigned" in p:
                 unsigned = p["unsigned"]
                 if "age" in unsigned:
@@ -300,6 +315,9 @@ class FederationServer(FederationBase):
             event = event_from_pdu_json(p, room_version)
             pdus_by_room.setdefault(room_id, []).append(event)
 
+            if event.origin_server_ts > newest_pdu_ts:
+                newest_pdu_ts = event.origin_server_ts
+
         pdu_results = {}
 
         # we can process different rooms in parallel (which is useful if they
@@ -340,6 +358,10 @@ class FederationServer(FederationBase):
             process_pdus_for_room, pdus_by_room.keys(), TRANSACTION_CONCURRENCY_LIMIT
         )
 
+        if newest_pdu_ts and origin in self._federation_metrics_domains:
+            newest_pdu_age = self._clock.time_msec() - newest_pdu_ts
+            last_pdu_age_metric.labels(server_name=origin).set(newest_pdu_age / 1000)
+
         return pdu_results
 
     async def _handle_edus_in_txn(self, origin: str, transaction: Transaction):
diff --git a/synapse/federation/sender/transaction_manager.py b/synapse/federation/sender/transaction_manager.py
index c84072ab73..3e07f925e0 100644
--- a/synapse/federation/sender/transaction_manager.py
+++ b/synapse/federation/sender/transaction_manager.py
@@ -15,6 +15,8 @@
 import logging
 from typing import TYPE_CHECKING, List
 
+from prometheus_client import Gauge
+
 from synapse.api.errors import HttpResponseException
 from synapse.events import EventBase
 from synapse.federation.persistence import TransactionActions
@@ -34,6 +36,12 @@ if TYPE_CHECKING:
 
 logger = logging.getLogger(__name__)
 
+last_pdu_age_metric = Gauge(
+    "synapse_federation_last_sent_pdu_age",
+    "The age (in seconds) of the last PDU successfully sent to the given domain",
+    labelnames=("server_name",),
+)
+
 
 class TransactionManager:
     """Helper class which handles building and sending transactions
@@ -48,6 +56,10 @@ class TransactionManager:
         self._transaction_actions = TransactionActions(self._store)
         self._transport_layer = hs.get_federation_transport_client()
 
+        self._federation_metrics_domains = (
+            hs.get_config().federation.federation_metrics_domains
+        )
+
         # HACK to get unique tx id
         self._next_txn_id = int(self.clock.time_msec())
 
@@ -119,6 +131,9 @@ class TransactionManager:
 
             # FIXME (erikj): This is a bit of a hack to make the Pdu age
             # keys work
+            # FIXME (richardv): I also believe it no longer works. We (now?) store
+            #  "age_ts" in "unsigned" rather than at the top level. See
+            #  https://github.com/matrix-org/synapse/issues/8429.
             def json_data_cb():
                 data = transaction.get_dict()
                 now = int(self.clock.time_msec())
@@ -167,5 +182,12 @@ class TransactionManager:
                     )
                 success = False
 
+            if success and pdus and destination in self._federation_metrics_domains:
+                last_pdu = pdus[-1]
+                last_pdu_age = self.clock.time_msec() - last_pdu.origin_server_ts
+                last_pdu_age_metric.labels(server_name=destination).set(
+                    last_pdu_age / 1000
+                )
+
             set_tag(tags.ERROR, not success)
             return success

From 4ff0201e6235b8b2efc5ce5a7dc3c479ea96df53 Mon Sep 17 00:00:00 2001
From: Patrick Cloke <clokep@users.noreply.github.com>
Date: Thu, 1 Oct 2020 08:09:18 -0400
Subject: [PATCH 2/9] Enable mypy checking for unreachable code and fix
 instances. (#8432)

---
 changelog.d/8432.misc                          |  1 +
 mypy.ini                                       |  1 +
 synapse/config/tls.py                          | 18 +++++++++---------
 synapse/federation/federation_server.py        |  5 ++---
 synapse/handlers/directory.py                  |  2 +-
 synapse/handlers/room.py                       |  2 --
 synapse/handlers/room_member.py                |  2 +-
 synapse/handlers/sync.py                       |  2 +-
 synapse/http/server.py                         |  4 ++--
 synapse/logging/_structured.py                 | 10 +---------
 synapse/push/push_rule_evaluator.py            |  4 ++--
 synapse/replication/tcp/protocol.py            | 10 ++++++----
 synapse/state/__init__.py                      |  2 +-
 .../storage/databases/main/censor_events.py    |  6 +++---
 synapse/storage/databases/main/events.py       | 18 +++++-------------
 synapse/storage/databases/main/stream.py       |  2 +-
 synapse/storage/util/id_generators.py          |  2 +-
 17 files changed, 38 insertions(+), 53 deletions(-)
 create mode 100644 changelog.d/8432.misc

diff --git a/changelog.d/8432.misc b/changelog.d/8432.misc
new file mode 100644
index 0000000000..01fdad4caf
--- /dev/null
+++ b/changelog.d/8432.misc
@@ -0,0 +1 @@
+Check for unreachable code with mypy.
diff --git a/mypy.ini b/mypy.ini
index 7986781432..c283f15b21 100644
--- a/mypy.ini
+++ b/mypy.ini
@@ -6,6 +6,7 @@ check_untyped_defs = True
 show_error_codes = True
 show_traceback = True
 mypy_path = stubs
+warn_unreachable = True
 files =
   synapse/api,
   synapse/appservice,
diff --git a/synapse/config/tls.py b/synapse/config/tls.py
index 9ddb8b546b..ad37b93c02 100644
--- a/synapse/config/tls.py
+++ b/synapse/config/tls.py
@@ -18,7 +18,7 @@ import os
 import warnings
 from datetime import datetime
 from hashlib import sha256
-from typing import List
+from typing import List, Optional
 
 from unpaddedbase64 import encode_base64
 
@@ -177,8 +177,8 @@ class TlsConfig(Config):
             "use_insecure_ssl_client_just_for_testing_do_not_use"
         )
 
-        self.tls_certificate = None
-        self.tls_private_key = None
+        self.tls_certificate = None  # type: Optional[crypto.X509]
+        self.tls_private_key = None  # type: Optional[crypto.PKey]
 
     def is_disk_cert_valid(self, allow_self_signed=True):
         """
@@ -226,12 +226,12 @@ class TlsConfig(Config):
         days_remaining = (expires_on - now).days
         return days_remaining
 
-    def read_certificate_from_disk(self, require_cert_and_key):
+    def read_certificate_from_disk(self, require_cert_and_key: bool):
         """
         Read the certificates and private key from disk.
 
         Args:
-            require_cert_and_key (bool): set to True to throw an error if the certificate
+            require_cert_and_key: set to True to throw an error if the certificate
                 and key file are not given
         """
         if require_cert_and_key:
@@ -479,13 +479,13 @@ class TlsConfig(Config):
             }
         )
 
-    def read_tls_certificate(self):
+    def read_tls_certificate(self) -> crypto.X509:
         """Reads the TLS certificate from the configured file, and returns it
 
         Also checks if it is self-signed, and warns if so
 
         Returns:
-            OpenSSL.crypto.X509: the certificate
+            The certificate
         """
         cert_path = self.tls_certificate_file
         logger.info("Loading TLS certificate from %s", cert_path)
@@ -504,11 +504,11 @@ class TlsConfig(Config):
 
         return cert
 
-    def read_tls_private_key(self):
+    def read_tls_private_key(self) -> crypto.PKey:
         """Reads the TLS private key from the configured file, and returns it
 
         Returns:
-            OpenSSL.crypto.PKey: the private key
+            The private key
         """
         private_key_path = self.tls_private_key_file
         logger.info("Loading TLS key from %s", private_key_path)
diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py
index 24329dd0e3..02f11e1209 100644
--- a/synapse/federation/federation_server.py
+++ b/synapse/federation/federation_server.py
@@ -22,7 +22,6 @@ from typing import (
     Callable,
     Dict,
     List,
-    Match,
     Optional,
     Tuple,
     Union,
@@ -825,14 +824,14 @@ def server_matches_acl_event(server_name: str, acl_event: EventBase) -> bool:
     return False
 
 
-def _acl_entry_matches(server_name: str, acl_entry: str) -> Match:
+def _acl_entry_matches(server_name: str, acl_entry: Any) -> bool:
     if not isinstance(acl_entry, str):
         logger.warning(
             "Ignoring non-str ACL entry '%s' (is %s)", acl_entry, type(acl_entry)
         )
         return False
     regex = glob_to_regex(acl_entry)
-    return regex.match(server_name)
+    return bool(regex.match(server_name))
 
 
 class FederationHandlerRegistry:
diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py
index 62aa9a2da8..6f15c68240 100644
--- a/synapse/handlers/directory.py
+++ b/synapse/handlers/directory.py
@@ -383,7 +383,7 @@ class DirectoryHandler(BaseHandler):
         """
         creator = await self.store.get_room_alias_creator(alias.to_string())
 
-        if creator is not None and creator == user_id:
+        if creator == user_id:
             return True
 
         # Resolve the alias to the corresponding room.
diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py
index d5f7c78edf..f1a6699cd4 100644
--- a/synapse/handlers/room.py
+++ b/synapse/handlers/room.py
@@ -962,8 +962,6 @@ class RoomCreationHandler(BaseHandler):
             try:
                 random_string = stringutils.random_string(18)
                 gen_room_id = RoomID(random_string, self.hs.hostname).to_string()
-                if isinstance(gen_room_id, bytes):
-                    gen_room_id = gen_room_id.decode("utf-8")
                 await self.store.store_room(
                     room_id=gen_room_id,
                     room_creator_user_id=creator_id,
diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py
index 8feba8c90a..5ec36f591d 100644
--- a/synapse/handlers/room_member.py
+++ b/synapse/handlers/room_member.py
@@ -642,7 +642,7 @@ class RoomMemberHandler(metaclass=abc.ABCMeta):
 
     async def send_membership_event(
         self,
-        requester: Requester,
+        requester: Optional[Requester],
         event: EventBase,
         context: EventContext,
         ratelimit: bool = True,
diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py
index bfe2583002..260ec19b41 100644
--- a/synapse/handlers/sync.py
+++ b/synapse/handlers/sync.py
@@ -87,7 +87,7 @@ class SyncConfig:
 class TimelineBatch:
     prev_batch = attr.ib(type=StreamToken)
     events = attr.ib(type=List[EventBase])
-    limited = attr.ib(bool)
+    limited = attr.ib(type=bool)
 
     def __bool__(self) -> bool:
         """Make the result appear empty if there are no updates. This is used
diff --git a/synapse/http/server.py b/synapse/http/server.py
index 996a31a9ec..09ed74f6ce 100644
--- a/synapse/http/server.py
+++ b/synapse/http/server.py
@@ -257,7 +257,7 @@ class _AsyncResource(resource.Resource, metaclass=abc.ABCMeta):
             if isinstance(raw_callback_return, (defer.Deferred, types.CoroutineType)):
                 callback_return = await raw_callback_return
             else:
-                callback_return = raw_callback_return
+                callback_return = raw_callback_return  # type: ignore
 
             return callback_return
 
@@ -406,7 +406,7 @@ class JsonResource(DirectServeJsonResource):
         if isinstance(raw_callback_return, (defer.Deferred, types.CoroutineType)):
             callback_return = await raw_callback_return
         else:
-            callback_return = raw_callback_return
+            callback_return = raw_callback_return  # type: ignore
 
         return callback_return
 
diff --git a/synapse/logging/_structured.py b/synapse/logging/_structured.py
index 144506c8f2..0fc2ea609e 100644
--- a/synapse/logging/_structured.py
+++ b/synapse/logging/_structured.py
@@ -12,7 +12,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 import logging
 import os.path
 import sys
@@ -89,14 +88,7 @@ class LogContextObserver:
         context = current_context()
 
         # Copy the context information to the log event.
-        if context is not None:
-            context.copy_to_twisted_log_entry(event)
-        else:
-            # If there's no logging context, not even the root one, we might be
-            # starting up or it might be from non-Synapse code. Log it as if it
-            # came from the root logger.
-            event["request"] = None
-            event["scope"] = None
+        context.copy_to_twisted_log_entry(event)
 
         self.observer(event)
 
diff --git a/synapse/push/push_rule_evaluator.py b/synapse/push/push_rule_evaluator.py
index 709ace01e5..3a68ce636f 100644
--- a/synapse/push/push_rule_evaluator.py
+++ b/synapse/push/push_rule_evaluator.py
@@ -16,7 +16,7 @@
 
 import logging
 import re
-from typing import Any, Dict, List, Pattern, Union
+from typing import Any, Dict, List, Optional, Pattern, Union
 
 from synapse.events import EventBase
 from synapse.types import UserID
@@ -181,7 +181,7 @@ class PushRuleEvaluatorForEvent:
 
         return r.search(body)
 
-    def _get_value(self, dotted_key: str) -> str:
+    def _get_value(self, dotted_key: str) -> Optional[str]:
         return self._value_cache.get(dotted_key, None)
 
 
diff --git a/synapse/replication/tcp/protocol.py b/synapse/replication/tcp/protocol.py
index 0b0d204e64..a509e599c2 100644
--- a/synapse/replication/tcp/protocol.py
+++ b/synapse/replication/tcp/protocol.py
@@ -51,10 +51,11 @@ import fcntl
 import logging
 import struct
 from inspect import isawaitable
-from typing import TYPE_CHECKING, List
+from typing import TYPE_CHECKING, List, Optional
 
 from prometheus_client import Counter
 
+from twisted.internet import task
 from twisted.protocols.basic import LineOnlyReceiver
 from twisted.python.failure import Failure
 
@@ -152,9 +153,10 @@ class BaseReplicationStreamProtocol(LineOnlyReceiver):
 
         self.last_received_command = self.clock.time_msec()
         self.last_sent_command = 0
-        self.time_we_closed = None  # When we requested the connection be closed
+        # When we requested the connection be closed
+        self.time_we_closed = None  # type: Optional[int]
 
-        self.received_ping = False  # Have we reecived a ping from the other side
+        self.received_ping = False  # Have we received a ping from the other side
 
         self.state = ConnectionStates.CONNECTING
 
@@ -165,7 +167,7 @@ class BaseReplicationStreamProtocol(LineOnlyReceiver):
         self.pending_commands = []  # type: List[Command]
 
         # The LoopingCall for sending pings.
-        self._send_ping_loop = None
+        self._send_ping_loop = None  # type: Optional[task.LoopingCall]
 
         # a logcontext which we use for processing incoming commands. We declare it as a
         # background process so that the CPU stats get reported to prometheus.
diff --git a/synapse/state/__init__.py b/synapse/state/__init__.py
index 31082bb16a..5b0900aa3c 100644
--- a/synapse/state/__init__.py
+++ b/synapse/state/__init__.py
@@ -738,7 +738,7 @@ def _make_state_cache_entry(
 
     # failing that, look for the closest match.
     prev_group = None
-    delta_ids = None
+    delta_ids = None  # type: Optional[StateMap[str]]
 
     for old_group, old_state in state_groups_ids.items():
         n_delta_ids = {k: v for k, v in new_state.items() if old_state.get(k) != v}
diff --git a/synapse/storage/databases/main/censor_events.py b/synapse/storage/databases/main/censor_events.py
index f211ddbaf8..4bb2b9c28c 100644
--- a/synapse/storage/databases/main/censor_events.py
+++ b/synapse/storage/databases/main/censor_events.py
@@ -21,8 +21,8 @@ from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.storage._base import SQLBaseStore
 from synapse.storage.database import DatabasePool
 from synapse.storage.databases.main.cache import CacheInvalidationWorkerStore
-from synapse.storage.databases.main.events import encode_json
 from synapse.storage.databases.main.events_worker import EventsWorkerStore
+from synapse.util.frozenutils import frozendict_json_encoder
 
 if TYPE_CHECKING:
     from synapse.server import HomeServer
@@ -105,7 +105,7 @@ class CensorEventsStore(EventsWorkerStore, CacheInvalidationWorkerStore, SQLBase
                 and original_event.internal_metadata.is_redacted()
             ):
                 # Redaction was allowed
-                pruned_json = encode_json(
+                pruned_json = frozendict_json_encoder.encode(
                     prune_event_dict(
                         original_event.room_version, original_event.get_dict()
                     )
@@ -171,7 +171,7 @@ class CensorEventsStore(EventsWorkerStore, CacheInvalidationWorkerStore, SQLBase
                 return
 
             # Prune the event's dict then convert it to JSON.
-            pruned_json = encode_json(
+            pruned_json = frozendict_json_encoder.encode(
                 prune_event_dict(event.room_version, event.get_dict())
             )
 
diff --git a/synapse/storage/databases/main/events.py b/synapse/storage/databases/main/events.py
index 18def01f50..78e645592f 100644
--- a/synapse/storage/databases/main/events.py
+++ b/synapse/storage/databases/main/events.py
@@ -52,16 +52,6 @@ event_counter = Counter(
 )
 
 
-def encode_json(json_object):
-    """
-    Encode a Python object as JSON and return it in a Unicode string.
-    """
-    out = frozendict_json_encoder.encode(json_object)
-    if isinstance(out, bytes):
-        out = out.decode("utf8")
-    return out
-
-
 _EventCacheEntry = namedtuple("_EventCacheEntry", ("event", "redacted_event"))
 
 
@@ -743,7 +733,9 @@ class PersistEventsStore:
                     logger.exception("")
                     raise
 
-                metadata_json = encode_json(event.internal_metadata.get_dict())
+                metadata_json = frozendict_json_encoder.encode(
+                    event.internal_metadata.get_dict()
+                )
 
                 sql = "UPDATE event_json SET internal_metadata = ? WHERE event_id = ?"
                 txn.execute(sql, (metadata_json, event.event_id))
@@ -797,10 +789,10 @@ class PersistEventsStore:
                 {
                     "event_id": event.event_id,
                     "room_id": event.room_id,
-                    "internal_metadata": encode_json(
+                    "internal_metadata": frozendict_json_encoder.encode(
                         event.internal_metadata.get_dict()
                     ),
-                    "json": encode_json(event_dict(event)),
+                    "json": frozendict_json_encoder.encode(event_dict(event)),
                     "format_version": event.format_version,
                 }
                 for event, _ in events_and_contexts
diff --git a/synapse/storage/databases/main/stream.py b/synapse/storage/databases/main/stream.py
index 37249f1e3f..1d27439536 100644
--- a/synapse/storage/databases/main/stream.py
+++ b/synapse/storage/databases/main/stream.py
@@ -546,7 +546,7 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore, metaclass=abc.ABCMeta):
 
     async def get_room_event_before_stream_ordering(
         self, room_id: str, stream_ordering: int
-    ) -> Tuple[int, int, str]:
+    ) -> Optional[Tuple[int, int, str]]:
         """Gets details of the first event in a room at or before a stream ordering
 
         Args:
diff --git a/synapse/storage/util/id_generators.py b/synapse/storage/util/id_generators.py
index 02fbb656e8..ec356b2e4f 100644
--- a/synapse/storage/util/id_generators.py
+++ b/synapse/storage/util/id_generators.py
@@ -421,7 +421,7 @@ class MultiWriterIdGenerator:
             self._unfinished_ids.discard(next_id)
             self._finished_ids.add(next_id)
 
-            new_cur = None
+            new_cur = None  # type: Optional[int]
 
             if self._unfinished_ids:
                 # If there are unfinished IDs then the new position will be the

From cc40a59b4a94534105667ae95fd5602ebdc57dce Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Thu, 1 Oct 2020 13:14:56 +0100
Subject: [PATCH 3/9] 1.21.0

---
 CHANGES.md               | 102 +++++++++++++++++++++++++++++++++++++++
 changelog.d/7124.bugfix  |   1 -
 changelog.d/7796.bugfix  |   1 -
 changelog.d/7905.bugfix  |   1 -
 changelog.d/8004.feature |   1 -
 changelog.d/8208.misc    |   1 -
 changelog.d/8216.misc    |   1 -
 changelog.d/8217.feature |   1 -
 changelog.d/8227.doc     |   1 -
 changelog.d/8230.bugfix  |   1 -
 changelog.d/8236.bugfix  |   1 -
 changelog.d/8243.misc    |   1 -
 changelog.d/8247.bugfix  |   1 -
 changelog.d/8248.feature |   1 -
 changelog.d/8250.misc    |   1 -
 changelog.d/8256.misc    |   1 -
 changelog.d/8257.misc    |   1 -
 changelog.d/8258.bugfix  |   1 -
 changelog.d/8259.misc    |   1 -
 changelog.d/8260.misc    |   1 -
 changelog.d/8261.misc    |   1 -
 changelog.d/8262.bugfix  |   1 -
 changelog.d/8265.bugfix  |   1 -
 changelog.d/8268.bugfix  |   1 -
 changelog.d/8272.bugfix  |   1 -
 changelog.d/8275.feature |   1 -
 changelog.d/8278.bugfix  |   1 -
 changelog.d/8279.misc    |   1 -
 changelog.d/8281.misc    |   1 -
 changelog.d/8282.misc    |   1 -
 changelog.d/8287.bugfix  |   1 -
 changelog.d/8288.misc    |   1 -
 changelog.d/8294.feature |   1 -
 changelog.d/8296.misc    |   1 -
 changelog.d/8305.feature |   1 -
 changelog.d/8306.feature |   1 -
 changelog.d/8317.feature |   1 -
 changelog.d/8320.feature |   1 -
 changelog.d/8322.bugfix  |   1 -
 changelog.d/8324.bugfix  |   1 -
 changelog.d/8326.misc    |   1 -
 changelog.d/8329.bugfix  |   1 -
 changelog.d/8330.misc    |   1 -
 changelog.d/8331.misc    |   1 -
 changelog.d/8335.misc    |   1 -
 changelog.d/8337.misc    |   1 -
 changelog.d/8344.misc    |   1 -
 changelog.d/8345.feature |   1 -
 changelog.d/8353.bugfix  |   1 -
 changelog.d/8354.misc    |   1 -
 changelog.d/8362.bugfix  |   1 -
 changelog.d/8364.bugfix  |   2 -
 changelog.d/8370.misc    |   1 -
 changelog.d/8371.misc    |   1 -
 changelog.d/8372.misc    |   1 -
 changelog.d/8373.bugfix  |   1 -
 changelog.d/8374.bugfix  |   1 -
 changelog.d/8375.doc     |   1 -
 changelog.d/8377.misc    |   1 -
 changelog.d/8383.misc    |   1 -
 changelog.d/8385.bugfix  |   1 -
 changelog.d/8386.bugfix  |   1 -
 changelog.d/8387.feature |   1 -
 changelog.d/8388.misc    |   1 -
 changelog.d/8396.feature |   1 -
 changelog.d/8398.bugfix  |   1 -
 changelog.d/8399.misc    |   1 -
 changelog.d/8400.bugfix  |   1 -
 changelog.d/8401.misc    |   1 -
 changelog.d/8402.misc    |   1 -
 changelog.d/8404.misc    |   1 -
 changelog.d/8405.feature |   1 -
 changelog.d/8406.feature |   1 -
 changelog.d/8410.bugfix  |   1 -
 changelog.d/8413.feature |   1 -
 changelog.d/8414.bugfix  |   1 -
 changelog.d/8415.doc     |   1 -
 changelog.d/8417.feature |   1 -
 changelog.d/8419.feature |   1 -
 changelog.d/8420.feature |   1 -
 changelog.d/8422.misc    |   1 -
 changelog.d/8423.misc    |   1 -
 changelog.d/8425.feature |   1 -
 changelog.d/8426.removal |   1 -
 changelog.d/8427.misc    |   1 -
 changelog.d/8430.feature |   1 -
 synapse/__init__.py      |   2 +-
 87 files changed, 103 insertions(+), 87 deletions(-)
 delete mode 100644 changelog.d/7124.bugfix
 delete mode 100644 changelog.d/7796.bugfix
 delete mode 100644 changelog.d/7905.bugfix
 delete mode 100644 changelog.d/8004.feature
 delete mode 100644 changelog.d/8208.misc
 delete mode 100644 changelog.d/8216.misc
 delete mode 100644 changelog.d/8217.feature
 delete mode 100644 changelog.d/8227.doc
 delete mode 100644 changelog.d/8230.bugfix
 delete mode 100644 changelog.d/8236.bugfix
 delete mode 100644 changelog.d/8243.misc
 delete mode 100644 changelog.d/8247.bugfix
 delete mode 100644 changelog.d/8248.feature
 delete mode 100644 changelog.d/8250.misc
 delete mode 100644 changelog.d/8256.misc
 delete mode 100644 changelog.d/8257.misc
 delete mode 100644 changelog.d/8258.bugfix
 delete mode 100644 changelog.d/8259.misc
 delete mode 100644 changelog.d/8260.misc
 delete mode 100644 changelog.d/8261.misc
 delete mode 100644 changelog.d/8262.bugfix
 delete mode 100644 changelog.d/8265.bugfix
 delete mode 100644 changelog.d/8268.bugfix
 delete mode 100644 changelog.d/8272.bugfix
 delete mode 100644 changelog.d/8275.feature
 delete mode 100644 changelog.d/8278.bugfix
 delete mode 100644 changelog.d/8279.misc
 delete mode 100644 changelog.d/8281.misc
 delete mode 100644 changelog.d/8282.misc
 delete mode 100644 changelog.d/8287.bugfix
 delete mode 100644 changelog.d/8288.misc
 delete mode 100644 changelog.d/8294.feature
 delete mode 100644 changelog.d/8296.misc
 delete mode 100644 changelog.d/8305.feature
 delete mode 100644 changelog.d/8306.feature
 delete mode 100644 changelog.d/8317.feature
 delete mode 100644 changelog.d/8320.feature
 delete mode 100644 changelog.d/8322.bugfix
 delete mode 100644 changelog.d/8324.bugfix
 delete mode 100644 changelog.d/8326.misc
 delete mode 100644 changelog.d/8329.bugfix
 delete mode 100644 changelog.d/8330.misc
 delete mode 100644 changelog.d/8331.misc
 delete mode 100644 changelog.d/8335.misc
 delete mode 100644 changelog.d/8337.misc
 delete mode 100644 changelog.d/8344.misc
 delete mode 100644 changelog.d/8345.feature
 delete mode 100644 changelog.d/8353.bugfix
 delete mode 100644 changelog.d/8354.misc
 delete mode 100644 changelog.d/8362.bugfix
 delete mode 100644 changelog.d/8364.bugfix
 delete mode 100644 changelog.d/8370.misc
 delete mode 100644 changelog.d/8371.misc
 delete mode 100644 changelog.d/8372.misc
 delete mode 100644 changelog.d/8373.bugfix
 delete mode 100644 changelog.d/8374.bugfix
 delete mode 100644 changelog.d/8375.doc
 delete mode 100644 changelog.d/8377.misc
 delete mode 100644 changelog.d/8383.misc
 delete mode 100644 changelog.d/8385.bugfix
 delete mode 100644 changelog.d/8386.bugfix
 delete mode 100644 changelog.d/8387.feature
 delete mode 100644 changelog.d/8388.misc
 delete mode 100644 changelog.d/8396.feature
 delete mode 100644 changelog.d/8398.bugfix
 delete mode 100644 changelog.d/8399.misc
 delete mode 100644 changelog.d/8400.bugfix
 delete mode 100644 changelog.d/8401.misc
 delete mode 100644 changelog.d/8402.misc
 delete mode 100644 changelog.d/8404.misc
 delete mode 100644 changelog.d/8405.feature
 delete mode 100644 changelog.d/8406.feature
 delete mode 100644 changelog.d/8410.bugfix
 delete mode 100644 changelog.d/8413.feature
 delete mode 100644 changelog.d/8414.bugfix
 delete mode 100644 changelog.d/8415.doc
 delete mode 100644 changelog.d/8417.feature
 delete mode 100644 changelog.d/8419.feature
 delete mode 100644 changelog.d/8420.feature
 delete mode 100644 changelog.d/8422.misc
 delete mode 100644 changelog.d/8423.misc
 delete mode 100644 changelog.d/8425.feature
 delete mode 100644 changelog.d/8426.removal
 delete mode 100644 changelog.d/8427.misc
 delete mode 100644 changelog.d/8430.feature

diff --git a/CHANGES.md b/CHANGES.md
index 5de819ea1e..38906ade49 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,3 +1,105 @@
+Synapse 1.21.0 (2020-10-01)
+===========================
+
+Features
+--------
+
+- Require the user to confirm that their password should be reset after clicking the email confirmation link. ([\#8004](https://github.com/matrix-org/synapse/issues/8004))
+- Add an admin API `GET /_synapse/admin/v1/event_reports` to read entries of table `event_reports`. Contributed by @dklimpel. ([\#8217](https://github.com/matrix-org/synapse/issues/8217))
+- Consolidate the SSO error template across all configuration. ([\#8248](https://github.com/matrix-org/synapse/issues/8248), [\#8405](https://github.com/matrix-org/synapse/issues/8405))
+- Add a configuration option to specify a whitelist of domains that a user can be redirected to after validating their email or phone number. ([\#8275](https://github.com/matrix-org/synapse/issues/8275), [\#8417](https://github.com/matrix-org/synapse/issues/8417))
+- Add experimental support for sharding event persister. ([\#8294](https://github.com/matrix-org/synapse/issues/8294), [\#8387](https://github.com/matrix-org/synapse/issues/8387), [\#8396](https://github.com/matrix-org/synapse/issues/8396), [\#8419](https://github.com/matrix-org/synapse/issues/8419))
+- Add the room topic and avatar to the room details admin API. ([\#8305](https://github.com/matrix-org/synapse/issues/8305))
+- Add an admin API for querying rooms where a user is a member. Contributed by @dklimpel. ([\#8306](https://github.com/matrix-org/synapse/issues/8306))
+- Add `uk.half-shot.msc2778.login.application_service` login type to allow appservices to login. ([\#8320](https://github.com/matrix-org/synapse/issues/8320))
+- Add a configuration option that allows existing users to log in with OpenID Connect. Contributed by @BBBSnowball and @OmmyZhang. ([\#8345](https://github.com/matrix-org/synapse/issues/8345))
+- Add prometheus metrics for replication requests. ([\#8406](https://github.com/matrix-org/synapse/issues/8406))
+- Support passing additional single sign-on parameters to the client. ([\#8413](https://github.com/matrix-org/synapse/issues/8413))
+- Add experimental reporting of metrics on expensive rooms for state-resolution. ([\#8420](https://github.com/matrix-org/synapse/issues/8420))
+- Add experimental prometheus metric to track numbers of "large" rooms for state resolutiom. ([\#8425](https://github.com/matrix-org/synapse/issues/8425))
+- Add prometheus metrics to track federation delays. ([\#8430](https://github.com/matrix-org/synapse/issues/8430))
+
+
+Bugfixes
+--------
+
+- Fix a bug in the media repository where remote thumbnails with the same size but different crop methods would overwrite each other. Contributed by @deepbluev7. ([\#7124](https://github.com/matrix-org/synapse/issues/7124))
+- Fix inconsistent handling of non-existent push rules, and stop tracking the `enabled` state of removed push rules. ([\#7796](https://github.com/matrix-org/synapse/issues/7796))
+- Fix a longstanding bug when storing a media file with an empty `upload_name`. ([\#7905](https://github.com/matrix-org/synapse/issues/7905))
+- Fix messages not being sent over federation until an event is sent into the same room. ([\#8230](https://github.com/matrix-org/synapse/issues/8230), [\#8247](https://github.com/matrix-org/synapse/issues/8247), [\#8258](https://github.com/matrix-org/synapse/issues/8258), [\#8272](https://github.com/matrix-org/synapse/issues/8272), [\#8322](https://github.com/matrix-org/synapse/issues/8322))
+- Fix a longstanding bug where files that could not be thumbnailed would result in an Internal Server Error. ([\#8236](https://github.com/matrix-org/synapse/issues/8236))
+- Upgrade minimum version of `canonicaljson` to version 1.4.0, to fix an unicode encoding issue. ([\#8262](https://github.com/matrix-org/synapse/issues/8262))
+- Fix logstanding bug which could lead to incomplete database upgrades on SQLite. ([\#8265](https://github.com/matrix-org/synapse/issues/8265))
+- Fix stack overflow when stderr is redirected to the logging system, and the logging system encounters an error. ([\#8268](https://github.com/matrix-org/synapse/issues/8268))
+- Fix a bug which cause the logging system to report errors, if `DEBUG` was enabled and no `context` filter was applied. ([\#8278](https://github.com/matrix-org/synapse/issues/8278))
+- Fix edge case where push could get delayed for a user until a later event was pushed. ([\#8287](https://github.com/matrix-org/synapse/issues/8287))
+- Fix fetching malformed events from remote servers. ([\#8324](https://github.com/matrix-org/synapse/issues/8324))
+- Fix `UnboundLocalError` from occuring when appservices send a malformed register request. ([\#8329](https://github.com/matrix-org/synapse/issues/8329))
+- Don't send push notifications to expired user accounts. ([\#8353](https://github.com/matrix-org/synapse/issues/8353))
+- Fix a regression in v1.19.0 with reactivating users through the admin API. ([\#8362](https://github.com/matrix-org/synapse/issues/8362))
+- Fix a bug where during device registration the length of the device name wasn't limited. ([\#8364](https://github.com/matrix-org/synapse/issues/8364))
+- Include `guest_access` in the fields that are checked for null bytes when updating `room_stats_state`. Broke in v1.7.2. ([\#8373](https://github.com/matrix-org/synapse/issues/8373))
+- Fix theoretical race condition where events are not sent down `/sync` if the synchrotron worker is restarted without restarting other workers. ([\#8374](https://github.com/matrix-org/synapse/issues/8374))
+- Fix a bug which could cause errors in rooms with malformed membership events, on servers using sqlite. ([\#8385](https://github.com/matrix-org/synapse/issues/8385))
+- Fix a bug introduced in v1.20.0 which caused the `synapse_port_db` script to fail. ([\#8386](https://github.com/matrix-org/synapse/issues/8386))
+- Fix "Re-starting finished log context" warning when receiving an event we already had over federation. ([\#8398](https://github.com/matrix-org/synapse/issues/8398))
+- Fix incorrect handling of timeouts on outgoing HTTP requests. ([\#8400](https://github.com/matrix-org/synapse/issues/8400))
+- Fix a regression in v1.20.0 in the `synapse_port_db` script regarding the `ui_auth_sessions_ips` table. ([\#8410](https://github.com/matrix-org/synapse/issues/8410))
+- Remove unnecessary 3PID registration check when resetting password via an email address. Bug introduced in v0.34.0rc2. ([\#8414](https://github.com/matrix-org/synapse/issues/8414))
+
+
+Improved Documentation
+----------------------
+
+- Add `/_synapse/client` to the reverse proxy documentation. ([\#8227](https://github.com/matrix-org/synapse/issues/8227))
+- Add note to the reverse proxy settings documentation about disabling Apache's mod_security2. Contributed by Julian Fietkau (@jfietkau). ([\#8375](https://github.com/matrix-org/synapse/issues/8375))
+- Improve description of `server_name` config option in `homserver.yaml`. ([\#8415](https://github.com/matrix-org/synapse/issues/8415))
+
+
+Deprecations and Removals
+-------------------------
+
+- Drop support for `prometheus_client` older than 0.4.0. ([\#8426](https://github.com/matrix-org/synapse/issues/8426))
+
+
+Internal Changes
+----------------
+
+- Fix tests on distros which disable TLSv1.0. Contributed by @danc86. ([\#8208](https://github.com/matrix-org/synapse/issues/8208))
+- Simplify the distributor code to avoid unnecessary work. ([\#8216](https://github.com/matrix-org/synapse/issues/8216))
+- Remove the `populate_stats_process_rooms_2` background job and restore functionality to `populate_stats_process_rooms`. ([\#8243](https://github.com/matrix-org/synapse/issues/8243))
+- Clean up type hints for `PaginationConfig`. ([\#8250](https://github.com/matrix-org/synapse/issues/8250), [\#8282](https://github.com/matrix-org/synapse/issues/8282))
+- Track the latest event for every destination and room for catch-up after federation outage. ([\#8256](https://github.com/matrix-org/synapse/issues/8256))
+- Fix non-user visible bug in implementation of `MultiWriterIdGenerator.get_current_token_for_writer`. ([\#8257](https://github.com/matrix-org/synapse/issues/8257))
+- Switch to the JSON implementation from the standard library. ([\#8259](https://github.com/matrix-org/synapse/issues/8259))
+- Add type hints to `synapse.util.async_helpers`. ([\#8260](https://github.com/matrix-org/synapse/issues/8260))
+- Simplify tests that mock asynchronous functions. ([\#8261](https://github.com/matrix-org/synapse/issues/8261))
+- Add type hints to `StreamToken` and `RoomStreamToken` classes. ([\#8279](https://github.com/matrix-org/synapse/issues/8279))
+- Change `StreamToken.room_key` to be a `RoomStreamToken` instance. ([\#8281](https://github.com/matrix-org/synapse/issues/8281))
+- Refactor notifier code to correctly use the max event stream position. ([\#8288](https://github.com/matrix-org/synapse/issues/8288))
+- Use slotted classes where possible. ([\#8296](https://github.com/matrix-org/synapse/issues/8296))
+- Support testing the local Synapse checkout against the [Complement homeserver test suite](https://github.com/matrix-org/complement/). ([\#8317](https://github.com/matrix-org/synapse/issues/8317))
+- Update outdated usages of `metaclass` to python 3 syntax. ([\#8326](https://github.com/matrix-org/synapse/issues/8326))
+- Move lint-related dependencies to package-extra field, update CONTRIBUTING.md to utilise this. ([\#8330](https://github.com/matrix-org/synapse/issues/8330), [\#8377](https://github.com/matrix-org/synapse/issues/8377))
+- Use the `admin_patterns` helper in additional locations. ([\#8331](https://github.com/matrix-org/synapse/issues/8331))
+- Fix test logging to allow braces in log output. ([\#8335](https://github.com/matrix-org/synapse/issues/8335))
+- Remove `__future__` imports related to Python 2 compatibility. ([\#8337](https://github.com/matrix-org/synapse/issues/8337))
+- Simplify `super()` calls to Python 3 syntax. ([\#8344](https://github.com/matrix-org/synapse/issues/8344))
+- Fix bad merge from `release-v1.20.0` branch to `develop`. ([\#8354](https://github.com/matrix-org/synapse/issues/8354))
+- Factor out a `_send_dummy_event_for_room` method. ([\#8370](https://github.com/matrix-org/synapse/issues/8370))
+- Improve logging of state resolution. ([\#8371](https://github.com/matrix-org/synapse/issues/8371))
+- Add type annotations to `SimpleHttpClient`. ([\#8372](https://github.com/matrix-org/synapse/issues/8372))
+- Refactor ID generators to use `async with` syntax. ([\#8383](https://github.com/matrix-org/synapse/issues/8383))
+- Add `EventStreamPosition` type. ([\#8388](https://github.com/matrix-org/synapse/issues/8388))
+- Create a mechanism for marking tests "logcontext clean". ([\#8399](https://github.com/matrix-org/synapse/issues/8399))
+- A pair of tiny cleanups in the federation request code. ([\#8401](https://github.com/matrix-org/synapse/issues/8401))
+- Add checks on startup that PostgreSQL sequences are consistent with their associated tables. ([\#8402](https://github.com/matrix-org/synapse/issues/8402))
+- Do not include appservice users when calculating the total MAU for a server. ([\#8404](https://github.com/matrix-org/synapse/issues/8404))
+- Typing fixes for `synapse.handlers.federation`. ([\#8422](https://github.com/matrix-org/synapse/issues/8422))
+- Various refactors to simplify stream token handling. ([\#8423](https://github.com/matrix-org/synapse/issues/8423))
+- Make stream token serializing/deserializing async. ([\#8427](https://github.com/matrix-org/synapse/issues/8427))
+
+
 Synapse 1.20.1 (2020-09-24)
 ===========================
 
diff --git a/changelog.d/7124.bugfix b/changelog.d/7124.bugfix
deleted file mode 100644
index 8fd177780d..0000000000
--- a/changelog.d/7124.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix a bug in the media repository where remote thumbnails with the same size but different crop methods would overwrite each other. Contributed by @deepbluev7.
diff --git a/changelog.d/7796.bugfix b/changelog.d/7796.bugfix
deleted file mode 100644
index 65e5eb42a2..0000000000
--- a/changelog.d/7796.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix inconsistent handling of non-existent push rules, and stop tracking the `enabled` state of removed push rules.
diff --git a/changelog.d/7905.bugfix b/changelog.d/7905.bugfix
deleted file mode 100644
index e60e624412..0000000000
--- a/changelog.d/7905.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix a longstanding bug when storing a media file with an empty `upload_name`.
diff --git a/changelog.d/8004.feature b/changelog.d/8004.feature
deleted file mode 100644
index a91b75e0e0..0000000000
--- a/changelog.d/8004.feature
+++ /dev/null
@@ -1 +0,0 @@
-Require the user to confirm that their password should be reset after clicking the email confirmation link.
\ No newline at end of file
diff --git a/changelog.d/8208.misc b/changelog.d/8208.misc
deleted file mode 100644
index e65da88c46..0000000000
--- a/changelog.d/8208.misc
+++ /dev/null
@@ -1 +0,0 @@
-Fix tests on distros which disable TLSv1.0. Contributed by @danc86.
diff --git a/changelog.d/8216.misc b/changelog.d/8216.misc
deleted file mode 100644
index b38911b0e5..0000000000
--- a/changelog.d/8216.misc
+++ /dev/null
@@ -1 +0,0 @@
-Simplify the distributor code to avoid unnecessary work.
diff --git a/changelog.d/8217.feature b/changelog.d/8217.feature
deleted file mode 100644
index 899cbf14ef..0000000000
--- a/changelog.d/8217.feature
+++ /dev/null
@@ -1 +0,0 @@
-Add an admin API `GET /_synapse/admin/v1/event_reports` to read entries of table `event_reports`. Contributed by @dklimpel.
\ No newline at end of file
diff --git a/changelog.d/8227.doc b/changelog.d/8227.doc
deleted file mode 100644
index 4a43015a83..0000000000
--- a/changelog.d/8227.doc
+++ /dev/null
@@ -1 +0,0 @@
-Add `/_synapse/client` to the reverse proxy documentation.
diff --git a/changelog.d/8230.bugfix b/changelog.d/8230.bugfix
deleted file mode 100644
index 532d0e22fe..0000000000
--- a/changelog.d/8230.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix messages over federation being lost until an event is sent into the same room.
diff --git a/changelog.d/8236.bugfix b/changelog.d/8236.bugfix
deleted file mode 100644
index 6f04871015..0000000000
--- a/changelog.d/8236.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix a longstanding bug where files that could not be thumbnailed would result in an Internal Server Error.
diff --git a/changelog.d/8243.misc b/changelog.d/8243.misc
deleted file mode 100644
index f7375d32d3..0000000000
--- a/changelog.d/8243.misc
+++ /dev/null
@@ -1 +0,0 @@
-Remove the 'populate_stats_process_rooms_2' background job and restore functionality to 'populate_stats_process_rooms'.
\ No newline at end of file
diff --git a/changelog.d/8247.bugfix b/changelog.d/8247.bugfix
deleted file mode 100644
index 532d0e22fe..0000000000
--- a/changelog.d/8247.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix messages over federation being lost until an event is sent into the same room.
diff --git a/changelog.d/8248.feature b/changelog.d/8248.feature
deleted file mode 100644
index f3c4a74bc7..0000000000
--- a/changelog.d/8248.feature
+++ /dev/null
@@ -1 +0,0 @@
-Consolidate the SSO error template across all configuration.
diff --git a/changelog.d/8250.misc b/changelog.d/8250.misc
deleted file mode 100644
index b6896a9300..0000000000
--- a/changelog.d/8250.misc
+++ /dev/null
@@ -1 +0,0 @@
-Clean up type hints for `PaginationConfig`.
diff --git a/changelog.d/8256.misc b/changelog.d/8256.misc
deleted file mode 100644
index bf0ba76730..0000000000
--- a/changelog.d/8256.misc
+++ /dev/null
@@ -1 +0,0 @@
-Track the latest event for every destination and room for catch-up after federation outage.
diff --git a/changelog.d/8257.misc b/changelog.d/8257.misc
deleted file mode 100644
index 47ac583eb4..0000000000
--- a/changelog.d/8257.misc
+++ /dev/null
@@ -1 +0,0 @@
-Fix non-user visible bug in implementation of `MultiWriterIdGenerator.get_current_token_for_writer`.
diff --git a/changelog.d/8258.bugfix b/changelog.d/8258.bugfix
deleted file mode 100644
index 532d0e22fe..0000000000
--- a/changelog.d/8258.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix messages over federation being lost until an event is sent into the same room.
diff --git a/changelog.d/8259.misc b/changelog.d/8259.misc
deleted file mode 100644
index a26779a664..0000000000
--- a/changelog.d/8259.misc
+++ /dev/null
@@ -1 +0,0 @@
-Switch to the JSON implementation from the standard library.
diff --git a/changelog.d/8260.misc b/changelog.d/8260.misc
deleted file mode 100644
index 164eea8b59..0000000000
--- a/changelog.d/8260.misc
+++ /dev/null
@@ -1 +0,0 @@
-Add type hints to `synapse.util.async_helpers`.
diff --git a/changelog.d/8261.misc b/changelog.d/8261.misc
deleted file mode 100644
index bc91e9375c..0000000000
--- a/changelog.d/8261.misc
+++ /dev/null
@@ -1 +0,0 @@
-Simplify tests that mock asynchronous functions.
diff --git a/changelog.d/8262.bugfix b/changelog.d/8262.bugfix
deleted file mode 100644
index 2b84927de3..0000000000
--- a/changelog.d/8262.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Upgrade canonicaljson to version 1.4.0 to fix an unicode encoding issue.
diff --git a/changelog.d/8265.bugfix b/changelog.d/8265.bugfix
deleted file mode 100644
index 981a836d21..0000000000
--- a/changelog.d/8265.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix logstanding bug which could lead to incomplete database upgrades on SQLite.
diff --git a/changelog.d/8268.bugfix b/changelog.d/8268.bugfix
deleted file mode 100644
index 4b15a60253..0000000000
--- a/changelog.d/8268.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix stack overflow when stderr is redirected to the logging system, and the logging system encounters an error.
diff --git a/changelog.d/8272.bugfix b/changelog.d/8272.bugfix
deleted file mode 100644
index 532d0e22fe..0000000000
--- a/changelog.d/8272.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix messages over federation being lost until an event is sent into the same room.
diff --git a/changelog.d/8275.feature b/changelog.d/8275.feature
deleted file mode 100644
index 17549c3df3..0000000000
--- a/changelog.d/8275.feature
+++ /dev/null
@@ -1 +0,0 @@
-Add a config option to specify a whitelist of domains that a user can be redirected to after validating their email or phone number.
\ No newline at end of file
diff --git a/changelog.d/8278.bugfix b/changelog.d/8278.bugfix
deleted file mode 100644
index 50e40ca2a9..0000000000
--- a/changelog.d/8278.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix a bug which cause the logging system to report errors, if `DEBUG` was enabled and no `context` filter was applied.
diff --git a/changelog.d/8279.misc b/changelog.d/8279.misc
deleted file mode 100644
index 99f669001f..0000000000
--- a/changelog.d/8279.misc
+++ /dev/null
@@ -1 +0,0 @@
-Add type hints to `StreamToken` and `RoomStreamToken` classes.
diff --git a/changelog.d/8281.misc b/changelog.d/8281.misc
deleted file mode 100644
index 74357120a7..0000000000
--- a/changelog.d/8281.misc
+++ /dev/null
@@ -1 +0,0 @@
-Change `StreamToken.room_key` to be a `RoomStreamToken` instance.
diff --git a/changelog.d/8282.misc b/changelog.d/8282.misc
deleted file mode 100644
index b6896a9300..0000000000
--- a/changelog.d/8282.misc
+++ /dev/null
@@ -1 +0,0 @@
-Clean up type hints for `PaginationConfig`.
diff --git a/changelog.d/8287.bugfix b/changelog.d/8287.bugfix
deleted file mode 100644
index 839781aa07..0000000000
--- a/changelog.d/8287.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix edge case where push could get delayed for a user until a later event was pushed.
diff --git a/changelog.d/8288.misc b/changelog.d/8288.misc
deleted file mode 100644
index c08a53a5ee..0000000000
--- a/changelog.d/8288.misc
+++ /dev/null
@@ -1 +0,0 @@
-Refactor notifier code to correctly use the max event stream position.
diff --git a/changelog.d/8294.feature b/changelog.d/8294.feature
deleted file mode 100644
index b363e929ea..0000000000
--- a/changelog.d/8294.feature
+++ /dev/null
@@ -1 +0,0 @@
-Add experimental support for sharding event persister.
diff --git a/changelog.d/8296.misc b/changelog.d/8296.misc
deleted file mode 100644
index f593a5b347..0000000000
--- a/changelog.d/8296.misc
+++ /dev/null
@@ -1 +0,0 @@
-Use slotted classes where possible.
diff --git a/changelog.d/8305.feature b/changelog.d/8305.feature
deleted file mode 100644
index 862dfdf959..0000000000
--- a/changelog.d/8305.feature
+++ /dev/null
@@ -1 +0,0 @@
-Add the room topic and avatar to the room details admin API.
diff --git a/changelog.d/8306.feature b/changelog.d/8306.feature
deleted file mode 100644
index 5c23da4030..0000000000
--- a/changelog.d/8306.feature
+++ /dev/null
@@ -1 +0,0 @@
-Add an admin API for querying rooms where a user is a member. Contributed by @dklimpel.
\ No newline at end of file
diff --git a/changelog.d/8317.feature b/changelog.d/8317.feature
deleted file mode 100644
index f9edda099c..0000000000
--- a/changelog.d/8317.feature
+++ /dev/null
@@ -1 +0,0 @@
-Support testing the local Synapse checkout against the [Complement homeserver test suite](https://github.com/matrix-org/complement/).
\ No newline at end of file
diff --git a/changelog.d/8320.feature b/changelog.d/8320.feature
deleted file mode 100644
index 475a5fe62d..0000000000
--- a/changelog.d/8320.feature
+++ /dev/null
@@ -1 +0,0 @@
-Add `uk.half-shot.msc2778.login.application_service` login type to allow appservices to login.
diff --git a/changelog.d/8322.bugfix b/changelog.d/8322.bugfix
deleted file mode 100644
index 532d0e22fe..0000000000
--- a/changelog.d/8322.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix messages over federation being lost until an event is sent into the same room.
diff --git a/changelog.d/8324.bugfix b/changelog.d/8324.bugfix
deleted file mode 100644
index 32788a9284..0000000000
--- a/changelog.d/8324.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix fetching events from remote servers that are malformed.
diff --git a/changelog.d/8326.misc b/changelog.d/8326.misc
deleted file mode 100644
index 985d2c027a..0000000000
--- a/changelog.d/8326.misc
+++ /dev/null
@@ -1 +0,0 @@
-Update outdated usages of `metaclass` to python 3 syntax.
\ No newline at end of file
diff --git a/changelog.d/8329.bugfix b/changelog.d/8329.bugfix
deleted file mode 100644
index 2f71f1f4b9..0000000000
--- a/changelog.d/8329.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix UnboundLocalError from occuring when appservices send malformed register request.
\ No newline at end of file
diff --git a/changelog.d/8330.misc b/changelog.d/8330.misc
deleted file mode 100644
index fbfdd52473..0000000000
--- a/changelog.d/8330.misc
+++ /dev/null
@@ -1 +0,0 @@
-Move lint-related dependencies to package-extra field, update CONTRIBUTING.md to utilise this.
diff --git a/changelog.d/8331.misc b/changelog.d/8331.misc
deleted file mode 100644
index 0e1bae20ef..0000000000
--- a/changelog.d/8331.misc
+++ /dev/null
@@ -1 +0,0 @@
-Use the `admin_patterns` helper in additional locations.
diff --git a/changelog.d/8335.misc b/changelog.d/8335.misc
deleted file mode 100644
index 7e0a4c7d83..0000000000
--- a/changelog.d/8335.misc
+++ /dev/null
@@ -1 +0,0 @@
-Fix test logging to allow braces in log output.
\ No newline at end of file
diff --git a/changelog.d/8337.misc b/changelog.d/8337.misc
deleted file mode 100644
index 4daf272204..0000000000
--- a/changelog.d/8337.misc
+++ /dev/null
@@ -1 +0,0 @@
-Remove `__future__` imports related to Python 2 compatibility.
\ No newline at end of file
diff --git a/changelog.d/8344.misc b/changelog.d/8344.misc
deleted file mode 100644
index 0b342d5137..0000000000
--- a/changelog.d/8344.misc
+++ /dev/null
@@ -1 +0,0 @@
-Simplify `super()` calls to Python 3 syntax.
diff --git a/changelog.d/8345.feature b/changelog.d/8345.feature
deleted file mode 100644
index 4ee5b6a56e..0000000000
--- a/changelog.d/8345.feature
+++ /dev/null
@@ -1 +0,0 @@
-Add a configuration option that allows existing users to log in with OpenID Connect. Contributed by @BBBSnowball and @OmmyZhang.
diff --git a/changelog.d/8353.bugfix b/changelog.d/8353.bugfix
deleted file mode 100644
index 45fc0adb8d..0000000000
--- a/changelog.d/8353.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Don't send push notifications to expired user accounts.
diff --git a/changelog.d/8354.misc b/changelog.d/8354.misc
deleted file mode 100644
index 1d33cde2da..0000000000
--- a/changelog.d/8354.misc
+++ /dev/null
@@ -1 +0,0 @@
-Fix bad merge from `release-v1.20.0` branch to `develop`.
diff --git a/changelog.d/8362.bugfix b/changelog.d/8362.bugfix
deleted file mode 100644
index 4e50067c87..0000000000
--- a/changelog.d/8362.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fixed a regression in v1.19.0 with reactivating users through the admin API.
diff --git a/changelog.d/8364.bugfix b/changelog.d/8364.bugfix
deleted file mode 100644
index 7b82cbc388..0000000000
--- a/changelog.d/8364.bugfix
+++ /dev/null
@@ -1,2 +0,0 @@
-Fix a bug where during device registration the length of the device name wasn't
-limited.
diff --git a/changelog.d/8370.misc b/changelog.d/8370.misc
deleted file mode 100644
index 1aaac1e0bf..0000000000
--- a/changelog.d/8370.misc
+++ /dev/null
@@ -1 +0,0 @@
-Factor out a `_send_dummy_event_for_room` method.
diff --git a/changelog.d/8371.misc b/changelog.d/8371.misc
deleted file mode 100644
index 6a54a9496a..0000000000
--- a/changelog.d/8371.misc
+++ /dev/null
@@ -1 +0,0 @@
-Improve logging of state resolution.
diff --git a/changelog.d/8372.misc b/changelog.d/8372.misc
deleted file mode 100644
index a56e36de4b..0000000000
--- a/changelog.d/8372.misc
+++ /dev/null
@@ -1 +0,0 @@
-Add type annotations to `SimpleHttpClient`.
diff --git a/changelog.d/8373.bugfix b/changelog.d/8373.bugfix
deleted file mode 100644
index e9d66a2088..0000000000
--- a/changelog.d/8373.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Include `guest_access` in the fields that are checked for null bytes when updating `room_stats_state`. Broke in v1.7.2.
\ No newline at end of file
diff --git a/changelog.d/8374.bugfix b/changelog.d/8374.bugfix
deleted file mode 100644
index 155bc3404f..0000000000
--- a/changelog.d/8374.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix theoretical race condition where events are not sent down `/sync` if the synchrotron worker is restarted without restarting other workers.
diff --git a/changelog.d/8375.doc b/changelog.d/8375.doc
deleted file mode 100644
index d291fb92fa..0000000000
--- a/changelog.d/8375.doc
+++ /dev/null
@@ -1 +0,0 @@
-Add note to the reverse proxy settings documentation about disabling Apache's mod_security2. Contributed by Julian Fietkau (@jfietkau).
diff --git a/changelog.d/8377.misc b/changelog.d/8377.misc
deleted file mode 100644
index fbfdd52473..0000000000
--- a/changelog.d/8377.misc
+++ /dev/null
@@ -1 +0,0 @@
-Move lint-related dependencies to package-extra field, update CONTRIBUTING.md to utilise this.
diff --git a/changelog.d/8383.misc b/changelog.d/8383.misc
deleted file mode 100644
index cb8318bf57..0000000000
--- a/changelog.d/8383.misc
+++ /dev/null
@@ -1 +0,0 @@
-Refactor ID generators to use `async with` syntax.
diff --git a/changelog.d/8385.bugfix b/changelog.d/8385.bugfix
deleted file mode 100644
index c42502a8e0..0000000000
--- a/changelog.d/8385.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix a bug which could cause errors in rooms with malformed membership events, on servers using sqlite.
diff --git a/changelog.d/8386.bugfix b/changelog.d/8386.bugfix
deleted file mode 100644
index 24983a1e95..0000000000
--- a/changelog.d/8386.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix a bug introduced in v1.20.0 which caused the `synapse_port_db` script to fail.
diff --git a/changelog.d/8387.feature b/changelog.d/8387.feature
deleted file mode 100644
index b363e929ea..0000000000
--- a/changelog.d/8387.feature
+++ /dev/null
@@ -1 +0,0 @@
-Add experimental support for sharding event persister.
diff --git a/changelog.d/8388.misc b/changelog.d/8388.misc
deleted file mode 100644
index aaaef88b66..0000000000
--- a/changelog.d/8388.misc
+++ /dev/null
@@ -1 +0,0 @@
-Add `EventStreamPosition` type.
diff --git a/changelog.d/8396.feature b/changelog.d/8396.feature
deleted file mode 100644
index b363e929ea..0000000000
--- a/changelog.d/8396.feature
+++ /dev/null
@@ -1 +0,0 @@
-Add experimental support for sharding event persister.
diff --git a/changelog.d/8398.bugfix b/changelog.d/8398.bugfix
deleted file mode 100644
index e432aeebf1..0000000000
--- a/changelog.d/8398.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix "Re-starting finished log context" warning when receiving an event we already had over federation.
diff --git a/changelog.d/8399.misc b/changelog.d/8399.misc
deleted file mode 100644
index ce6e8123cf..0000000000
--- a/changelog.d/8399.misc
+++ /dev/null
@@ -1 +0,0 @@
-Create a mechanism for marking tests "logcontext clean".
diff --git a/changelog.d/8400.bugfix b/changelog.d/8400.bugfix
deleted file mode 100644
index 835658ba5e..0000000000
--- a/changelog.d/8400.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix incorrect handling of timeouts on outgoing HTTP requests.
diff --git a/changelog.d/8401.misc b/changelog.d/8401.misc
deleted file mode 100644
index 27fd7ab129..0000000000
--- a/changelog.d/8401.misc
+++ /dev/null
@@ -1 +0,0 @@
-A pair of tiny cleanups in the federation request code.
diff --git a/changelog.d/8402.misc b/changelog.d/8402.misc
deleted file mode 100644
index ad1804d207..0000000000
--- a/changelog.d/8402.misc
+++ /dev/null
@@ -1 +0,0 @@
-Add checks on startup that PostgreSQL sequences are consistent with their associated tables.
diff --git a/changelog.d/8404.misc b/changelog.d/8404.misc
deleted file mode 100644
index 7aadded6c1..0000000000
--- a/changelog.d/8404.misc
+++ /dev/null
@@ -1 +0,0 @@
-Do not include appservice users when calculating the total MAU for a server.
diff --git a/changelog.d/8405.feature b/changelog.d/8405.feature
deleted file mode 100644
index f3c4a74bc7..0000000000
--- a/changelog.d/8405.feature
+++ /dev/null
@@ -1 +0,0 @@
-Consolidate the SSO error template across all configuration.
diff --git a/changelog.d/8406.feature b/changelog.d/8406.feature
deleted file mode 100644
index 1c6472ae7e..0000000000
--- a/changelog.d/8406.feature
+++ /dev/null
@@ -1 +0,0 @@
-Add prometheus metrics for replication requests.
diff --git a/changelog.d/8410.bugfix b/changelog.d/8410.bugfix
deleted file mode 100644
index 1323ddc525..0000000000
--- a/changelog.d/8410.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix a v1.20.0 regression in the `synapse_port_db` script regarding the `ui_auth_sessions_ips` table.
diff --git a/changelog.d/8413.feature b/changelog.d/8413.feature
deleted file mode 100644
index abe40a901c..0000000000
--- a/changelog.d/8413.feature
+++ /dev/null
@@ -1 +0,0 @@
-Support passing additional single sign-on parameters to the client.
diff --git a/changelog.d/8414.bugfix b/changelog.d/8414.bugfix
deleted file mode 100644
index 315876e892..0000000000
--- a/changelog.d/8414.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Remove unnecessary 3PID registration check when resetting password via an email address. Bug introduced in v0.34.0rc2.
\ No newline at end of file
diff --git a/changelog.d/8415.doc b/changelog.d/8415.doc
deleted file mode 100644
index 28b5798533..0000000000
--- a/changelog.d/8415.doc
+++ /dev/null
@@ -1 +0,0 @@
-Improve description of `server_name` config option in `homserver.yaml`.
\ No newline at end of file
diff --git a/changelog.d/8417.feature b/changelog.d/8417.feature
deleted file mode 100644
index 17549c3df3..0000000000
--- a/changelog.d/8417.feature
+++ /dev/null
@@ -1 +0,0 @@
-Add a config option to specify a whitelist of domains that a user can be redirected to after validating their email or phone number.
\ No newline at end of file
diff --git a/changelog.d/8419.feature b/changelog.d/8419.feature
deleted file mode 100644
index b363e929ea..0000000000
--- a/changelog.d/8419.feature
+++ /dev/null
@@ -1 +0,0 @@
-Add experimental support for sharding event persister.
diff --git a/changelog.d/8420.feature b/changelog.d/8420.feature
deleted file mode 100644
index 9d6849624d..0000000000
--- a/changelog.d/8420.feature
+++ /dev/null
@@ -1 +0,0 @@
-Add experimental reporting of metrics on expensive rooms for state-resolution.
diff --git a/changelog.d/8422.misc b/changelog.d/8422.misc
deleted file mode 100644
index 03fba120c6..0000000000
--- a/changelog.d/8422.misc
+++ /dev/null
@@ -1 +0,0 @@
-Typing fixes for `synapse.handlers.federation`.
diff --git a/changelog.d/8423.misc b/changelog.d/8423.misc
deleted file mode 100644
index 7260e3fa41..0000000000
--- a/changelog.d/8423.misc
+++ /dev/null
@@ -1 +0,0 @@
-Various refactors to simplify stream token handling.
diff --git a/changelog.d/8425.feature b/changelog.d/8425.feature
deleted file mode 100644
index b4ee5bb74b..0000000000
--- a/changelog.d/8425.feature
+++ /dev/null
@@ -1 +0,0 @@
-Add experimental prometheus metric to track numbers of "large" rooms for state resolutiom.
diff --git a/changelog.d/8426.removal b/changelog.d/8426.removal
deleted file mode 100644
index a56277fe7a..0000000000
--- a/changelog.d/8426.removal
+++ /dev/null
@@ -1 +0,0 @@
-Drop support for `prometheus_client` older than 0.4.0.
diff --git a/changelog.d/8427.misc b/changelog.d/8427.misc
deleted file mode 100644
index c9656b9112..0000000000
--- a/changelog.d/8427.misc
+++ /dev/null
@@ -1 +0,0 @@
-Make stream token serializing/deserializing async.
diff --git a/changelog.d/8430.feature b/changelog.d/8430.feature
deleted file mode 100644
index 1f31d42bc1..0000000000
--- a/changelog.d/8430.feature
+++ /dev/null
@@ -1 +0,0 @@
-Add prometheus metrics to track federation delays.
diff --git a/synapse/__init__.py b/synapse/__init__.py
index e40b582bd5..57f818125a 100644
--- a/synapse/__init__.py
+++ b/synapse/__init__.py
@@ -48,7 +48,7 @@ try:
 except ImportError:
     pass
 
-__version__ = "1.20.1"
+__version__ = "1.21.0"
 
 if bool(os.environ.get("SYNAPSE_TEST_PATCH_LOG_CONTEXTS", False)):
     # We import here so that we don't have to install a bunch of deps when

From c501c80e467a0c7a2429633a5160a580195a8826 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Thu, 1 Oct 2020 13:17:59 +0100
Subject: [PATCH 4/9] fix version number

we're not doing a final release yet!
---
 CHANGES.md          | 4 ++--
 synapse/__init__.py | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/CHANGES.md b/CHANGES.md
index 38906ade49..8368ac0b92 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,5 +1,5 @@
-Synapse 1.21.0 (2020-10-01)
-===========================
+Synapse 1.21.0rc1 (2020-10-01)
+==============================
 
 Features
 --------
diff --git a/synapse/__init__.py b/synapse/__init__.py
index 57f818125a..4706974508 100644
--- a/synapse/__init__.py
+++ b/synapse/__init__.py
@@ -48,7 +48,7 @@ try:
 except ImportError:
     pass
 
-__version__ = "1.21.0"
+__version__ = "1.21.0rc1"
 
 if bool(os.environ.get("SYNAPSE_TEST_PATCH_LOG_CONTEXTS", False)):
     # We import here so that we don't have to install a bunch of deps when

From 50e5174e8687ae3d368386dc020d869006cb6750 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Thu, 1 Oct 2020 13:27:01 +0100
Subject: [PATCH 5/9] changelog fixes

---
 CHANGES.md | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/CHANGES.md b/CHANGES.md
index 8368ac0b92..267909d3e9 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -29,7 +29,7 @@ Bugfixes
 - Fix messages not being sent over federation until an event is sent into the same room. ([\#8230](https://github.com/matrix-org/synapse/issues/8230), [\#8247](https://github.com/matrix-org/synapse/issues/8247), [\#8258](https://github.com/matrix-org/synapse/issues/8258), [\#8272](https://github.com/matrix-org/synapse/issues/8272), [\#8322](https://github.com/matrix-org/synapse/issues/8322))
 - Fix a longstanding bug where files that could not be thumbnailed would result in an Internal Server Error. ([\#8236](https://github.com/matrix-org/synapse/issues/8236))
 - Upgrade minimum version of `canonicaljson` to version 1.4.0, to fix an unicode encoding issue. ([\#8262](https://github.com/matrix-org/synapse/issues/8262))
-- Fix logstanding bug which could lead to incomplete database upgrades on SQLite. ([\#8265](https://github.com/matrix-org/synapse/issues/8265))
+- Fix longstanding bug which could lead to incomplete database upgrades on SQLite. ([\#8265](https://github.com/matrix-org/synapse/issues/8265))
 - Fix stack overflow when stderr is redirected to the logging system, and the logging system encounters an error. ([\#8268](https://github.com/matrix-org/synapse/issues/8268))
 - Fix a bug which cause the logging system to report errors, if `DEBUG` was enabled and no `context` filter was applied. ([\#8278](https://github.com/matrix-org/synapse/issues/8278))
 - Fix edge case where push could get delayed for a user until a later event was pushed. ([\#8287](https://github.com/matrix-org/synapse/issues/8287))
@@ -41,7 +41,6 @@ Bugfixes
 - Include `guest_access` in the fields that are checked for null bytes when updating `room_stats_state`. Broke in v1.7.2. ([\#8373](https://github.com/matrix-org/synapse/issues/8373))
 - Fix theoretical race condition where events are not sent down `/sync` if the synchrotron worker is restarted without restarting other workers. ([\#8374](https://github.com/matrix-org/synapse/issues/8374))
 - Fix a bug which could cause errors in rooms with malformed membership events, on servers using sqlite. ([\#8385](https://github.com/matrix-org/synapse/issues/8385))
-- Fix a bug introduced in v1.20.0 which caused the `synapse_port_db` script to fail. ([\#8386](https://github.com/matrix-org/synapse/issues/8386))
 - Fix "Re-starting finished log context" warning when receiving an event we already had over federation. ([\#8398](https://github.com/matrix-org/synapse/issues/8398))
 - Fix incorrect handling of timeouts on outgoing HTTP requests. ([\#8400](https://github.com/matrix-org/synapse/issues/8400))
 - Fix a regression in v1.20.0 in the `synapse_port_db` script regarding the `ui_auth_sessions_ips` table. ([\#8410](https://github.com/matrix-org/synapse/issues/8410))

From b1f4e6e4fc3d0cf5e10d6a79ef89abdcc9e63e8c Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com>
Date: Thu, 1 Oct 2020 13:34:24 +0100
Subject: [PATCH 6/9] fix a logging error in thumbnailer (#8435)

Introduced in #8236
---
 changelog.d/8435.bugfix                   | 1 +
 synapse/rest/media/v1/media_repository.py | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)
 create mode 100644 changelog.d/8435.bugfix

diff --git a/changelog.d/8435.bugfix b/changelog.d/8435.bugfix
new file mode 100644
index 0000000000..6f04871015
--- /dev/null
+++ b/changelog.d/8435.bugfix
@@ -0,0 +1 @@
+Fix a longstanding bug where files that could not be thumbnailed would result in an Internal Server Error.
diff --git a/synapse/rest/media/v1/media_repository.py b/synapse/rest/media/v1/media_repository.py
index ae6822d6e7..e1192b47cd 100644
--- a/synapse/rest/media/v1/media_repository.py
+++ b/synapse/rest/media/v1/media_repository.py
@@ -637,7 +637,7 @@ class MediaRepository:
             thumbnailer = Thumbnailer(input_path)
         except ThumbnailError as e:
             logger.warning(
-                "Unable to generate thumbnails for remote media %s from %s using a method of %s and type of %s: %s",
+                "Unable to generate thumbnails for remote media %s from %s of type %s: %s",
                 media_id,
                 server_name,
                 media_type,

From 2eb947e0eee7d66a77fc4a7e7af5234cacece3e6 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Thu, 1 Oct 2020 13:38:26 +0100
Subject: [PATCH 7/9] update changelog

---
 CHANGES.md              | 2 +-
 changelog.d/8435.bugfix | 1 -
 2 files changed, 1 insertion(+), 2 deletions(-)
 delete mode 100644 changelog.d/8435.bugfix

diff --git a/CHANGES.md b/CHANGES.md
index 267909d3e9..29711c60ce 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -27,7 +27,7 @@ Bugfixes
 - Fix inconsistent handling of non-existent push rules, and stop tracking the `enabled` state of removed push rules. ([\#7796](https://github.com/matrix-org/synapse/issues/7796))
 - Fix a longstanding bug when storing a media file with an empty `upload_name`. ([\#7905](https://github.com/matrix-org/synapse/issues/7905))
 - Fix messages not being sent over federation until an event is sent into the same room. ([\#8230](https://github.com/matrix-org/synapse/issues/8230), [\#8247](https://github.com/matrix-org/synapse/issues/8247), [\#8258](https://github.com/matrix-org/synapse/issues/8258), [\#8272](https://github.com/matrix-org/synapse/issues/8272), [\#8322](https://github.com/matrix-org/synapse/issues/8322))
-- Fix a longstanding bug where files that could not be thumbnailed would result in an Internal Server Error. ([\#8236](https://github.com/matrix-org/synapse/issues/8236))
+- Fix a longstanding bug where files that could not be thumbnailed would result in an Internal Server Error. ([\#8236](https://github.com/matrix-org/synapse/issues/8236), [\#8435](https://github.com/matrix-org/synapse/issues/8435))
 - Upgrade minimum version of `canonicaljson` to version 1.4.0, to fix an unicode encoding issue. ([\#8262](https://github.com/matrix-org/synapse/issues/8262))
 - Fix longstanding bug which could lead to incomplete database upgrades on SQLite. ([\#8265](https://github.com/matrix-org/synapse/issues/8265))
 - Fix stack overflow when stderr is redirected to the logging system, and the logging system encounters an error. ([\#8268](https://github.com/matrix-org/synapse/issues/8268))
diff --git a/changelog.d/8435.bugfix b/changelog.d/8435.bugfix
deleted file mode 100644
index 6f04871015..0000000000
--- a/changelog.d/8435.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix a longstanding bug where files that could not be thumbnailed would result in an Internal Server Error.

From 05ee048f2c9ce0bb8a7d2430b21ca3682ef5858b Mon Sep 17 00:00:00 2001
From: BBBSnowball <bbbsnowball@gmail.com>
Date: Thu, 1 Oct 2020 19:54:35 +0200
Subject: [PATCH 8/9] Add config option for always using "userinfo endpoint"
 for OIDC  (#7658)

This allows for connecting to certain IdPs, e.g. GitLab.
---
 changelog.d/7658.feature         |  1 +
 docs/openid.md                   | 41 +++++++++++++++++++++++++-------
 docs/sample_config.yaml          |  8 +++++++
 synapse/config/oidc_config.py    |  9 +++++++
 synapse/handlers/oidc_handler.py | 11 +++++----
 tests/handlers/test_oidc.py      | 10 ++++++--
 6 files changed, 65 insertions(+), 15 deletions(-)
 create mode 100644 changelog.d/7658.feature

diff --git a/changelog.d/7658.feature b/changelog.d/7658.feature
new file mode 100644
index 0000000000..fbf345988d
--- /dev/null
+++ b/changelog.d/7658.feature
@@ -0,0 +1 @@
+Add a configuration option for always using the "userinfo endpoint" for OpenID Connect. This fixes support for some identity providers, e.g. GitLab. Contributed by Benjamin Koch.
diff --git a/docs/openid.md b/docs/openid.md
index 70b37f858b..4873681999 100644
--- a/docs/openid.md
+++ b/docs/openid.md
@@ -238,13 +238,36 @@ Synapse config:
 
 ```yaml
 oidc_config:
-   enabled: true
-   issuer: "https://id.twitch.tv/oauth2/"
-   client_id: "your-client-id" # TO BE FILLED
-   client_secret: "your-client-secret" # TO BE FILLED
-   client_auth_method: "client_secret_post"
-   user_mapping_provider:
-     config:
-       localpart_template: '{{ user.preferred_username }}'
-       display_name_template: '{{ user.name }}'
+  enabled: true
+  issuer: "https://id.twitch.tv/oauth2/"
+  client_id: "your-client-id" # TO BE FILLED
+  client_secret: "your-client-secret" # TO BE FILLED
+  client_auth_method: "client_secret_post"
+  user_mapping_provider:
+    config:
+      localpart_template: "{{ user.preferred_username }}"
+      display_name_template: "{{ user.name }}"
+```
+
+### GitLab
+
+1. Create a [new application](https://gitlab.com/profile/applications).
+2. Add the `read_user` and `openid` scopes.
+3. Add this Callback URL: `[synapse public baseurl]/_synapse/oidc/callback`
+
+Synapse config:
+
+```yaml
+oidc_config:
+  enabled: true
+  issuer: "https://gitlab.com/"
+  client_id: "your-client-id" # TO BE FILLED
+  client_secret: "your-client-secret" # TO BE FILLED
+  client_auth_method: "client_secret_post"
+  scopes: ["openid", "read_user"]
+  user_profile_method: "userinfo_endpoint"
+  user_mapping_provider:
+    config:
+      localpart_template: '{{ user.nickname }}'
+      display_name_template: '{{ user.name }}'
 ```
diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml
index 8a3206e845..b2c1d7a737 100644
--- a/docs/sample_config.yaml
+++ b/docs/sample_config.yaml
@@ -1714,6 +1714,14 @@ oidc_config:
   #
   #skip_verification: true
 
+  # Whether to fetch the user profile from the userinfo endpoint. Valid
+  # values are: "auto" or "userinfo_endpoint".
+  #
+  # Defaults to "auto", which fetches the userinfo endpoint if "openid" is included
+  # in `scopes`. Uncomment the following to always fetch the userinfo endpoint.
+  #
+  #user_profile_method: "userinfo_endpoint"
+
   # Uncomment to allow a user logging in via OIDC to match a pre-existing account instead
   # of failing. This could be used if switching from password logins to OIDC. Defaults to false.
   #
diff --git a/synapse/config/oidc_config.py b/synapse/config/oidc_config.py
index f924116819..7597fbc864 100644
--- a/synapse/config/oidc_config.py
+++ b/synapse/config/oidc_config.py
@@ -56,6 +56,7 @@ class OIDCConfig(Config):
         self.oidc_userinfo_endpoint = oidc_config.get("userinfo_endpoint")
         self.oidc_jwks_uri = oidc_config.get("jwks_uri")
         self.oidc_skip_verification = oidc_config.get("skip_verification", False)
+        self.oidc_user_profile_method = oidc_config.get("user_profile_method", "auto")
         self.oidc_allow_existing_users = oidc_config.get("allow_existing_users", False)
 
         ump_config = oidc_config.get("user_mapping_provider", {})
@@ -159,6 +160,14 @@ class OIDCConfig(Config):
           #
           #skip_verification: true
 
+          # Whether to fetch the user profile from the userinfo endpoint. Valid
+          # values are: "auto" or "userinfo_endpoint".
+          #
+          # Defaults to "auto", which fetches the userinfo endpoint if "openid" is included
+          # in `scopes`. Uncomment the following to always fetch the userinfo endpoint.
+          #
+          #user_profile_method: "userinfo_endpoint"
+
           # Uncomment to allow a user logging in via OIDC to match a pre-existing account instead
           # of failing. This could be used if switching from password logins to OIDC. Defaults to false.
           #
diff --git a/synapse/handlers/oidc_handler.py b/synapse/handlers/oidc_handler.py
index 19cd652675..05ac86e697 100644
--- a/synapse/handlers/oidc_handler.py
+++ b/synapse/handlers/oidc_handler.py
@@ -96,6 +96,7 @@ class OidcHandler:
         self.hs = hs
         self._callback_url = hs.config.oidc_callback_url  # type: str
         self._scopes = hs.config.oidc_scopes  # type: List[str]
+        self._user_profile_method = hs.config.oidc_user_profile_method  # type: str
         self._client_auth = ClientAuth(
             hs.config.oidc_client_id,
             hs.config.oidc_client_secret,
@@ -196,11 +197,11 @@ class OidcHandler:
                     % (m["response_types_supported"],)
                 )
 
-        # If the openid scope was not requested, we need a userinfo endpoint to fetch user infos
+        # Ensure there's a userinfo endpoint to fetch from if it is required.
         if self._uses_userinfo:
             if m.get("userinfo_endpoint") is None:
                 raise ValueError(
-                    'provider has no "userinfo_endpoint", even though it is required because the "openid" scope is not requested'
+                    'provider has no "userinfo_endpoint", even though it is required'
                 )
         else:
             # If we're not using userinfo, we need a valid jwks to validate the ID token
@@ -220,8 +221,10 @@ class OidcHandler:
         ``access_token`` with the ``userinfo_endpoint``.
         """
 
-        # Maybe that should be user-configurable and not inferred?
-        return "openid" not in self._scopes
+        return (
+            "openid" not in self._scopes
+            or self._user_profile_method == "userinfo_endpoint"
+        )
 
     async def load_metadata(self) -> OpenIDProviderMetadata:
         """Load and validate the provider metadata.
diff --git a/tests/handlers/test_oidc.py b/tests/handlers/test_oidc.py
index d5087e58be..b6f436c016 100644
--- a/tests/handlers/test_oidc.py
+++ b/tests/handlers/test_oidc.py
@@ -286,9 +286,15 @@ class OidcHandlerTestCase(HomeserverTestCase):
                 h._validate_metadata,
             )
 
-        # Tests for configs that the userinfo endpoint
+        # Tests for configs that require the userinfo endpoint
         self.assertFalse(h._uses_userinfo)
-        h._scopes = []  # do not request the openid scope
+        self.assertEqual(h._user_profile_method, "auto")
+        h._user_profile_method = "userinfo_endpoint"
+        self.assertTrue(h._uses_userinfo)
+
+        # Revert the profile method and do not request the "openid" scope.
+        h._user_profile_method = "auto"
+        h._scopes = []
         self.assertTrue(h._uses_userinfo)
         self.assertRaisesRegex(ValueError, "userinfo_endpoint", h._validate_metadata)
 

From 6c5d5e507e629cf57ae8c1034879e8ffaef33e9f Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 2 Oct 2020 09:57:12 +0100
Subject: [PATCH 9/9] Add unit test for event persister sharding (#8433)

---
 changelog.d/8433.misc                         |   1 +
 mypy.ini                                      |   3 +
 stubs/txredisapi.pyi                          |  20 +-
 synapse/replication/tcp/handler.py            |   6 +-
 synapse/replication/tcp/redis.py              |  40 +++-
 tests/replication/_base.py                    | 224 ++++++++++++++++--
 .../test_sharded_event_persister.py           | 102 ++++++++
 tests/unittest.py                             |   2 +-
 8 files changed, 371 insertions(+), 27 deletions(-)
 create mode 100644 changelog.d/8433.misc
 create mode 100644 tests/replication/test_sharded_event_persister.py

diff --git a/changelog.d/8433.misc b/changelog.d/8433.misc
new file mode 100644
index 0000000000..05f8b5bbf4
--- /dev/null
+++ b/changelog.d/8433.misc
@@ -0,0 +1 @@
+Add unit test for event persister sharding.
diff --git a/mypy.ini b/mypy.ini
index c283f15b21..e84ad04e41 100644
--- a/mypy.ini
+++ b/mypy.ini
@@ -143,3 +143,6 @@ ignore_missing_imports = True
 
 [mypy-nacl.*]
 ignore_missing_imports = True
+
+[mypy-hiredis]
+ignore_missing_imports = True
diff --git a/stubs/txredisapi.pyi b/stubs/txredisapi.pyi
index c66413f003..522244bb57 100644
--- a/stubs/txredisapi.pyi
+++ b/stubs/txredisapi.pyi
@@ -16,7 +16,7 @@
 """Contains *incomplete* type hints for txredisapi.
 """
 
-from typing import List, Optional, Union
+from typing import List, Optional, Union, Type
 
 class RedisProtocol:
     def publish(self, channel: str, message: bytes): ...
@@ -42,3 +42,21 @@ def lazyConnection(
 
 class SubscriberFactory:
     def buildProtocol(self, addr): ...
+
+class ConnectionHandler: ...
+
+class RedisFactory:
+    continueTrying: bool
+    handler: RedisProtocol
+    def __init__(
+        self,
+        uuid: str,
+        dbid: Optional[int],
+        poolsize: int,
+        isLazy: bool = False,
+        handler: Type = ConnectionHandler,
+        charset: str = "utf-8",
+        password: Optional[str] = None,
+        replyTimeout: Optional[int] = None,
+        convertNumbers: Optional[int] = True,
+    ): ...
diff --git a/synapse/replication/tcp/handler.py b/synapse/replication/tcp/handler.py
index b323841f73..e92da7b263 100644
--- a/synapse/replication/tcp/handler.py
+++ b/synapse/replication/tcp/handler.py
@@ -251,10 +251,9 @@ class ReplicationCommandHandler:
         using TCP.
         """
         if hs.config.redis.redis_enabled:
-            import txredisapi
-
             from synapse.replication.tcp.redis import (
                 RedisDirectTcpReplicationClientFactory,
+                lazyConnection,
             )
 
             logger.info(
@@ -271,7 +270,8 @@ class ReplicationCommandHandler:
             # connection after SUBSCRIBE is called).
 
             # First create the connection for sending commands.
-            outbound_redis_connection = txredisapi.lazyConnection(
+            outbound_redis_connection = lazyConnection(
+                reactor=hs.get_reactor(),
                 host=hs.config.redis_host,
                 port=hs.config.redis_port,
                 password=hs.config.redis.redis_password,
diff --git a/synapse/replication/tcp/redis.py b/synapse/replication/tcp/redis.py
index f225e533de..de19705c1f 100644
--- a/synapse/replication/tcp/redis.py
+++ b/synapse/replication/tcp/redis.py
@@ -15,7 +15,7 @@
 
 import logging
 from inspect import isawaitable
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Optional
 
 import txredisapi
 
@@ -228,3 +228,41 @@ class RedisDirectTcpReplicationClientFactory(txredisapi.SubscriberFactory):
         p.password = self.password
 
         return p
+
+
+def lazyConnection(
+    reactor,
+    host: str = "localhost",
+    port: int = 6379,
+    dbid: Optional[int] = None,
+    reconnect: bool = True,
+    charset: str = "utf-8",
+    password: Optional[str] = None,
+    connectTimeout: Optional[int] = None,
+    replyTimeout: Optional[int] = None,
+    convertNumbers: bool = True,
+) -> txredisapi.RedisProtocol:
+    """Equivalent to `txredisapi.lazyConnection`, except allows specifying a
+    reactor.
+    """
+
+    isLazy = True
+    poolsize = 1
+
+    uuid = "%s:%d" % (host, port)
+    factory = txredisapi.RedisFactory(
+        uuid,
+        dbid,
+        poolsize,
+        isLazy,
+        txredisapi.ConnectionHandler,
+        charset,
+        password,
+        replyTimeout,
+        convertNumbers,
+    )
+    factory.continueTrying = reconnect
+    for x in range(poolsize):
+        reactor.connectTCP(host, port, factory, connectTimeout)
+
+    return factory.handler
diff --git a/tests/replication/_base.py b/tests/replication/_base.py
index ae60874ec3..81ea985b9f 100644
--- a/tests/replication/_base.py
+++ b/tests/replication/_base.py
@@ -12,13 +12,14 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 import logging
 from typing import Any, Callable, List, Optional, Tuple
 
 import attr
+import hiredis
 
 from twisted.internet.interfaces import IConsumer, IPullProducer, IReactorTime
+from twisted.internet.protocol import Protocol
 from twisted.internet.task import LoopingCall
 from twisted.web.http import HTTPChannel
 
@@ -27,7 +28,7 @@ from synapse.app.generic_worker import (
     GenericWorkerServer,
 )
 from synapse.http.server import JsonResource
-from synapse.http.site import SynapseRequest
+from synapse.http.site import SynapseRequest, SynapseSite
 from synapse.replication.http import ReplicationRestResource, streams
 from synapse.replication.tcp.handler import ReplicationCommandHandler
 from synapse.replication.tcp.protocol import ClientReplicationStreamProtocol
@@ -197,19 +198,37 @@ class BaseMultiWorkerStreamTestCase(unittest.HomeserverTestCase):
         self.server_factory = ReplicationStreamProtocolFactory(self.hs)
         self.streamer = self.hs.get_replication_streamer()
 
+        # Fake in memory Redis server that servers can connect to.
+        self._redis_server = FakeRedisPubSubServer()
+
         store = self.hs.get_datastore()
         self.database_pool = store.db_pool
 
         self.reactor.lookups["testserv"] = "1.2.3.4"
+        self.reactor.lookups["localhost"] = "127.0.0.1"
 
-        self._worker_hs_to_resource = {}
+        # A map from a HS instance to the associated HTTP Site to use for
+        # handling inbound HTTP requests to that instance.
+        self._hs_to_site = {self.hs: self.site}
+
+        if self.hs.config.redis.redis_enabled:
+            # Handle attempts to connect to fake redis server.
+            self.reactor.add_tcp_client_callback(
+                "localhost", 6379, self.connect_any_redis_attempts,
+            )
+
+            self.hs.get_tcp_replication().start_replication(self.hs)
 
         # When we see a connection attempt to the master replication listener we
         # automatically set up the connection. This is so that tests don't
         # manually have to go and explicitly set it up each time (plus sometimes
         # it is impossible to write the handling explicitly in the tests).
+        #
+        # Register the master replication listener:
         self.reactor.add_tcp_client_callback(
-            "1.2.3.4", 8765, self._handle_http_replication_attempt
+            "1.2.3.4",
+            8765,
+            lambda: self._handle_http_replication_attempt(self.hs, 8765),
         )
 
     def create_test_json_resource(self):
@@ -253,28 +272,63 @@ class BaseMultiWorkerStreamTestCase(unittest.HomeserverTestCase):
             **kwargs
         )
 
+        # If the instance is in the `instance_map` config then workers may try
+        # and send HTTP requests to it, so we register it with
+        # `_handle_http_replication_attempt` like we do with the master HS.
+        instance_name = worker_hs.get_instance_name()
+        instance_loc = worker_hs.config.worker.instance_map.get(instance_name)
+        if instance_loc:
+            # Ensure the host is one that has a fake DNS entry.
+            if instance_loc.host not in self.reactor.lookups:
+                raise Exception(
+                    "Host does not have an IP for instance_map[%r].host = %r"
+                    % (instance_name, instance_loc.host,)
+                )
+
+            self.reactor.add_tcp_client_callback(
+                self.reactor.lookups[instance_loc.host],
+                instance_loc.port,
+                lambda: self._handle_http_replication_attempt(
+                    worker_hs, instance_loc.port
+                ),
+            )
+
         store = worker_hs.get_datastore()
         store.db_pool._db_pool = self.database_pool._db_pool
 
-        repl_handler = ReplicationCommandHandler(worker_hs)
-        client = ClientReplicationStreamProtocol(
-            worker_hs, "client", "test", self.clock, repl_handler,
-        )
-        server = self.server_factory.buildProtocol(None)
+        # Set up TCP replication between master and the new worker if we don't
+        # have Redis support enabled.
+        if not worker_hs.config.redis_enabled:
+            repl_handler = ReplicationCommandHandler(worker_hs)
+            client = ClientReplicationStreamProtocol(
+                worker_hs, "client", "test", self.clock, repl_handler,
+            )
+            server = self.server_factory.buildProtocol(None)
 
-        client_transport = FakeTransport(server, self.reactor)
-        client.makeConnection(client_transport)
+            client_transport = FakeTransport(server, self.reactor)
+            client.makeConnection(client_transport)
 
-        server_transport = FakeTransport(client, self.reactor)
-        server.makeConnection(server_transport)
+            server_transport = FakeTransport(client, self.reactor)
+            server.makeConnection(server_transport)
 
         # Set up a resource for the worker
-        resource = ReplicationRestResource(self.hs)
+        resource = ReplicationRestResource(worker_hs)
 
         for servlet in self.servlets:
             servlet(worker_hs, resource)
 
-        self._worker_hs_to_resource[worker_hs] = resource
+        self._hs_to_site[worker_hs] = SynapseSite(
+            logger_name="synapse.access.http.fake",
+            site_tag="{}-{}".format(
+                worker_hs.config.server.server_name, worker_hs.get_instance_name()
+            ),
+            config=worker_hs.config.server.listeners[0],
+            resource=resource,
+            server_version_string="1",
+        )
+
+        if worker_hs.config.redis.redis_enabled:
+            worker_hs.get_tcp_replication().start_replication(worker_hs)
 
         return worker_hs
 
@@ -285,7 +339,7 @@ class BaseMultiWorkerStreamTestCase(unittest.HomeserverTestCase):
         return config
 
     def render_on_worker(self, worker_hs: HomeServer, request: SynapseRequest):
-        render(request, self._worker_hs_to_resource[worker_hs], self.reactor)
+        render(request, self._hs_to_site[worker_hs].resource, self.reactor)
 
     def replicate(self):
         """Tell the master side of replication that something has happened, and then
@@ -294,9 +348,9 @@ class BaseMultiWorkerStreamTestCase(unittest.HomeserverTestCase):
         self.streamer.on_notifier_poke()
         self.pump()
 
-    def _handle_http_replication_attempt(self):
-        """Handles a connection attempt to the master replication HTTP
-        listener.
+    def _handle_http_replication_attempt(self, hs, repl_port):
+        """Handles a connection attempt to the given HS replication HTTP
+        listener on the given port.
         """
 
         # We should have at least one outbound connection attempt, where the
@@ -305,7 +359,7 @@ class BaseMultiWorkerStreamTestCase(unittest.HomeserverTestCase):
         self.assertGreaterEqual(len(clients), 1)
         (host, port, client_factory, _timeout, _bindAddress) = clients.pop()
         self.assertEqual(host, "1.2.3.4")
-        self.assertEqual(port, 8765)
+        self.assertEqual(port, repl_port)
 
         # Set up client side protocol
         client_protocol = client_factory.buildProtocol(None)
@@ -315,7 +369,7 @@ class BaseMultiWorkerStreamTestCase(unittest.HomeserverTestCase):
         # Set up the server side protocol
         channel = _PushHTTPChannel(self.reactor)
         channel.requestFactory = request_factory
-        channel.site = self.site
+        channel.site = self._hs_to_site[hs]
 
         # Connect client to server and vice versa.
         client_to_server_transport = FakeTransport(
@@ -333,6 +387,32 @@ class BaseMultiWorkerStreamTestCase(unittest.HomeserverTestCase):
         # inside `connecTCP` before the connection has been passed back to the
         # code that requested the TCP connection.
 
+    def connect_any_redis_attempts(self):
+        """If redis is enabled we need to deal with workers connecting to a
+        redis server. We don't want to use a real Redis server so we use a
+        fake one.
+        """
+        clients = self.reactor.tcpClients
+        self.assertEqual(len(clients), 1)
+        (host, port, client_factory, _timeout, _bindAddress) = clients.pop(0)
+        self.assertEqual(host, "localhost")
+        self.assertEqual(port, 6379)
+
+        client_protocol = client_factory.buildProtocol(None)
+        server_protocol = self._redis_server.buildProtocol(None)
+
+        client_to_server_transport = FakeTransport(
+            server_protocol, self.reactor, client_protocol
+        )
+        client_protocol.makeConnection(client_to_server_transport)
+
+        server_to_client_transport = FakeTransport(
+            client_protocol, self.reactor, server_protocol
+        )
+        server_protocol.makeConnection(server_to_client_transport)
+
+        return client_to_server_transport, server_to_client_transport
+
 
 class TestReplicationDataHandler(GenericWorkerReplicationHandler):
     """Drop-in for ReplicationDataHandler which just collects RDATA rows"""
@@ -467,3 +547,105 @@ class _PullToPushProducer:
                 pass
 
             self.stopProducing()
+
+
+class FakeRedisPubSubServer:
+    """A fake Redis server for pub/sub.
+    """
+
+    def __init__(self):
+        self._subscribers = set()
+
+    def add_subscriber(self, conn):
+        """A connection has called SUBSCRIBE
+        """
+        self._subscribers.add(conn)
+
+    def remove_subscriber(self, conn):
+        """A connection has called UNSUBSCRIBE
+        """
+        self._subscribers.discard(conn)
+
+    def publish(self, conn, channel, msg) -> int:
+        """A connection want to publish a message to subscribers.
+        """
+        for sub in self._subscribers:
+            sub.send(["message", channel, msg])
+
+        return len(self._subscribers)
+
+    def buildProtocol(self, addr):
+        return FakeRedisPubSubProtocol(self)
+
+
+class FakeRedisPubSubProtocol(Protocol):
+    """A connection from a client talking to the fake Redis server.
+    """
+
+    def __init__(self, server: FakeRedisPubSubServer):
+        self._server = server
+        self._reader = hiredis.Reader()
+
+    def dataReceived(self, data):
+        self._reader.feed(data)
+
+        # We might get multiple messages in one packet.
+        while True:
+            msg = self._reader.gets()
+
+            if msg is False:
+                # No more messages.
+                return
+
+            if not isinstance(msg, list):
+                # Inbound commands should always be a list
+                raise Exception("Expected redis list")
+
+            self.handle_command(msg[0], *msg[1:])
+
+    def handle_command(self, command, *args):
+        """Received a Redis command from the client.
+        """
+
+        # We currently only support pub/sub.
+        if command == b"PUBLISH":
+            channel, message = args
+            num_subscribers = self._server.publish(self, channel, message)
+            self.send(num_subscribers)
+        elif command == b"SUBSCRIBE":
+            (channel,) = args
+            self._server.add_subscriber(self)
+            self.send(["subscribe", channel, 1])
+        else:
+            raise Exception("Unknown command")
+
+    def send(self, msg):
+        """Send a message back to the client.
+        """
+        raw = self.encode(msg).encode("utf-8")
+
+        self.transport.write(raw)
+        self.transport.flush()
+
+    def encode(self, obj):
+        """Encode an object to its Redis format.
+
+        Supports: strings/bytes, integers and list/tuples.
+        """
+
+        if isinstance(obj, bytes):
+            # We assume bytes are just unicode strings.
+            obj = obj.decode("utf-8")
+
+        if isinstance(obj, str):
+            return "${len}\r\n{str}\r\n".format(len=len(obj), str=obj)
+        if isinstance(obj, int):
+            return ":{val}\r\n".format(val=obj)
+        if isinstance(obj, (list, tuple)):
+            items = "".join(self.encode(a) for a in obj)
+            return "*{len}\r\n{items}".format(len=len(obj), items=items)
+
+        raise Exception("Unrecognized type for encoding redis: %r: %r", type(obj), obj)
+
+    def connectionLost(self, reason):
+        self._server.remove_subscriber(self)
diff --git a/tests/replication/test_sharded_event_persister.py b/tests/replication/test_sharded_event_persister.py
new file mode 100644
index 0000000000..6068d14905
--- /dev/null
+++ b/tests/replication/test_sharded_event_persister.py
@@ -0,0 +1,102 @@
+# -*- coding: utf-8 -*-
+# Copyright 2020 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import logging
+
+from synapse.rest import admin
+from synapse.rest.client.v1 import login, room
+
+from tests.replication._base import BaseMultiWorkerStreamTestCase
+from tests.utils import USE_POSTGRES_FOR_TESTS
+
+logger = logging.getLogger(__name__)
+
+
+class EventPersisterShardTestCase(BaseMultiWorkerStreamTestCase):
+    """Checks event persisting sharding works
+    """
+
+    # Event persister sharding requires postgres (due to needing
+    # `MutliWriterIdGenerator`).
+    if not USE_POSTGRES_FOR_TESTS:
+        skip = "Requires Postgres"
+
+    servlets = [
+        admin.register_servlets_for_client_rest_resource,
+        room.register_servlets,
+        login.register_servlets,
+    ]
+
+    def prepare(self, reactor, clock, hs):
+        # Register a user who sends a message that we'll get notified about
+        self.other_user_id = self.register_user("otheruser", "pass")
+        self.other_access_token = self.login("otheruser", "pass")
+
+    def default_config(self):
+        conf = super().default_config()
+        conf["redis"] = {"enabled": "true"}
+        conf["stream_writers"] = {"events": ["worker1", "worker2"]}
+        conf["instance_map"] = {
+            "worker1": {"host": "testserv", "port": 1001},
+            "worker2": {"host": "testserv", "port": 1002},
+        }
+        return conf
+
+    def test_basic(self):
+        """Simple test to ensure that multiple rooms can be created and joined,
+        and that different rooms get handled by different instances.
+        """
+
+        self.make_worker_hs(
+            "synapse.app.generic_worker", {"worker_name": "worker1"},
+        )
+
+        self.make_worker_hs(
+            "synapse.app.generic_worker", {"worker_name": "worker2"},
+        )
+
+        persisted_on_1 = False
+        persisted_on_2 = False
+
+        store = self.hs.get_datastore()
+
+        user_id = self.register_user("user", "pass")
+        access_token = self.login("user", "pass")
+
+        # Keep making new rooms until we see rooms being persisted on both
+        # workers.
+        for _ in range(10):
+            # Create a room
+            room = self.helper.create_room_as(user_id, tok=access_token)
+
+            # The other user joins
+            self.helper.join(
+                room=room, user=self.other_user_id, tok=self.other_access_token
+            )
+
+            # The other user sends some messages
+            rseponse = self.helper.send(room, body="Hi!", tok=self.other_access_token)
+            event_id = rseponse["event_id"]
+
+            # The event position includes which instance persisted the event.
+            pos = self.get_success(store.get_position_for_event(event_id))
+
+            persisted_on_1 |= pos.instance_name == "worker1"
+            persisted_on_2 |= pos.instance_name == "worker2"
+
+            if persisted_on_1 and persisted_on_2:
+                break
+
+        self.assertTrue(persisted_on_1)
+        self.assertTrue(persisted_on_2)
diff --git a/tests/unittest.py b/tests/unittest.py
index e654c0442d..82ede9de34 100644
--- a/tests/unittest.py
+++ b/tests/unittest.py
@@ -241,7 +241,7 @@ class HomeserverTestCase(TestCase):
         # create a site to wrap the resource.
         self.site = SynapseSite(
             logger_name="synapse.access.http.fake",
-            site_tag="test",
+            site_tag=self.hs.config.server.server_name,
             config=self.hs.config.server.listeners[0],
             resource=self.resource,
             server_version_string="1",