Add linearizer to protect stream catchup

synapse/replication/tcp/handler.py

@@ -35,6 +35,7 @@ from synapse.replication.tcp.commands import (
     UserSyncCommand,
 )
 from synapse.replication.tcp.streams import STREAMS_MAP, Stream
+from synapse.util.async_helpers import Linearizer
 
 logger = logging.getLogger(__name__)
 
@@ -60,6 +61,8 @@ class ReplicationCommandHandler:
             stream.NAME: stream(hs) for stream in STREAMS_MAP.values()
         }  # type: Dict[str, Stream]
 
+        self._position_linearizer = Linearizer("replication_position")
+
         # Map of stream to batched updates. See RdataCommand for info on how
         # batching works.
         self.pending_batches = {}  # type: Dict[str, List[Any]]
@@ -122,13 +125,17 @@ class ReplicationCommandHandler:
         # to stop RDATA being handled at the same time.
         self.streams_connecting.add(cmd.stream_name)
 
+        # We protect catching up with a linearizer in case the replicaiton
+        # connection reconnects under us.
+        with await self._position_linearizer.queue(cmd.stream_name):
             # Find where we previously streamed up to.
             current_token = self.replication_data_handler.get_streams_to_replicate().get(
                 cmd.stream_name
             )
             if current_token is None:
                 logger.warning(
-                "Got POSITION for stream we're not subscribed to: %s", cmd.stream_name
+                    "Got POSITION for stream we're not subscribed to: %s",
+                    cmd.stream_name,
                 )
                 return
 

tests/replication/slave/storage/_base.py

@@ -15,9 +15,11 @@
 
 from mock import Mock, NonCallableMock
 
-from synapse.replication.tcp.client import ReplicationClientFactory
+from synapse.replication.tcp.client import (
+    ReplicationClientFactory,
+    ReplicationDataHandler,
+)
 from synapse.replication.tcp.handler import ReplicationCommandHandler
-from synapse.replication.tcp.client import ReplicationDataHandler
 from synapse.replication.tcp.resource import ReplicationStreamProtocolFactory
 from synapse.storage.database import make_conn