OpenCloud
/
MatrixSynapse
mirror of https://github.com/matrix-org/synapse
1
0
Fork 0
Code Issues Releases Wiki Activity

Add linearizer to protect stream catchup

pull/7185/head
Erik Johnston 2020-03-31 13:30:57 +01:00
parent 90bd1708b5
commit 6ac1ecad95
2 changed files with 32 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
synapse/replication/tcp/handler.py
View File

@ -35,6 +35,7 @@ from synapse.replication.tcp.commands import (
UserSyncCommand, UserSyncCommand,
) )
from synapse.replication.tcp.streams import STREAMS_MAP, Stream from synapse.replication.tcp.streams import STREAMS_MAP, Stream
from synapse.util.async_helpers import Linearizer
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -60,6 +61,8 @@ class ReplicationCommandHandler:
stream.NAME: stream(hs) for stream in STREAMS_MAP.values() stream.NAME: stream(hs) for stream in STREAMS_MAP.values()
} # type: Dict[str, Stream] } # type: Dict[str, Stream]
self._position_linearizer = Linearizer("replication_position")
# Map of stream to batched updates. See RdataCommand for info on how # Map of stream to batched updates. See RdataCommand for info on how
# batching works. # batching works.
self.pending_batches = {} # type: Dict[str, List[Any]] self.pending_batches = {} # type: Dict[str, List[Any]]
@ -122,13 +125,17 @@ class ReplicationCommandHandler:
# to stop RDATA being handled at the same time. # to stop RDATA being handled at the same time.
self.streams_connecting.add(cmd.stream_name) self.streams_connecting.add(cmd.stream_name)
# We protect catching up with a linearizer in case the replicaiton
# connection reconnects under us.
with await self._position_linearizer.queue(cmd.stream_name):
# Find where we previously streamed up to. # Find where we previously streamed up to.
current_token = self.replication_data_handler.get_streams_to_replicate().get( current_token = self.replication_data_handler.get_streams_to_replicate().get(
cmd.stream_name cmd.stream_name
) )
if current_token is None: if current_token is None:
logger.warning( logger.warning(
"Got POSITION for stream we're not subscribed to: %s", cmd.stream_name "Got POSITION for stream we're not subscribed to: %s",
cmd.stream_name,
) )
return return

6
tests/replication/slave/storage/_base.py
View File

@ -15,9 +15,11 @@
from mock import Mock, NonCallableMock from mock import Mock, NonCallableMock
from synapse.replication.tcp.client import ReplicationClientFactory from synapse.replication.tcp.client import (
ReplicationClientFactory,
ReplicationDataHandler,
)
from synapse.replication.tcp.handler import ReplicationCommandHandler from synapse.replication.tcp.handler import ReplicationCommandHandler
from synapse.replication.tcp.client import ReplicationDataHandler
from synapse.replication.tcp.resource import ReplicationStreamProtocolFactory from synapse.replication.tcp.resource import ReplicationStreamProtocolFactory
from synapse.storage.database import make_conn from synapse.storage.database import make_conn