Revert "Remove slaved id tracker (#14376)" (#14463)

This reverts commit 36097e88c4.
pull/14469/head
Erik Johnston 2022-11-16 13:50:07 +00:00 committed by GitHub
parent 945a0928c7
commit d63814fd73
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 176 additions and 74 deletions

View File

@ -1 +0,0 @@
Remove old stream ID tracking code. Contributed by Nick @Beeper (@fizzadar).

View File

@ -0,0 +1,13 @@
# Copyright 2016 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

View File

@ -0,0 +1,13 @@
# Copyright 2016 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

View File

@ -0,0 +1,50 @@
# Copyright 2016 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from typing import List, Optional, Tuple
from synapse.storage.database import LoggingDatabaseConnection
from synapse.storage.util.id_generators import AbstractStreamIdTracker, _load_current_id
class SlavedIdTracker(AbstractStreamIdTracker):
"""Tracks the "current" stream ID of a stream with a single writer.
See `AbstractStreamIdTracker` for more details.
Note that this class does not work correctly when there are multiple
writers.
"""
def __init__(
self,
db_conn: LoggingDatabaseConnection,
table: str,
column: str,
extra_tables: Optional[List[Tuple[str, str]]] = None,
step: int = 1,
):
self.step = step
self._current = _load_current_id(db_conn, table, column, step)
if extra_tables:
for table, column in extra_tables:
self.advance(None, _load_current_id(db_conn, table, column))
def advance(self, instance_name: Optional[str], new_id: int) -> None:
self._current = (max if self.step > 0 else min)(self._current, new_id)
def get_current_token(self) -> int:
return self._current
def get_current_token_for_writer(self, instance_name: str) -> int:
return self.get_current_token()

View File

@ -27,6 +27,7 @@ from typing import (
) )
from synapse.api.constants import AccountDataTypes from synapse.api.constants import AccountDataTypes
from synapse.replication.slave.storage._slaved_id_tracker import SlavedIdTracker
from synapse.replication.tcp.streams import AccountDataStream, TagAccountDataStream from synapse.replication.tcp.streams import AccountDataStream, TagAccountDataStream
from synapse.storage._base import db_to_json from synapse.storage._base import db_to_json
from synapse.storage.database import ( from synapse.storage.database import (
@ -67,11 +68,12 @@ class AccountDataWorkerStore(PushRulesWorkerStore, CacheInvalidationWorkerStore)
# to write account data. A value of `True` implies that `_account_data_id_gen` # to write account data. A value of `True` implies that `_account_data_id_gen`
# is an `AbstractStreamIdGenerator` and not just a tracker. # is an `AbstractStreamIdGenerator` and not just a tracker.
self._account_data_id_gen: AbstractStreamIdTracker self._account_data_id_gen: AbstractStreamIdTracker
self._can_write_to_account_data = (
self._instance_name in hs.config.worker.writers.account_data
)
if isinstance(database.engine, PostgresEngine): if isinstance(database.engine, PostgresEngine):
self._can_write_to_account_data = (
self._instance_name in hs.config.worker.writers.account_data
)
self._account_data_id_gen = MultiWriterIdGenerator( self._account_data_id_gen = MultiWriterIdGenerator(
db_conn=db_conn, db_conn=db_conn,
db=database, db=database,
@ -93,13 +95,21 @@ class AccountDataWorkerStore(PushRulesWorkerStore, CacheInvalidationWorkerStore)
# `StreamIdGenerator`, otherwise we use `SlavedIdTracker` which gets # `StreamIdGenerator`, otherwise we use `SlavedIdTracker` which gets
# updated over replication. (Multiple writers are not supported for # updated over replication. (Multiple writers are not supported for
# SQLite). # SQLite).
self._account_data_id_gen = StreamIdGenerator( if self._instance_name in hs.config.worker.writers.account_data:
db_conn, self._can_write_to_account_data = True
"room_account_data", self._account_data_id_gen = StreamIdGenerator(
"stream_id", db_conn,
extra_tables=[("room_tags_revisions", "stream_id")], "room_account_data",
is_writer=self._instance_name in hs.config.worker.writers.account_data, "stream_id",
) extra_tables=[("room_tags_revisions", "stream_id")],
)
else:
self._account_data_id_gen = SlavedIdTracker(
db_conn,
"room_account_data",
"stream_id",
extra_tables=[("room_tags_revisions", "stream_id")],
)
account_max = self.get_max_account_data_stream_id() account_max = self.get_max_account_data_stream_id()
self._account_data_stream_cache = StreamChangeCache( self._account_data_stream_cache = StreamChangeCache(

View File

@ -38,6 +38,7 @@ from synapse.logging.opentracing import (
whitelisted_homeserver, whitelisted_homeserver,
) )
from synapse.metrics.background_process_metrics import wrap_as_background_process from synapse.metrics.background_process_metrics import wrap_as_background_process
from synapse.replication.slave.storage._slaved_id_tracker import SlavedIdTracker
from synapse.replication.tcp.streams._base import DeviceListsStream, UserSignatureStream from synapse.replication.tcp.streams._base import DeviceListsStream, UserSignatureStream
from synapse.storage._base import SQLBaseStore, db_to_json, make_in_list_sql_clause from synapse.storage._base import SQLBaseStore, db_to_json, make_in_list_sql_clause
from synapse.storage.database import ( from synapse.storage.database import (
@ -85,19 +86,28 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore):
): ):
super().__init__(database, db_conn, hs) super().__init__(database, db_conn, hs)
# In the worker store this is an ID tracker which we overwrite in the non-worker if hs.config.worker.worker_app is None:
# class below that is used on the main process. self._device_list_id_gen: AbstractStreamIdTracker = StreamIdGenerator(
self._device_list_id_gen: AbstractStreamIdTracker = StreamIdGenerator( db_conn,
db_conn, "device_lists_stream",
"device_lists_stream", "stream_id",
"stream_id", extra_tables=[
extra_tables=[ ("user_signature_stream", "stream_id"),
("user_signature_stream", "stream_id"), ("device_lists_outbound_pokes", "stream_id"),
("device_lists_outbound_pokes", "stream_id"), ("device_lists_changes_in_room", "stream_id"),
("device_lists_changes_in_room", "stream_id"), ],
], )
is_writer=hs.config.worker.worker_app is None, else:
) self._device_list_id_gen = SlavedIdTracker(
db_conn,
"device_lists_stream",
"stream_id",
extra_tables=[
("user_signature_stream", "stream_id"),
("device_lists_outbound_pokes", "stream_id"),
("device_lists_changes_in_room", "stream_id"),
],
)
# Type-ignore: _device_list_id_gen is mixed in from either DataStore (as a # Type-ignore: _device_list_id_gen is mixed in from either DataStore (as a
# StreamIdGenerator) or SlavedDataStore (as a SlavedIdTracker). # StreamIdGenerator) or SlavedDataStore (as a SlavedIdTracker).

View File

@ -59,6 +59,7 @@ from synapse.metrics.background_process_metrics import (
run_as_background_process, run_as_background_process,
wrap_as_background_process, wrap_as_background_process,
) )
from synapse.replication.slave.storage._slaved_id_tracker import SlavedIdTracker
from synapse.replication.tcp.streams import BackfillStream from synapse.replication.tcp.streams import BackfillStream
from synapse.replication.tcp.streams.events import EventsStream from synapse.replication.tcp.streams.events import EventsStream
from synapse.storage._base import SQLBaseStore, db_to_json, make_in_list_sql_clause from synapse.storage._base import SQLBaseStore, db_to_json, make_in_list_sql_clause
@ -212,20 +213,26 @@ class EventsWorkerStore(SQLBaseStore):
# `StreamIdGenerator`, otherwise we use `SlavedIdTracker` which gets # `StreamIdGenerator`, otherwise we use `SlavedIdTracker` which gets
# updated over replication. (Multiple writers are not supported for # updated over replication. (Multiple writers are not supported for
# SQLite). # SQLite).
self._stream_id_gen = StreamIdGenerator( if hs.get_instance_name() in hs.config.worker.writers.events:
db_conn, self._stream_id_gen = StreamIdGenerator(
"events", db_conn,
"stream_ordering", "events",
is_writer=hs.get_instance_name() in hs.config.worker.writers.events, "stream_ordering",
) )
self._backfill_id_gen = StreamIdGenerator( self._backfill_id_gen = StreamIdGenerator(
db_conn, db_conn,
"events", "events",
"stream_ordering", "stream_ordering",
step=-1, step=-1,
extra_tables=[("ex_outlier_stream", "event_stream_ordering")], extra_tables=[("ex_outlier_stream", "event_stream_ordering")],
is_writer=hs.get_instance_name() in hs.config.worker.writers.events, )
) else:
self._stream_id_gen = SlavedIdTracker(
db_conn, "events", "stream_ordering"
)
self._backfill_id_gen = SlavedIdTracker(
db_conn, "events", "stream_ordering", step=-1
)
events_max = self._stream_id_gen.get_current_token() events_max = self._stream_id_gen.get_current_token()
curr_state_delta_prefill, min_curr_state_delta_id = self.db_pool.get_cache_dict( curr_state_delta_prefill, min_curr_state_delta_id = self.db_pool.get_cache_dict(

View File

@ -30,6 +30,7 @@ from typing import (
from synapse.api.errors import StoreError from synapse.api.errors import StoreError
from synapse.config.homeserver import ExperimentalConfig from synapse.config.homeserver import ExperimentalConfig
from synapse.replication.slave.storage._slaved_id_tracker import SlavedIdTracker
from synapse.replication.tcp.streams import PushRulesStream from synapse.replication.tcp.streams import PushRulesStream
from synapse.storage._base import SQLBaseStore from synapse.storage._base import SQLBaseStore
from synapse.storage.database import ( from synapse.storage.database import (
@ -110,14 +111,14 @@ class PushRulesWorkerStore(
): ):
super().__init__(database, db_conn, hs) super().__init__(database, db_conn, hs)
# In the worker store this is an ID tracker which we overwrite in the non-worker if hs.config.worker.worker_app is None:
# class below that is used on the main process. self._push_rules_stream_id_gen: AbstractStreamIdTracker = StreamIdGenerator(
self._push_rules_stream_id_gen: AbstractStreamIdTracker = StreamIdGenerator( db_conn, "push_rules_stream", "stream_id"
db_conn, )
"push_rules_stream", else:
"stream_id", self._push_rules_stream_id_gen = SlavedIdTracker(
is_writer=hs.config.worker.worker_app is None, db_conn, "push_rules_stream", "stream_id"
) )
push_rules_prefill, push_rules_id = self.db_pool.get_cache_dict( push_rules_prefill, push_rules_id = self.db_pool.get_cache_dict(
db_conn, db_conn,

View File

@ -27,6 +27,7 @@ from typing import (
) )
from synapse.push import PusherConfig, ThrottleParams from synapse.push import PusherConfig, ThrottleParams
from synapse.replication.slave.storage._slaved_id_tracker import SlavedIdTracker
from synapse.replication.tcp.streams import PushersStream from synapse.replication.tcp.streams import PushersStream
from synapse.storage._base import SQLBaseStore, db_to_json from synapse.storage._base import SQLBaseStore, db_to_json
from synapse.storage.database import ( from synapse.storage.database import (
@ -58,15 +59,20 @@ class PusherWorkerStore(SQLBaseStore):
): ):
super().__init__(database, db_conn, hs) super().__init__(database, db_conn, hs)
# In the worker store this is an ID tracker which we overwrite in the non-worker if hs.config.worker.worker_app is None:
# class below that is used on the main process. self._pushers_id_gen: AbstractStreamIdTracker = StreamIdGenerator(
self._pushers_id_gen: AbstractStreamIdTracker = StreamIdGenerator( db_conn,
db_conn, "pushers",
"pushers", "id",
"id", extra_tables=[("deleted_pushers", "stream_id")],
extra_tables=[("deleted_pushers", "stream_id")], )
is_writer=hs.config.worker.worker_app is None, else:
) self._pushers_id_gen = SlavedIdTracker(
db_conn,
"pushers",
"id",
extra_tables=[("deleted_pushers", "stream_id")],
)
self.db_pool.updates.register_background_update_handler( self.db_pool.updates.register_background_update_handler(
"remove_deactivated_pushers", "remove_deactivated_pushers",

View File

@ -27,6 +27,7 @@ from typing import (
) )
from synapse.api.constants import EduTypes from synapse.api.constants import EduTypes
from synapse.replication.slave.storage._slaved_id_tracker import SlavedIdTracker
from synapse.replication.tcp.streams import ReceiptsStream from synapse.replication.tcp.streams import ReceiptsStream
from synapse.storage._base import SQLBaseStore, db_to_json, make_in_list_sql_clause from synapse.storage._base import SQLBaseStore, db_to_json, make_in_list_sql_clause
from synapse.storage.database import ( from synapse.storage.database import (
@ -60,9 +61,6 @@ class ReceiptsWorkerStore(SQLBaseStore):
hs: "HomeServer", hs: "HomeServer",
): ):
self._instance_name = hs.get_instance_name() self._instance_name = hs.get_instance_name()
# In the worker store this is an ID tracker which we overwrite in the non-worker
# class below that is used on the main process.
self._receipts_id_gen: AbstractStreamIdTracker self._receipts_id_gen: AbstractStreamIdTracker
if isinstance(database.engine, PostgresEngine): if isinstance(database.engine, PostgresEngine):
@ -89,12 +87,14 @@ class ReceiptsWorkerStore(SQLBaseStore):
# `StreamIdGenerator`, otherwise we use `SlavedIdTracker` which gets # `StreamIdGenerator`, otherwise we use `SlavedIdTracker` which gets
# updated over replication. (Multiple writers are not supported for # updated over replication. (Multiple writers are not supported for
# SQLite). # SQLite).
self._receipts_id_gen = StreamIdGenerator( if hs.get_instance_name() in hs.config.worker.writers.receipts:
db_conn, self._receipts_id_gen = StreamIdGenerator(
"receipts_linearized", db_conn, "receipts_linearized", "stream_id"
"stream_id", )
is_writer=hs.get_instance_name() in hs.config.worker.writers.receipts, else:
) self._receipts_id_gen = SlavedIdTracker(
db_conn, "receipts_linearized", "stream_id"
)
super().__init__(database, db_conn, hs) super().__init__(database, db_conn, hs)

View File

@ -186,13 +186,11 @@ class StreamIdGenerator(AbstractStreamIdGenerator):
column: str, column: str,
extra_tables: Iterable[Tuple[str, str]] = (), extra_tables: Iterable[Tuple[str, str]] = (),
step: int = 1, step: int = 1,
is_writer: bool = True,
) -> None: ) -> None:
assert step != 0 assert step != 0
self._lock = threading.Lock() self._lock = threading.Lock()
self._step: int = step self._step: int = step
self._current: int = _load_current_id(db_conn, table, column, step) self._current: int = _load_current_id(db_conn, table, column, step)
self._is_writer = is_writer
for table, column in extra_tables: for table, column in extra_tables:
self._current = (max if step > 0 else min)( self._current = (max if step > 0 else min)(
self._current, _load_current_id(db_conn, table, column, step) self._current, _load_current_id(db_conn, table, column, step)
@ -206,11 +204,9 @@ class StreamIdGenerator(AbstractStreamIdGenerator):
self._unfinished_ids: OrderedDict[int, int] = OrderedDict() self._unfinished_ids: OrderedDict[int, int] = OrderedDict()
def advance(self, instance_name: str, new_id: int) -> None: def advance(self, instance_name: str, new_id: int) -> None:
# Advance should never be called on a writer instance, only over replication # `StreamIdGenerator` should only be used when there is a single writer,
if self._is_writer: # so replication should never happen.
raise Exception("Replication is not supported by writer StreamIdGenerator") raise Exception("Replication is not supported by StreamIdGenerator")
self._current = (max if self._step > 0 else min)(self._current, new_id)
def get_next(self) -> AsyncContextManager[int]: def get_next(self) -> AsyncContextManager[int]:
with self._lock: with self._lock:
@ -253,9 +249,6 @@ class StreamIdGenerator(AbstractStreamIdGenerator):
return _AsyncCtxManagerWrapper(manager()) return _AsyncCtxManagerWrapper(manager())
def get_current_token(self) -> int: def get_current_token(self) -> int:
if self._is_writer:
return self._current
with self._lock: with self._lock:
if self._unfinished_ids: if self._unfinished_ids:
return next(iter(self._unfinished_ids)) - self._step return next(iter(self._unfinished_ids)) - self._step