2019-10-21 13:56:42 +02:00
|
|
|
# Copyright 2014-2016 OpenMarket Ltd
|
2020-01-31 11:28:15 +01:00
|
|
|
# Copyright 2020 The Matrix.org Foundation C.I.C.
|
2019-10-21 13:56:42 +02:00
|
|
|
#
|
|
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
# you may not use this file except in compliance with the License.
|
|
|
|
# You may obtain a copy of the License at
|
|
|
|
#
|
|
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
#
|
|
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
# See the License for the specific language governing permissions and
|
|
|
|
# limitations under the License.
|
2022-04-27 15:00:07 +02:00
|
|
|
import collections.abc
|
2019-10-21 13:56:42 +02:00
|
|
|
import logging
|
2023-09-19 21:26:44 +02:00
|
|
|
from typing import (
|
|
|
|
TYPE_CHECKING,
|
|
|
|
Any,
|
|
|
|
Collection,
|
|
|
|
Dict,
|
|
|
|
Iterable,
|
|
|
|
Mapping,
|
|
|
|
Optional,
|
|
|
|
Set,
|
|
|
|
Tuple,
|
|
|
|
)
|
2022-03-31 19:38:09 +02:00
|
|
|
|
2022-05-26 11:48:12 +02:00
|
|
|
import attr
|
|
|
|
|
2020-01-30 18:17:44 +01:00
|
|
|
from synapse.api.constants import EventTypes, Membership
|
2020-01-31 11:28:15 +01:00
|
|
|
from synapse.api.errors import NotFoundError, UnsupportedRoomVersionError
|
|
|
|
from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, RoomVersion
|
2020-07-30 13:20:41 +02:00
|
|
|
from synapse.events import EventBase
|
2022-04-12 15:23:43 +02:00
|
|
|
from synapse.events.snapshot import EventContext
|
2022-09-07 18:41:52 +02:00
|
|
|
from synapse.logging.opentracing import trace
|
2022-12-19 15:57:51 +01:00
|
|
|
from synapse.replication.tcp.streams import UnPartialStatedEventStream
|
|
|
|
from synapse.replication.tcp.streams.partial_state import UnPartialStatedEventStreamRow
|
2019-10-21 13:56:42 +02:00
|
|
|
from synapse.storage._base import SQLBaseStore
|
2021-12-13 18:05:00 +01:00
|
|
|
from synapse.storage.database import (
|
|
|
|
DatabasePool,
|
|
|
|
LoggingDatabaseConnection,
|
|
|
|
LoggingTransaction,
|
2022-05-26 11:48:12 +02:00
|
|
|
make_in_list_sql_clause,
|
2021-12-13 18:05:00 +01:00
|
|
|
)
|
2020-08-05 22:38:57 +02:00
|
|
|
from synapse.storage.databases.main.events_worker import EventsWorkerStore
|
|
|
|
from synapse.storage.databases.main.roommember import RoomMemberWorkerStore
|
2022-03-31 19:38:09 +02:00
|
|
|
from synapse.types import JsonDict, JsonMapping, StateMap
|
2022-12-12 17:19:30 +01:00
|
|
|
from synapse.types.state import StateFilter
|
2019-12-20 11:48:24 +01:00
|
|
|
from synapse.util.caches import intern_string
|
2019-10-21 13:56:42 +02:00
|
|
|
from synapse.util.caches.descriptors import cached, cachedList
|
2022-09-07 13:03:32 +02:00
|
|
|
from synapse.util.cancellation import cancellable
|
2022-05-26 11:48:12 +02:00
|
|
|
from synapse.util.iterutils import batch_iter
|
2019-10-21 13:56:42 +02:00
|
|
|
|
2021-10-22 19:15:41 +02:00
|
|
|
if TYPE_CHECKING:
|
|
|
|
from synapse.server import HomeServer
|
|
|
|
|
2019-10-21 13:56:42 +02:00
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
|
|
|
|
|
|
MAX_STATE_DELTA_HOPS = 100
|
|
|
|
|
|
|
|
|
2022-05-26 11:48:12 +02:00
|
|
|
@attr.s(slots=True, frozen=True, auto_attribs=True)
|
|
|
|
class EventMetadata:
|
|
|
|
"""Returned by `get_metadata_for_events`"""
|
|
|
|
|
|
|
|
room_id: str
|
|
|
|
event_type: str
|
|
|
|
state_key: Optional[str]
|
2022-06-01 13:29:51 +02:00
|
|
|
rejection_reason: Optional[str]
|
2022-05-26 11:48:12 +02:00
|
|
|
|
|
|
|
|
2022-03-02 11:35:34 +01:00
|
|
|
def _retrieve_and_check_room_version(room_id: str, room_version_id: str) -> RoomVersion:
|
|
|
|
v = KNOWN_ROOM_VERSIONS.get(room_version_id)
|
|
|
|
if not v:
|
|
|
|
raise UnsupportedRoomVersionError(
|
|
|
|
"Room %s uses a room version %s which is no longer supported"
|
|
|
|
% (room_id, room_version_id)
|
|
|
|
)
|
|
|
|
return v
|
|
|
|
|
|
|
|
|
2019-10-21 13:56:42 +02:00
|
|
|
# this inherits from EventsWorkerStore because it calls self.get_events
|
2019-12-20 11:48:24 +01:00
|
|
|
class StateGroupWorkerStore(EventsWorkerStore, SQLBaseStore):
|
2021-02-16 23:32:34 +01:00
|
|
|
"""The parts of StateGroupStore that can be called from workers."""
|
2019-10-21 13:56:42 +02:00
|
|
|
|
2021-12-13 18:05:00 +01:00
|
|
|
def __init__(
|
|
|
|
self,
|
|
|
|
database: DatabasePool,
|
|
|
|
db_conn: LoggingDatabaseConnection,
|
|
|
|
hs: "HomeServer",
|
|
|
|
):
|
2020-09-18 15:56:44 +02:00
|
|
|
super().__init__(database, db_conn, hs)
|
2022-12-14 15:47:11 +01:00
|
|
|
self._instance_name: str = hs.get_instance_name()
|
2019-10-21 13:56:42 +02:00
|
|
|
|
2022-12-19 15:57:51 +01:00
|
|
|
def process_replication_rows(
|
|
|
|
self,
|
|
|
|
stream_name: str,
|
|
|
|
instance_name: str,
|
|
|
|
token: int,
|
|
|
|
rows: Iterable[Any],
|
|
|
|
) -> None:
|
|
|
|
if stream_name == UnPartialStatedEventStream.NAME:
|
|
|
|
for row in rows:
|
|
|
|
assert isinstance(row, UnPartialStatedEventStreamRow)
|
|
|
|
self._get_state_group_for_event.invalidate((row.event_id,))
|
2023-01-22 22:10:11 +01:00
|
|
|
self.is_partial_state_event.invalidate((row.event_id,))
|
2022-12-19 15:57:51 +01:00
|
|
|
|
|
|
|
super().process_replication_rows(stream_name, instance_name, token, rows)
|
|
|
|
|
2020-01-31 11:28:15 +01:00
|
|
|
async def get_room_version(self, room_id: str) -> RoomVersion:
|
|
|
|
"""Get the room_version of a given room
|
|
|
|
Raises:
|
|
|
|
NotFoundError: if the room is unknown
|
2021-07-28 17:46:37 +02:00
|
|
|
UnsupportedRoomVersionError: if the room uses an unknown room version.
|
|
|
|
Typically this happens if support for the room's version has been
|
|
|
|
removed from Synapse.
|
|
|
|
"""
|
2022-03-02 11:35:34 +01:00
|
|
|
room_version_id = await self.get_room_version_id(room_id)
|
|
|
|
return _retrieve_and_check_room_version(room_id, room_version_id)
|
2020-01-31 11:28:15 +01:00
|
|
|
|
2021-07-28 17:46:37 +02:00
|
|
|
def get_room_version_txn(
|
|
|
|
self, txn: LoggingTransaction, room_id: str
|
|
|
|
) -> RoomVersion:
|
|
|
|
"""Get the room_version of a given room
|
|
|
|
Args:
|
|
|
|
txn: Transaction object
|
|
|
|
room_id: The room_id of the room you are trying to get the version for
|
|
|
|
Raises:
|
|
|
|
NotFoundError: if the room is unknown
|
2020-01-31 11:28:15 +01:00
|
|
|
UnsupportedRoomVersionError: if the room uses an unknown room version.
|
|
|
|
Typically this happens if support for the room's version has been
|
|
|
|
removed from Synapse.
|
|
|
|
"""
|
2021-07-28 17:46:37 +02:00
|
|
|
room_version_id = self.get_room_version_id_txn(txn, room_id)
|
2022-03-02 11:35:34 +01:00
|
|
|
return _retrieve_and_check_room_version(room_id, room_version_id)
|
2020-01-31 11:28:15 +01:00
|
|
|
|
2020-01-27 15:30:57 +01:00
|
|
|
@cached(max_entries=10000)
|
2020-01-31 11:06:21 +01:00
|
|
|
async def get_room_version_id(self, room_id: str) -> str:
|
2019-10-21 13:56:42 +02:00
|
|
|
"""Get the room_version of a given room
|
2021-07-28 17:46:37 +02:00
|
|
|
Raises:
|
|
|
|
NotFoundError: if the room is unknown
|
|
|
|
"""
|
|
|
|
return await self.db_pool.runInteraction(
|
|
|
|
"get_room_version_id_txn",
|
|
|
|
self.get_room_version_id_txn,
|
|
|
|
room_id,
|
|
|
|
)
|
2019-10-21 13:56:42 +02:00
|
|
|
|
2021-07-28 17:46:37 +02:00
|
|
|
def get_room_version_id_txn(self, txn: LoggingTransaction, room_id: str) -> str:
|
|
|
|
"""Get the room_version of a given room
|
|
|
|
Args:
|
|
|
|
txn: Transaction object
|
|
|
|
room_id: The room_id of the room you are trying to get the version for
|
2019-10-21 13:56:42 +02:00
|
|
|
Raises:
|
2020-01-27 15:30:57 +01:00
|
|
|
NotFoundError: if the room is unknown
|
2019-10-21 13:56:42 +02:00
|
|
|
"""
|
2020-01-27 15:30:57 +01:00
|
|
|
|
|
|
|
# We really should have an entry in the rooms table for every room we
|
2022-06-07 13:44:31 +02:00
|
|
|
# care about, but let's be a bit paranoid.
|
2021-07-28 17:46:37 +02:00
|
|
|
room_version = self.db_pool.simple_select_one_onecol_txn(
|
|
|
|
txn,
|
2020-01-27 15:30:57 +01:00
|
|
|
table="rooms",
|
|
|
|
keyvalues={"room_id": room_id},
|
|
|
|
retcol="room_version",
|
|
|
|
allow_none=True,
|
|
|
|
)
|
|
|
|
|
2021-07-28 17:46:37 +02:00
|
|
|
if room_version is None:
|
2022-04-18 20:41:55 +02:00
|
|
|
raise NotFoundError("Could not find room_version for %s" % (room_id,))
|
2019-10-21 13:56:42 +02:00
|
|
|
|
2021-07-28 17:46:37 +02:00
|
|
|
return room_version
|
2019-10-21 13:56:42 +02:00
|
|
|
|
2022-09-07 18:41:52 +02:00
|
|
|
@trace
|
2022-05-26 11:48:12 +02:00
|
|
|
async def get_metadata_for_events(
|
|
|
|
self, event_ids: Collection[str]
|
|
|
|
) -> Dict[str, EventMetadata]:
|
|
|
|
"""Get some metadata (room_id, type, state_key) for the given events.
|
|
|
|
|
|
|
|
This method is a faster alternative than fetching the full events from
|
|
|
|
the DB, and should be used when the full event is not needed.
|
|
|
|
|
|
|
|
Returns metadata for rejected and redacted events. Events that have not
|
|
|
|
been persisted are omitted from the returned dict.
|
|
|
|
"""
|
|
|
|
|
|
|
|
def get_metadata_for_events_txn(
|
|
|
|
txn: LoggingTransaction,
|
|
|
|
batch_ids: Collection[str],
|
|
|
|
) -> Dict[str, EventMetadata]:
|
|
|
|
clause, args = make_in_list_sql_clause(
|
|
|
|
self.database_engine, "e.event_id", batch_ids
|
|
|
|
)
|
|
|
|
|
|
|
|
sql = f"""
|
2022-06-01 13:29:51 +02:00
|
|
|
SELECT e.event_id, e.room_id, e.type, se.state_key, r.reason
|
|
|
|
FROM events AS e
|
2022-05-30 11:51:09 +02:00
|
|
|
LEFT JOIN state_events se USING (event_id)
|
2022-06-01 13:29:51 +02:00
|
|
|
LEFT JOIN rejections r USING (event_id)
|
2022-05-26 11:48:12 +02:00
|
|
|
WHERE {clause}
|
|
|
|
"""
|
|
|
|
|
|
|
|
txn.execute(sql, args)
|
|
|
|
return {
|
|
|
|
event_id: EventMetadata(
|
2022-06-01 13:29:51 +02:00
|
|
|
room_id=room_id,
|
|
|
|
event_type=event_type,
|
|
|
|
state_key=state_key,
|
|
|
|
rejection_reason=rejection_reason,
|
2022-05-26 11:48:12 +02:00
|
|
|
)
|
2022-06-01 13:29:51 +02:00
|
|
|
for event_id, room_id, event_type, state_key, rejection_reason in txn
|
2022-05-26 11:48:12 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
result_map: Dict[str, EventMetadata] = {}
|
|
|
|
for batch_ids in batch_iter(event_ids, 1000):
|
|
|
|
result_map.update(
|
|
|
|
await self.db_pool.runInteraction(
|
|
|
|
"get_metadata_for_events",
|
|
|
|
get_metadata_for_events_txn,
|
|
|
|
batch_ids=batch_ids,
|
|
|
|
)
|
|
|
|
)
|
|
|
|
|
|
|
|
return result_map
|
|
|
|
|
2022-03-31 19:38:09 +02:00
|
|
|
async def get_room_predecessor(self, room_id: str) -> Optional[JsonMapping]:
|
2019-12-11 14:07:25 +01:00
|
|
|
"""Get the predecessor of an upgraded room if it exists.
|
2019-10-21 13:56:42 +02:00
|
|
|
Otherwise return None.
|
|
|
|
|
|
|
|
Args:
|
2020-07-30 13:20:41 +02:00
|
|
|
room_id: The room ID.
|
2019-10-21 13:56:42 +02:00
|
|
|
|
|
|
|
Returns:
|
2020-07-30 13:20:41 +02:00
|
|
|
A dictionary containing the structure of the predecessor
|
|
|
|
field from the room's create event. The structure is subject to other servers,
|
|
|
|
but it is expected to be:
|
|
|
|
* room_id (str): The room ID of the predecessor room
|
|
|
|
* event_id (str): The ID of the tombstone event in the predecessor room
|
2019-10-21 13:56:42 +02:00
|
|
|
|
2020-07-30 13:20:41 +02:00
|
|
|
None if a predecessor key is not found, or is not a dictionary.
|
2019-12-11 14:07:25 +01:00
|
|
|
|
2019-10-21 13:56:42 +02:00
|
|
|
Raises:
|
2019-12-11 14:07:25 +01:00
|
|
|
NotFoundError if the given room is unknown
|
2019-10-21 13:56:42 +02:00
|
|
|
"""
|
|
|
|
# Retrieve the room's create event
|
2020-07-30 13:20:41 +02:00
|
|
|
create_event = await self.get_create_event_for_room(room_id)
|
2019-10-21 13:56:42 +02:00
|
|
|
|
2019-12-11 14:07:25 +01:00
|
|
|
# Retrieve the predecessor key of the create event
|
|
|
|
predecessor = create_event.content.get("predecessor", None)
|
|
|
|
|
|
|
|
# Ensure the key is a dictionary
|
2022-04-27 15:00:07 +02:00
|
|
|
if not isinstance(predecessor, collections.abc.Mapping):
|
2019-12-11 14:07:25 +01:00
|
|
|
return None
|
|
|
|
|
2022-03-31 19:38:09 +02:00
|
|
|
# The keys must be strings since the data is JSON.
|
2019-12-11 14:07:25 +01:00
|
|
|
return predecessor
|
2019-10-21 13:56:42 +02:00
|
|
|
|
2020-07-30 13:20:41 +02:00
|
|
|
async def get_create_event_for_room(self, room_id: str) -> EventBase:
|
2019-10-21 13:56:42 +02:00
|
|
|
"""Get the create state event for a room.
|
|
|
|
|
|
|
|
Args:
|
2020-07-30 13:20:41 +02:00
|
|
|
room_id: The room ID.
|
2019-10-21 13:56:42 +02:00
|
|
|
|
|
|
|
Returns:
|
2020-07-30 13:20:41 +02:00
|
|
|
The room creation event.
|
2019-10-21 13:56:42 +02:00
|
|
|
|
|
|
|
Raises:
|
|
|
|
NotFoundError if the room is unknown
|
|
|
|
"""
|
2022-06-01 17:02:53 +02:00
|
|
|
state_ids = await self.get_partial_current_state_ids(room_id)
|
2022-01-04 17:10:05 +01:00
|
|
|
|
|
|
|
if not state_ids:
|
|
|
|
raise NotFoundError(f"Current state for room {room_id} is empty")
|
|
|
|
|
2019-10-21 13:56:42 +02:00
|
|
|
create_id = state_ids.get((EventTypes.Create, ""))
|
|
|
|
|
|
|
|
# If we can't find the create event, assume we've hit a dead end
|
|
|
|
if not create_id:
|
2022-01-04 17:10:05 +01:00
|
|
|
raise NotFoundError(f"No create event in current state for room {room_id}")
|
2019-10-21 13:56:42 +02:00
|
|
|
|
|
|
|
# Retrieve the room's create event and return
|
2020-07-30 13:20:41 +02:00
|
|
|
create_event = await self.get_event(create_id)
|
2019-10-21 13:56:42 +02:00
|
|
|
return create_event
|
|
|
|
|
|
|
|
@cached(max_entries=100000, iterable=True)
|
2022-06-01 17:02:53 +02:00
|
|
|
async def get_partial_current_state_ids(self, room_id: str) -> StateMap[str]:
|
2019-10-21 13:56:42 +02:00
|
|
|
"""Get the current state event ids for a room based on the
|
|
|
|
current_state_events table.
|
|
|
|
|
2022-06-01 17:02:53 +02:00
|
|
|
This may be the partial state if we're lazy joining the room.
|
|
|
|
|
2019-10-21 13:56:42 +02:00
|
|
|
Args:
|
2020-08-28 15:37:55 +02:00
|
|
|
room_id: The room to get the state IDs of.
|
2019-10-21 13:56:42 +02:00
|
|
|
|
|
|
|
Returns:
|
2020-08-28 15:37:55 +02:00
|
|
|
The current state of the room.
|
2019-10-21 13:56:42 +02:00
|
|
|
"""
|
|
|
|
|
2022-04-12 12:54:00 +02:00
|
|
|
def _get_current_state_ids_txn(txn: LoggingTransaction) -> StateMap[str]:
|
2019-10-21 13:56:42 +02:00
|
|
|
txn.execute(
|
|
|
|
"""SELECT type, state_key, event_id FROM current_state_events
|
|
|
|
WHERE room_id = ?
|
|
|
|
""",
|
|
|
|
(room_id,),
|
|
|
|
)
|
|
|
|
|
2020-05-15 20:12:03 +02:00
|
|
|
return {(intern_string(r[0]), intern_string(r[1])): r[2] for r in txn}
|
2019-10-21 13:56:42 +02:00
|
|
|
|
2020-08-28 15:37:55 +02:00
|
|
|
return await self.db_pool.runInteraction(
|
2022-06-01 17:02:53 +02:00
|
|
|
"get_partial_current_state_ids", _get_current_state_ids_txn
|
2019-12-04 14:52:46 +01:00
|
|
|
)
|
2019-10-21 13:56:42 +02:00
|
|
|
|
|
|
|
# FIXME: how should this be cached?
|
2022-09-07 13:03:32 +02:00
|
|
|
@cancellable
|
2022-06-01 17:02:53 +02:00
|
|
|
async def get_partial_filtered_current_state_ids(
|
2021-04-08 23:38:54 +02:00
|
|
|
self, room_id: str, state_filter: Optional[StateFilter] = None
|
2020-08-28 15:37:55 +02:00
|
|
|
) -> StateMap[str]:
|
2019-10-21 13:56:42 +02:00
|
|
|
"""Get the current state event of a given type for a room based on the
|
|
|
|
current_state_events table. This may not be as up-to-date as the result
|
|
|
|
of doing a fresh state resolution as per state_handler.get_current_state
|
|
|
|
|
2022-06-01 17:02:53 +02:00
|
|
|
This may be the partial state if we're lazy joining the room.
|
|
|
|
|
2019-10-21 13:56:42 +02:00
|
|
|
Args:
|
2020-01-16 14:31:22 +01:00
|
|
|
room_id
|
|
|
|
state_filter: The state filter used to fetch state
|
2019-10-21 13:56:42 +02:00
|
|
|
from the database.
|
|
|
|
|
|
|
|
Returns:
|
2020-08-28 15:37:55 +02:00
|
|
|
Map from type/state_key to event ID.
|
2019-10-21 13:56:42 +02:00
|
|
|
"""
|
|
|
|
|
2021-04-08 23:38:54 +02:00
|
|
|
where_clause, where_args = (
|
|
|
|
state_filter or StateFilter.all()
|
|
|
|
).make_sql_filter_clause()
|
2019-10-21 13:56:42 +02:00
|
|
|
|
|
|
|
if not where_clause:
|
|
|
|
# We delegate to the cached version
|
2022-06-01 17:02:53 +02:00
|
|
|
return await self.get_partial_current_state_ids(room_id)
|
2019-10-21 13:56:42 +02:00
|
|
|
|
2022-03-28 20:11:14 +02:00
|
|
|
def _get_filtered_current_state_ids_txn(
|
|
|
|
txn: LoggingTransaction,
|
|
|
|
) -> StateMap[str]:
|
2019-10-21 13:56:42 +02:00
|
|
|
results = {}
|
|
|
|
sql = """
|
|
|
|
SELECT type, state_key, event_id FROM current_state_events
|
|
|
|
WHERE room_id = ?
|
|
|
|
"""
|
|
|
|
|
|
|
|
if where_clause:
|
|
|
|
sql += " AND (%s)" % (where_clause,)
|
|
|
|
|
|
|
|
args = [room_id]
|
|
|
|
args.extend(where_args)
|
|
|
|
txn.execute(sql, args)
|
|
|
|
for row in txn:
|
|
|
|
typ, state_key, event_id = row
|
|
|
|
key = (intern_string(typ), intern_string(state_key))
|
|
|
|
results[key] = event_id
|
|
|
|
|
|
|
|
return results
|
|
|
|
|
2020-08-28 15:37:55 +02:00
|
|
|
return await self.db_pool.runInteraction(
|
2019-10-21 13:56:42 +02:00
|
|
|
"get_filtered_current_state_ids", _get_filtered_current_state_ids_txn
|
|
|
|
)
|
|
|
|
|
|
|
|
@cached(max_entries=50000)
|
2020-08-26 13:19:32 +02:00
|
|
|
async def _get_state_group_for_event(self, event_id: str) -> Optional[int]:
|
|
|
|
return await self.db_pool.simple_select_one_onecol(
|
2019-10-21 13:56:42 +02:00
|
|
|
table="event_to_state_groups",
|
|
|
|
keyvalues={"event_id": event_id},
|
|
|
|
retcol="state_group",
|
|
|
|
allow_none=True,
|
|
|
|
desc="_get_state_group_for_event",
|
|
|
|
)
|
|
|
|
|
|
|
|
@cachedList(
|
|
|
|
cached_method_name="_get_state_group_for_event",
|
|
|
|
list_name="event_ids",
|
|
|
|
num_args=1,
|
|
|
|
)
|
2022-03-31 19:38:09 +02:00
|
|
|
async def _get_state_group_for_events(
|
2022-04-01 14:01:49 +02:00
|
|
|
self, event_ids: Collection[str]
|
2023-09-19 21:26:44 +02:00
|
|
|
) -> Mapping[str, int]:
|
2022-04-01 14:01:49 +02:00
|
|
|
"""Returns mapping event_id -> state_group.
|
|
|
|
|
|
|
|
Raises:
|
|
|
|
RuntimeError if the state is unknown at any of the given events
|
|
|
|
"""
|
2020-08-14 13:24:26 +02:00
|
|
|
rows = await self.db_pool.simple_select_many_batch(
|
2019-10-21 13:56:42 +02:00
|
|
|
table="event_to_state_groups",
|
|
|
|
column="event_id",
|
|
|
|
iterable=event_ids,
|
|
|
|
keyvalues={},
|
|
|
|
retcols=("event_id", "state_group"),
|
|
|
|
desc="_get_state_group_for_events",
|
|
|
|
)
|
|
|
|
|
2022-04-01 14:01:49 +02:00
|
|
|
res = {row["event_id"]: row["state_group"] for row in rows}
|
|
|
|
for e in event_ids:
|
|
|
|
if e not in res:
|
|
|
|
raise RuntimeError("No state group for unknown or outlier event %s" % e)
|
|
|
|
return res
|
2019-10-21 13:56:42 +02:00
|
|
|
|
2020-07-30 13:20:41 +02:00
|
|
|
async def get_referenced_state_groups(
|
|
|
|
self, state_groups: Iterable[int]
|
|
|
|
) -> Set[int]:
|
2019-10-30 16:12:49 +01:00
|
|
|
"""Check if the state groups are referenced by events.
|
|
|
|
|
|
|
|
Args:
|
2020-07-30 13:20:41 +02:00
|
|
|
state_groups
|
2019-10-30 16:12:49 +01:00
|
|
|
|
|
|
|
Returns:
|
2020-07-30 13:20:41 +02:00
|
|
|
The subset of state groups that are referenced.
|
2019-10-30 16:12:49 +01:00
|
|
|
"""
|
|
|
|
|
2020-08-05 22:38:57 +02:00
|
|
|
rows = await self.db_pool.simple_select_many_batch(
|
2019-10-30 16:12:49 +01:00
|
|
|
table="event_to_state_groups",
|
|
|
|
column="state_group",
|
|
|
|
iterable=state_groups,
|
|
|
|
keyvalues={},
|
|
|
|
retcols=("DISTINCT state_group",),
|
|
|
|
desc="get_referenced_state_groups",
|
|
|
|
)
|
|
|
|
|
2020-02-21 13:15:07 +01:00
|
|
|
return {row["state_group"] for row in rows}
|
2019-10-30 16:12:49 +01:00
|
|
|
|
2022-04-12 15:23:43 +02:00
|
|
|
async def update_state_for_partial_state_event(
|
|
|
|
self,
|
|
|
|
event: EventBase,
|
|
|
|
context: EventContext,
|
|
|
|
) -> None:
|
|
|
|
"""Update the state group for a partial state event"""
|
2022-12-14 15:47:11 +01:00
|
|
|
async with self._un_partial_stated_events_stream_id_gen.get_next() as un_partial_state_event_stream_id:
|
|
|
|
await self.db_pool.runInteraction(
|
|
|
|
"update_state_for_partial_state_event",
|
|
|
|
self._update_state_for_partial_state_event_txn,
|
|
|
|
event,
|
|
|
|
context,
|
|
|
|
un_partial_state_event_stream_id,
|
|
|
|
)
|
2022-04-12 15:23:43 +02:00
|
|
|
|
|
|
|
def _update_state_for_partial_state_event_txn(
|
|
|
|
self,
|
2022-04-27 14:05:00 +02:00
|
|
|
txn: LoggingTransaction,
|
2022-04-12 15:23:43 +02:00
|
|
|
event: EventBase,
|
|
|
|
context: EventContext,
|
2022-12-14 15:47:11 +01:00
|
|
|
un_partial_state_event_stream_id: int,
|
2022-04-27 14:05:00 +02:00
|
|
|
) -> None:
|
2022-04-12 15:23:43 +02:00
|
|
|
# we shouldn't have any outliers here
|
|
|
|
assert not event.internal_metadata.is_outlier()
|
|
|
|
|
|
|
|
# anything that was rejected should have the same state as its
|
|
|
|
# predecessor.
|
|
|
|
if context.rejected:
|
2022-08-01 12:20:05 +02:00
|
|
|
state_group = context.state_group_before_event
|
|
|
|
else:
|
|
|
|
state_group = context.state_group
|
2022-04-12 15:23:43 +02:00
|
|
|
|
|
|
|
self.db_pool.simple_update_txn(
|
|
|
|
txn,
|
|
|
|
table="event_to_state_groups",
|
|
|
|
keyvalues={"event_id": event.event_id},
|
2022-08-01 12:20:05 +02:00
|
|
|
updatevalues={"state_group": state_group},
|
2022-04-12 15:23:43 +02:00
|
|
|
)
|
|
|
|
|
2022-08-11 12:42:24 +02:00
|
|
|
# the event may now be rejected where it was not before, or vice versa,
|
|
|
|
# in which case we need to update the rejected flags.
|
2022-12-14 15:47:11 +01:00
|
|
|
rejection_status_changed = bool(context.rejected) != (
|
|
|
|
event.rejected_reason is not None
|
|
|
|
)
|
|
|
|
if rejection_status_changed:
|
2022-08-11 12:42:24 +02:00
|
|
|
self.mark_event_rejected_txn(txn, event.event_id, context.rejected)
|
|
|
|
|
2022-04-12 15:23:43 +02:00
|
|
|
self.db_pool.simple_delete_one_txn(
|
|
|
|
txn,
|
|
|
|
table="partial_state_events",
|
|
|
|
keyvalues={"event_id": event.event_id},
|
|
|
|
)
|
|
|
|
|
2022-04-21 08:42:03 +02:00
|
|
|
txn.call_after(self.is_partial_state_event.invalidate, (event.event_id,))
|
2022-04-12 15:23:43 +02:00
|
|
|
txn.call_after(
|
|
|
|
self._get_state_group_for_event.prefill,
|
|
|
|
(event.event_id,),
|
2022-08-01 12:20:05 +02:00
|
|
|
state_group,
|
2022-04-12 15:23:43 +02:00
|
|
|
)
|
|
|
|
|
2022-12-14 15:47:11 +01:00
|
|
|
self.db_pool.simple_insert_txn(
|
|
|
|
txn,
|
|
|
|
"un_partial_stated_event_stream",
|
|
|
|
{
|
|
|
|
"stream_id": un_partial_state_event_stream_id,
|
|
|
|
"instance_name": self._instance_name,
|
|
|
|
"event_id": event.event_id,
|
|
|
|
"rejection_status_changed": rejection_status_changed,
|
|
|
|
},
|
|
|
|
)
|
2023-01-22 22:10:11 +01:00
|
|
|
txn.call_after(self.hs.get_notifier().on_new_replication_data)
|
2022-12-14 15:47:11 +01:00
|
|
|
|
2019-10-21 13:56:42 +02:00
|
|
|
|
2020-01-30 18:17:44 +01:00
|
|
|
class MainStateBackgroundUpdateStore(RoomMemberWorkerStore):
|
2019-10-21 13:56:42 +02:00
|
|
|
CURRENT_STATE_INDEX_UPDATE_NAME = "current_state_members_idx"
|
|
|
|
EVENT_STATE_GROUP_INDEX_UPDATE_NAME = "event_to_state_groups_sg_index"
|
2020-01-30 18:17:44 +01:00
|
|
|
DELETE_CURRENT_STATE_UPDATE_NAME = "delete_old_current_state_events"
|
2019-10-21 13:56:42 +02:00
|
|
|
|
2021-12-13 18:05:00 +01:00
|
|
|
def __init__(
|
|
|
|
self,
|
|
|
|
database: DatabasePool,
|
|
|
|
db_conn: LoggingDatabaseConnection,
|
|
|
|
hs: "HomeServer",
|
|
|
|
):
|
2020-09-18 15:56:44 +02:00
|
|
|
super().__init__(database, db_conn, hs)
|
2019-12-20 11:48:24 +01:00
|
|
|
|
2022-03-28 20:11:14 +02:00
|
|
|
self.server_name: str = hs.hostname
|
2020-01-30 18:17:44 +01:00
|
|
|
|
2020-08-05 22:38:57 +02:00
|
|
|
self.db_pool.updates.register_background_index_update(
|
2019-10-21 13:56:42 +02:00
|
|
|
self.CURRENT_STATE_INDEX_UPDATE_NAME,
|
|
|
|
index_name="current_state_events_member_index",
|
|
|
|
table="current_state_events",
|
|
|
|
columns=["state_key"],
|
|
|
|
where_clause="type='m.room.member'",
|
|
|
|
)
|
2020-08-05 22:38:57 +02:00
|
|
|
self.db_pool.updates.register_background_index_update(
|
2019-10-21 13:56:42 +02:00
|
|
|
self.EVENT_STATE_GROUP_INDEX_UPDATE_NAME,
|
|
|
|
index_name="event_to_state_groups_sg_index",
|
|
|
|
table="event_to_state_groups",
|
|
|
|
columns=["state_group"],
|
|
|
|
)
|
2020-08-05 22:38:57 +02:00
|
|
|
self.db_pool.updates.register_background_update_handler(
|
2021-02-16 23:32:34 +01:00
|
|
|
self.DELETE_CURRENT_STATE_UPDATE_NAME,
|
|
|
|
self._background_remove_left_rooms,
|
2020-01-30 18:17:44 +01:00
|
|
|
)
|
|
|
|
|
2022-03-28 20:11:14 +02:00
|
|
|
async def _background_remove_left_rooms(
|
|
|
|
self, progress: JsonDict, batch_size: int
|
|
|
|
) -> int:
|
2020-01-30 18:17:44 +01:00
|
|
|
"""Background update to delete rows from `current_state_events` and
|
|
|
|
`event_forward_extremities` tables of rooms that the server is no
|
|
|
|
longer joined to.
|
|
|
|
"""
|
|
|
|
|
|
|
|
last_room_id = progress.get("last_room_id", "")
|
|
|
|
|
2022-03-28 20:11:14 +02:00
|
|
|
def _background_remove_left_rooms_txn(
|
|
|
|
txn: LoggingTransaction,
|
|
|
|
) -> Tuple[bool, Set[str]]:
|
2020-07-15 19:33:03 +02:00
|
|
|
# get a batch of room ids to consider
|
2020-01-30 18:17:44 +01:00
|
|
|
sql = """
|
|
|
|
SELECT DISTINCT room_id FROM current_state_events
|
|
|
|
WHERE room_id > ? ORDER BY room_id LIMIT ?
|
|
|
|
"""
|
|
|
|
|
|
|
|
txn.execute(sql, (last_room_id, batch_size))
|
2020-02-21 13:15:07 +01:00
|
|
|
room_ids = [row[0] for row in txn]
|
2020-01-30 18:17:44 +01:00
|
|
|
if not room_ids:
|
|
|
|
return True, set()
|
|
|
|
|
2020-07-15 19:33:03 +02:00
|
|
|
###########################################################################
|
|
|
|
#
|
|
|
|
# exclude rooms where we have active members
|
|
|
|
|
2020-01-30 18:17:44 +01:00
|
|
|
sql = """
|
|
|
|
SELECT room_id
|
2020-07-15 19:33:03 +02:00
|
|
|
FROM local_current_membership
|
2020-01-30 18:17:44 +01:00
|
|
|
WHERE
|
|
|
|
room_id > ? AND room_id <= ?
|
|
|
|
AND membership = 'join'
|
|
|
|
GROUP BY room_id
|
|
|
|
"""
|
|
|
|
|
2020-07-15 19:33:03 +02:00
|
|
|
txn.execute(sql, (last_room_id, room_ids[-1]))
|
2020-02-21 13:15:07 +01:00
|
|
|
joined_room_ids = {row[0] for row in txn}
|
2020-07-15 19:33:03 +02:00
|
|
|
to_delete = set(room_ids) - joined_room_ids
|
|
|
|
|
|
|
|
###########################################################################
|
|
|
|
#
|
|
|
|
# exclude rooms which we are in the process of constructing; these otherwise
|
|
|
|
# qualify as "rooms with no local users", and would have their
|
|
|
|
# forward extremities cleaned up.
|
|
|
|
|
|
|
|
# the following query will return a list of rooms which have forward
|
|
|
|
# extremities that are *not* also the create event in the room - ie
|
|
|
|
# those that are not being created currently.
|
|
|
|
|
|
|
|
sql = """
|
|
|
|
SELECT DISTINCT efe.room_id
|
|
|
|
FROM event_forward_extremities efe
|
|
|
|
LEFT JOIN current_state_events cse ON
|
|
|
|
cse.event_id = efe.event_id
|
|
|
|
AND cse.type = 'm.room.create'
|
|
|
|
AND cse.state_key = ''
|
|
|
|
WHERE
|
|
|
|
cse.event_id IS NULL
|
|
|
|
AND efe.room_id > ? AND efe.room_id <= ?
|
|
|
|
"""
|
|
|
|
|
|
|
|
txn.execute(sql, (last_room_id, room_ids[-1]))
|
|
|
|
|
|
|
|
# build a set of those rooms within `to_delete` that do not appear in
|
|
|
|
# the above, leaving us with the rooms in `to_delete` that *are* being
|
|
|
|
# created.
|
|
|
|
creating_rooms = to_delete.difference(row[0] for row in txn)
|
|
|
|
logger.info("skipping rooms which are being created: %s", creating_rooms)
|
|
|
|
|
|
|
|
# now remove the rooms being created from the list of those to delete.
|
|
|
|
#
|
|
|
|
# (we could have just taken the intersection of `to_delete` with the result
|
|
|
|
# of the sql query, but it's useful to be able to log `creating_rooms`; and
|
|
|
|
# having done so, it's quicker to remove the (few) creating rooms from
|
|
|
|
# `to_delete` than it is to form the intersection with the (larger) list of
|
|
|
|
# not-creating-rooms)
|
|
|
|
|
|
|
|
to_delete -= creating_rooms
|
2020-01-30 18:17:44 +01:00
|
|
|
|
2020-07-15 19:33:03 +02:00
|
|
|
###########################################################################
|
|
|
|
#
|
|
|
|
# now clear the state for the rooms
|
2020-01-30 18:17:44 +01:00
|
|
|
|
2020-07-15 19:33:03 +02:00
|
|
|
logger.info("Deleting current state left rooms: %r", to_delete)
|
2020-01-30 18:55:34 +01:00
|
|
|
|
2020-01-30 18:17:44 +01:00
|
|
|
# First we get all users that we still think were joined to the
|
|
|
|
# room. This is so that we can mark those device lists as
|
|
|
|
# potentially stale, since there may have been a period where the
|
|
|
|
# server didn't share a room with the remote user and therefore may
|
|
|
|
# have missed any device updates.
|
2020-08-05 22:38:57 +02:00
|
|
|
rows = self.db_pool.simple_select_many_txn(
|
2020-01-30 18:17:44 +01:00
|
|
|
txn,
|
|
|
|
table="current_state_events",
|
|
|
|
column="room_id",
|
2020-07-15 19:33:03 +02:00
|
|
|
iterable=to_delete,
|
2020-01-30 18:17:44 +01:00
|
|
|
keyvalues={"type": EventTypes.Member, "membership": Membership.JOIN},
|
|
|
|
retcols=("state_key",),
|
|
|
|
)
|
|
|
|
|
2020-02-21 13:15:07 +01:00
|
|
|
potentially_left_users = {row["state_key"] for row in rows}
|
2020-01-30 18:17:44 +01:00
|
|
|
|
|
|
|
# Now lets actually delete the rooms from the DB.
|
2020-08-05 22:38:57 +02:00
|
|
|
self.db_pool.simple_delete_many_txn(
|
2020-01-30 18:17:44 +01:00
|
|
|
txn,
|
|
|
|
table="current_state_events",
|
|
|
|
column="room_id",
|
2021-09-20 11:26:13 +02:00
|
|
|
values=to_delete,
|
2020-01-30 18:17:44 +01:00
|
|
|
keyvalues={},
|
|
|
|
)
|
|
|
|
|
2020-08-05 22:38:57 +02:00
|
|
|
self.db_pool.simple_delete_many_txn(
|
2020-01-30 18:17:44 +01:00
|
|
|
txn,
|
|
|
|
table="event_forward_extremities",
|
|
|
|
column="room_id",
|
2021-09-20 11:26:13 +02:00
|
|
|
values=to_delete,
|
2020-01-30 18:17:44 +01:00
|
|
|
keyvalues={},
|
|
|
|
)
|
|
|
|
|
2020-08-05 22:38:57 +02:00
|
|
|
self.db_pool.updates._background_update_progress_txn(
|
2020-01-30 18:17:44 +01:00
|
|
|
txn,
|
|
|
|
self.DELETE_CURRENT_STATE_UPDATE_NAME,
|
|
|
|
{"last_room_id": room_ids[-1]},
|
|
|
|
)
|
|
|
|
|
|
|
|
return False, potentially_left_users
|
|
|
|
|
2020-08-05 22:38:57 +02:00
|
|
|
finished, potentially_left_users = await self.db_pool.runInteraction(
|
2020-01-30 18:17:44 +01:00
|
|
|
"_background_remove_left_rooms", _background_remove_left_rooms_txn
|
|
|
|
)
|
|
|
|
|
|
|
|
if finished:
|
2020-08-05 22:38:57 +02:00
|
|
|
await self.db_pool.updates._end_background_update(
|
2020-01-30 18:17:44 +01:00
|
|
|
self.DELETE_CURRENT_STATE_UPDATE_NAME
|
|
|
|
)
|
|
|
|
|
|
|
|
# Now go and check if we still share a room with the remote users in
|
|
|
|
# the deleted rooms. If not mark their device lists as stale.
|
|
|
|
joined_users = await self.get_users_server_still_shares_room_with(
|
|
|
|
potentially_left_users
|
|
|
|
)
|
|
|
|
|
|
|
|
for user_id in potentially_left_users - joined_users:
|
2022-03-31 19:38:09 +02:00
|
|
|
await self.mark_remote_user_device_list_as_unsubscribed(user_id) # type: ignore[attr-defined]
|
2020-01-30 18:17:44 +01:00
|
|
|
|
|
|
|
return batch_size
|
2019-10-21 13:56:42 +02:00
|
|
|
|
|
|
|
|
2019-12-20 11:48:24 +01:00
|
|
|
class StateStore(StateGroupWorkerStore, MainStateBackgroundUpdateStore):
|
2021-02-16 23:32:34 +01:00
|
|
|
"""Keeps track of the state at a given event.
|
2019-10-21 13:56:42 +02:00
|
|
|
|
|
|
|
This is done by the concept of `state groups`. Every event is a assigned
|
|
|
|
a state group (identified by an arbitrary string), which references a
|
|
|
|
collection of state events. The current state of an event is then the
|
|
|
|
collection of state events referenced by the event's state group.
|
|
|
|
|
|
|
|
Hence, every change in the current state causes a new state group to be
|
|
|
|
generated. However, if no change happens (e.g., if we get a message event
|
|
|
|
with only one parent it inherits the state group from its parent.)
|
|
|
|
|
|
|
|
There are three tables:
|
|
|
|
* `state_groups`: Stores group name, first event with in the group and
|
|
|
|
room id.
|
|
|
|
* `event_to_state_groups`: Maps events to state groups.
|
|
|
|
* `state_groups_state`: Maps state group to state events.
|
|
|
|
"""
|
|
|
|
|
2021-12-13 18:05:00 +01:00
|
|
|
def __init__(
|
|
|
|
self,
|
|
|
|
database: DatabasePool,
|
|
|
|
db_conn: LoggingDatabaseConnection,
|
|
|
|
hs: "HomeServer",
|
|
|
|
):
|
2020-09-18 15:56:44 +02:00
|
|
|
super().__init__(database, db_conn, hs)
|