Ensure rejected events get added to some metadata tables (#9016)
Co-authored-by: Patrick Cloke <clokep@users.noreply.github.com>pull/9069/head
Erik Johnston
GitHub
parent
2fb1c2b6e6
commit
63f4990298
|
|
@ -0,0 +1 @@
|
|||
Ensure rejected events get added to some metadata tables.
|
||||
|
|
@ -799,7 +799,8 @@ class PersistEventsStore:
|
|||
return [ec for ec in events_and_contexts if ec[0] not in to_remove]
|
||||
|
||||
def _store_event_txn(self, txn, events_and_contexts):
|
||||
"""Insert new events into the event and event_json tables
|
||||
"""Insert new events into the event, event_json, redaction and
|
||||
state_events tables.
|
||||
|
||||
Args:
|
||||
txn (twisted.enterprise.adbapi.Connection): db connection
|
||||
|
|
@ -871,6 +872,29 @@ class PersistEventsStore:
|
|||
updatevalues={"have_censored": False},
|
||||
)
|
||||
|
||||
state_events_and_contexts = [
|
||||
ec for ec in events_and_contexts if ec[0].is_state()
|
||||
]
|
||||
|
||||
state_values = []
|
||||
for event, context in state_events_and_contexts:
|
||||
vals = {
|
||||
"event_id": event.event_id,
|
||||
"room_id": event.room_id,
|
||||
"type": event.type,
|
||||
"state_key": event.state_key,
|
||||
}
|
||||
|
||||
# TODO: How does this work with backfilling?
|
||||
if hasattr(event, "replaces_state"):
|
||||
vals["prev_state"] = event.replaces_state
|
||||
|
||||
state_values.append(vals)
|
||||
|
||||
self.db_pool.simple_insert_many_txn(
|
||||
txn, table="state_events", values=state_values
|
||||
)
|
||||
|
||||
def _store_rejected_events_txn(self, txn, events_and_contexts):
|
||||
"""Add rows to the 'rejections' table for received events which were
|
||||
rejected
|
||||
|
|
@ -987,29 +1011,6 @@ class PersistEventsStore:
|
|||
txn, [event for event, _ in events_and_contexts]
|
||||
)
|
||||
|
||||
state_events_and_contexts = [
|
||||
ec for ec in events_and_contexts if ec[0].is_state()
|
||||
]
|
||||
|
||||
state_values = []
|
||||
for event, context in state_events_and_contexts:
|
||||
vals = {
|
||||
"event_id": event.event_id,
|
||||
"room_id": event.room_id,
|
||||
"type": event.type,
|
||||
"state_key": event.state_key,
|
||||
}
|
||||
|
||||
# TODO: How does this work with backfilling?
|
||||
if hasattr(event, "replaces_state"):
|
||||
vals["prev_state"] = event.replaces_state
|
||||
|
||||
state_values.append(vals)
|
||||
|
||||
self.db_pool.simple_insert_many_txn(
|
||||
txn, table="state_events", values=state_values
|
||||
)
|
||||
|
||||
# Prefill the event cache
|
||||
self._add_to_cache(txn, events_and_contexts)
|
||||
|
||||
|
|
|
|||
|
|
@ -14,10 +14,15 @@
|
|||
# limitations under the License.
|
||||
|
||||
import logging
|
||||
from typing import List, Tuple
|
||||
|
||||
from synapse.api.constants import EventContentFields
|
||||
from synapse.api.room_versions import KNOWN_ROOM_VERSIONS
|
||||
from synapse.events import make_event_from_dict
|
||||
from synapse.storage._base import SQLBaseStore, db_to_json, make_in_list_sql_clause
|
||||
from synapse.storage.database import DatabasePool
|
||||
from synapse.storage.types import Cursor
|
||||
from synapse.types import JsonDict
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -99,6 +104,10 @@ class EventsBackgroundUpdatesStore(SQLBaseStore):
|
|||
columns=["user_id", "created_ts"],
|
||||
)
|
||||
|
||||
self.db_pool.updates.register_background_update_handler(
|
||||
"rejected_events_metadata", self._rejected_events_metadata,
|
||||
)
|
||||
|
||||
async def _background_reindex_fields_sender(self, progress, batch_size):
|
||||
target_min_stream_id = progress["target_min_stream_id_inclusive"]
|
||||
max_stream_id = progress["max_stream_id_exclusive"]
|
||||
|
|
@ -582,3 +591,118 @@ class EventsBackgroundUpdatesStore(SQLBaseStore):
|
|||
await self.db_pool.updates._end_background_update("event_store_labels")
|
||||
|
||||
return num_rows
|
||||
|
||||
async def _rejected_events_metadata(self, progress: dict, batch_size: int) -> int:
|
||||
"""Adds rejected events to the `state_events` and `event_auth` metadata
|
||||
tables.
|
||||
"""
|
||||
|
||||
last_event_id = progress.get("last_event_id", "")
|
||||
|
||||
def get_rejected_events(
|
||||
txn: Cursor,
|
||||
) -> List[Tuple[str, str, JsonDict, bool, bool]]:
|
||||
# Fetch rejected event json, their room version and whether we have
|
||||
# inserted them into the state_events or auth_events tables.
|
||||
#
|
||||
# Note we can assume that events that don't have a corresponding
|
||||
# room version are V1 rooms.
|
||||
sql = """
|
||||
SELECT DISTINCT
|
||||
event_id,
|
||||
COALESCE(room_version, '1'),
|
||||
json,
|
||||
state_events.event_id IS NOT NULL,
|
||||
event_auth.event_id IS NOT NULL
|
||||
FROM rejections
|
||||
INNER JOIN event_json USING (event_id)
|
||||
LEFT JOIN rooms USING (room_id)
|
||||
LEFT JOIN state_events USING (event_id)
|
||||
LEFT JOIN event_auth USING (event_id)
|
||||
WHERE event_id > ?
|
||||
ORDER BY event_id
|
||||
LIMIT ?
|
||||
"""
|
||||
|
||||
txn.execute(sql, (last_event_id, batch_size,))
|
||||
|
||||
return [(row[0], row[1], db_to_json(row[2]), row[3], row[4]) for row in txn] # type: ignore
|
||||
|
||||
results = await self.db_pool.runInteraction(
|
||||
desc="_rejected_events_metadata_get", func=get_rejected_events
|
||||
)
|
||||
|
||||
if not results:
|
||||
await self.db_pool.updates._end_background_update(
|
||||
"rejected_events_metadata"
|
||||
)
|
||||
return 0
|
||||
|
||||
state_events = []
|
||||
auth_events = []
|
||||
for event_id, room_version, event_json, has_state, has_event_auth in results:
|
||||
last_event_id = event_id
|
||||
|
||||
if has_state and has_event_auth:
|
||||
continue
|
||||
|
||||
room_version_obj = KNOWN_ROOM_VERSIONS.get(room_version)
|
||||
if not room_version_obj:
|
||||
# We no longer support this room version, so we just ignore the
|
||||
# events entirely.
|
||||
logger.info(
|
||||
"Ignoring event with unknown room version %r: %r",
|
||||
room_version,
|
||||
event_id,
|
||||
)
|
||||
continue
|
||||
|
||||
event = make_event_from_dict(event_json, room_version_obj)
|
||||
|
||||
if not event.is_state():
|
||||
continue
|
||||
|
||||
if not has_state:
|
||||
state_events.append(
|
||||
{
|
||||
"event_id": event.event_id,
|
||||
"room_id": event.room_id,
|
||||
"type": event.type,
|
||||
"state_key": event.state_key,
|
||||
}
|
||||
)
|
||||
|
||||
if not has_event_auth:
|
||||
for auth_id in event.auth_event_ids():
|
||||
auth_events.append(
|
||||
{
|
||||
"room_id": event.room_id,
|
||||
"event_id": event.event_id,
|
||||
"auth_id": auth_id,
|
||||
}
|
||||
)
|
||||
|
||||
if state_events:
|
||||
await self.db_pool.simple_insert_many(
|
||||
table="state_events",
|
||||
values=state_events,
|
||||
desc="_rejected_events_metadata_state_events",
|
||||
)
|
||||
|
||||
if auth_events:
|
||||
await self.db_pool.simple_insert_many(
|
||||
table="event_auth",
|
||||
values=auth_events,
|
||||
desc="_rejected_events_metadata_event_auth",
|
||||
)
|
||||
|
||||
await self.db_pool.updates._background_update_progress(
|
||||
"rejected_events_metadata", {"last_event_id": last_event_id}
|
||||
)
|
||||
|
||||
if len(results) < batch_size:
|
||||
await self.db_pool.updates._end_background_update(
|
||||
"rejected_events_metadata"
|
||||
)
|
||||
|
||||
return len(results)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,17 @@
|
|||
/* Copyright 2020 The Matrix.org Foundation C.I.C
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
INSERT INTO background_updates (ordering, update_name, progress_json) VALUES
|
||||
(5828, 'rejected_events_metadata', '{}');
|
||||
Loading…
Reference in New Issue