From b2be8cec393b2f37eefe5cf19c86e26575664ad6 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Mon, 19 Jul 2021 23:13:33 -0500 Subject: [PATCH 1/3] Switch to chunk events so we can auth via power_levels Previously, we were using `content.chunk_id` to connect one chunk to another. But these events can be from any `sender` and we can't tell who should be able to send historical events. We know we only want the application service to do it but these events have the sender of a real historical message, not the application service user ID as the sender. Other federated homeservers also have no indicator which senders are an application service on the originating homeserver. So we want to auth all of the MSC2716 events via power_levels and have them be sent by the application service with proper PL levels in the room. --- synapse/api/constants.py | 6 ++++-- synapse/rest/client/v1/room.py | 17 +++++++++++++---- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/synapse/api/constants.py b/synapse/api/constants.py index 78364100e1..fba96822af 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -120,6 +120,7 @@ class EventTypes: SpaceParent = "m.space.parent" MSC2716_INSERTION = "org.matrix.msc2716.insertion" + MSC2716_CHUNK = "org.matrix.msc2716.chunk" MSC2716_MARKER = "org.matrix.msc2716.marker" @@ -190,9 +191,10 @@ class EventContentFields: # Used on normal messages to indicate they were historically imported after the fact MSC2716_HISTORICAL = "org.matrix.msc2716.historical" - # For "insertion" events + # For "insertion" events to indicate what the next chunk ID should be in + # order to connect to it MSC2716_NEXT_CHUNK_ID = "org.matrix.msc2716.next_chunk_id" - # Used on normal message events to indicate where the chunk connects to + # Used on "chunk" events to indicate which insertion event it connects to MSC2716_CHUNK_ID = "org.matrix.msc2716.chunk_id" # For "marker" events MSC2716_MARKER_INSERTION = "org.matrix.msc2716.marker.insertion" diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index ebf4e32230..d4e16560fa 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -553,9 +553,18 @@ class RoomBatchSendEventRestServlet(TransactionRestServlet): ] # Connect this current chunk to the insertion event from the previous chunk - last_event_in_chunk["content"][ - EventContentFields.MSC2716_CHUNK_ID - ] = chunk_id_to_connect_to + chunk_event = { + "type": EventTypes.MSC2716_CHUNK, + "sender": requester.user.to_string(), + "room_id": room_id, + "content": {EventContentFields.MSC2716_CHUNK_ID: chunk_id_to_connect_to}, + # Since the chunk event is put at the end of the chunk, + # where the newest-in-time event is, copy the origin_server_ts from + # the last event we're inserting + "origin_server_ts": last_event_in_chunk["origin_server_ts"], + } + # Add the chunk event to the end of the chunk (newest-in-time) + events_to_create.append(chunk_event) # Add an "insertion" event to the start of each chunk (next to the oldest-in-time # event in the chunk) so the next chunk can be connected to this one. @@ -567,7 +576,7 @@ class RoomBatchSendEventRestServlet(TransactionRestServlet): # the first event we're inserting origin_server_ts=events_to_create[0]["origin_server_ts"], ) - # Prepend the insertion event to the start of the chunk + # Prepend the insertion event to the start of the chunk (oldest-in-time) events_to_create = [insertion_event] + events_to_create event_ids = [] From 04a29fed4700637a90b04d7bedded3584605af0b Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 20 Jul 2021 00:48:08 -0500 Subject: [PATCH 2/3] Switch to chunk events for federation --- .../databases/main/event_federation.py | 2 +- synapse/storage/databases/main/events.py | 22 +++++++++++-------- .../delta/61/01insertion_event_lookups.sql | 6 ++--- 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/synapse/storage/databases/main/event_federation.py b/synapse/storage/databases/main/event_federation.py index 37b99c7f81..baef3e0072 100644 --- a/synapse/storage/databases/main/event_federation.py +++ b/synapse/storage/databases/main/event_federation.py @@ -962,7 +962,7 @@ class EventFederationWorkerStore(EventsWorkerStore, SignatureWorkerStore, SQLBas chunk_connection_query = """ SELECT e.depth, c.event_id FROM insertion_events AS i /* Find the chunk that connects to the given insertion event */ - INNER JOIN chunk_edges AS c + INNER JOIN chunk_events AS c ON i.next_chunk_id = c.chunk_id /* Get the depth of the chunk start event from the events table */ INNER JOIN events AS e USING (event_id) diff --git a/synapse/storage/databases/main/events.py b/synapse/storage/databases/main/events.py index 081d79f1ac..466aa4e8d8 100644 --- a/synapse/storage/databases/main/events.py +++ b/synapse/storage/databases/main/events.py @@ -1505,7 +1505,7 @@ class PersistEventsStore: self._handle_event_relations(txn, event) self._handle_insertion_event(txn, event) - self._handle_chunk_id(txn, event) + self._handle_chunk_event(txn, event) # Store the labels for this event. labels = event.content.get(EventContentFields.LABELS) @@ -1804,26 +1804,30 @@ class PersistEventsStore: }, ) - def _handle_chunk_id(self, txn: LoggingTransaction, event: EventBase): - """Handles inserting the chunk edges/connections between the event at the - start of a chunk and an insertion event. Part of MSC2716. + def _handle_chunk_event(self, txn: LoggingTransaction, event: EventBase): + """Handles inserting the chunk edges/connections between the chunk event + and an insertion event. Part of MSC2716. Args: txn: The database transaction object event: The event to process """ - chunk_id = event.content.get(EventContentFields.MSC2716_CHUNK_ID) - if chunk_id is None: - # No chunk connection to persist + if event.type != EventTypes.MSC2716_CHUNK: + # Not a chunk event return - logger.debug("_handle_chunk_id %s %s", chunk_id, event) + chunk_id = event.content.get(EventContentFields.MSC2716_CHUNK_ID) + if chunk_id is None: + # Invalid chunk event without a chunk ID + return + + logger.debug("_handle_chunk_event chunk_id=%s %s", chunk_id, event) # Keep track of the insertion event and the chunk ID self.db_pool.simple_insert_txn( txn, - table="chunk_edges", + table="chunk_events", values={ "event_id": event.event_id, "room_id": event.room_id, diff --git a/synapse/storage/schema/main/delta/61/01insertion_event_lookups.sql b/synapse/storage/schema/main/delta/61/01insertion_event_lookups.sql index 9f00b037a4..70c05b1b58 100644 --- a/synapse/storage/schema/main/delta/61/01insertion_event_lookups.sql +++ b/synapse/storage/schema/main/delta/61/01insertion_event_lookups.sql @@ -41,12 +41,12 @@ CREATE INDEX IF NOT EXISTS insertion_event_edges_event_id ON insertion_event_edg CREATE INDEX IF NOT EXISTS insertion_event_edges_insertion_prev_event_id ON insertion_event_edges(insertion_prev_event_id); -- Add a table that keeps track of how each chunk is labeled. The chunks are --- connected together based insertion points `next_chunk_id`. -CREATE TABLE IF NOT EXISTS chunk_edges( +-- connected together based on an insertion events `next_chunk_id`. +CREATE TABLE IF NOT EXISTS chunk_events( event_id TEXT NOT NULL, room_id TEXT NOT NULL, chunk_id TEXT NOT NULL, UNIQUE (event_id) ); -CREATE INDEX IF NOT EXISTS chunk_edges_chunk_id ON chunk_edges(chunk_id); +CREATE INDEX IF NOT EXISTS chunk_events_chunk_id ON chunk_events(chunk_id); From 258fa57d7ea956c1aedf94d72d1273a098dfde73 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 20 Jul 2021 01:41:44 -0500 Subject: [PATCH 3/3] Add unstable room version to support new historical PL --- synapse/api/room_versions.py | 27 +++++++++++++++++++++++++ synapse/event_auth.py | 39 ++++++++++++++++++++++++++++++++++++ synapse/events/utils.py | 3 +++ synapse/handlers/room.py | 1 + 4 files changed, 70 insertions(+) diff --git a/synapse/api/room_versions.py b/synapse/api/room_versions.py index f6c1c97b40..e9491065c8 100644 --- a/synapse/api/room_versions.py +++ b/synapse/api/room_versions.py @@ -73,6 +73,9 @@ class RoomVersion: # MSC2403: Allows join_rules to be set to 'knock', changes auth rules to allow sending # m.room.membership event with membership 'knock'. msc2403_knocking = attr.ib(type=bool) + # MSC2716: Adds m.room.power_levels -> content.historical field to control + # whether "insertion", "chunk", "marker" events can be sent + msc2716_historical = attr.ib(type=bool) class RoomVersions: @@ -88,6 +91,7 @@ class RoomVersions: msc2176_redaction_rules=False, msc3083_join_rules=False, msc2403_knocking=False, + msc2716_historical=False, ) V2 = RoomVersion( "2", @@ -101,6 +105,7 @@ class RoomVersions: msc2176_redaction_rules=False, msc3083_join_rules=False, msc2403_knocking=False, + msc2716_historical=False, ) V3 = RoomVersion( "3", @@ -114,6 +119,7 @@ class RoomVersions: msc2176_redaction_rules=False, msc3083_join_rules=False, msc2403_knocking=False, + msc2716_historical=False, ) V4 = RoomVersion( "4", @@ -127,6 +133,7 @@ class RoomVersions: msc2176_redaction_rules=False, msc3083_join_rules=False, msc2403_knocking=False, + msc2716_historical=False, ) V5 = RoomVersion( "5", @@ -140,6 +147,7 @@ class RoomVersions: msc2176_redaction_rules=False, msc3083_join_rules=False, msc2403_knocking=False, + msc2716_historical=False, ) V6 = RoomVersion( "6", @@ -153,6 +161,7 @@ class RoomVersions: msc2176_redaction_rules=False, msc3083_join_rules=False, msc2403_knocking=False, + msc2716_historical=False, ) MSC2176 = RoomVersion( "org.matrix.msc2176", @@ -166,6 +175,7 @@ class RoomVersions: msc2176_redaction_rules=True, msc3083_join_rules=False, msc2403_knocking=False, + msc2716_historical=False, ) MSC3083 = RoomVersion( "org.matrix.msc3083", @@ -179,6 +189,7 @@ class RoomVersions: msc2176_redaction_rules=False, msc3083_join_rules=True, msc2403_knocking=False, + msc2716_historical=False, ) V7 = RoomVersion( "7", @@ -192,6 +203,21 @@ class RoomVersions: msc2176_redaction_rules=False, msc3083_join_rules=False, msc2403_knocking=True, + msc2716_historical=False, + ) + MSC2716 = RoomVersion( + "org.matrix.msc2716", + RoomDisposition.STABLE, + EventFormatVersions.V3, + StateResolutionVersions.V2, + enforce_key_validity=True, + special_case_aliases_auth=False, + strict_canonicaljson=True, + limit_notifications_power_levels=True, + msc2176_redaction_rules=False, + msc3083_join_rules=False, + msc2403_knocking=True, + msc2716_historical=True, ) @@ -207,6 +233,7 @@ KNOWN_ROOM_VERSIONS = { RoomVersions.MSC2176, RoomVersions.MSC3083, RoomVersions.V7, + RoomVersions.MSC2716, ) # Note that we do not include MSC2043 here unless it is enabled in the config. } # type: Dict[str, RoomVersion] diff --git a/synapse/event_auth.py b/synapse/event_auth.py index 89bcf81515..f2df0f0b7a 100644 --- a/synapse/event_auth.py +++ b/synapse/event_auth.py @@ -190,6 +190,13 @@ def check( if event.type == EventTypes.Redaction: check_redaction(room_version_obj, event, auth_events) + if ( + event.type == EventTypes.MSC2716_INSERTION + or event.type == EventTypes.MSC2716_CHUNK + or event.type == EventTypes.MSC2716_MARKER + ): + check_historical(room_version_obj, event, auth_events) + logger.debug("Allowing! %s", event) @@ -501,6 +508,38 @@ def check_redaction( raise AuthError(403, "You don't have permission to redact events") +def check_historical( + room_version_obj: RoomVersion, + event: EventBase, + auth_events: StateMap[EventBase], +) -> None: + """Check whether the event sender is allowed to send historical related + events like "insertion", "chunk", and "marker". + + Returns: + None + + Raises: + AuthError if the event sender is not allowed to send historical related events + ("insertion", "chunk", and "marker"). + """ + if not room_version_obj.msc2716_historical: + raise AuthError( + 403, + "Historical events not supported with your room version", + ) + + user_level = get_user_power_level(event.user_id, auth_events) + + historical_level = _get_named_level(auth_events, "historical", 100) + + if user_level < historical_level: + raise AuthError( + 403, + 'You don\'t have permission to send send historical related events ("insertion", "chunk", and "marker")', + ) + + def _check_power_levels( room_version_obj: RoomVersion, event: EventBase, diff --git a/synapse/events/utils.py b/synapse/events/utils.py index ec96999e4e..21637bd0c7 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -124,6 +124,9 @@ def prune_event_dict(room_version: RoomVersion, event_dict: dict) -> dict: if room_version.msc2176_redaction_rules: add_fields("invite") + if room_version.msc2716_historical: + add_fields("historical") + elif event_type == EventTypes.Aliases and room_version.special_case_aliases_auth: add_fields("aliases") elif event_type == EventTypes.RoomHistoryVisibility: diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 579b1b93c5..c3f41d8808 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -955,6 +955,7 @@ class RoomCreationHandler(BaseHandler): "kick": 50, "redact": 50, "invite": 50, + "historical": 100, } # type: JsonDict if config["original_invitees_have_ops"]: