OpenCloud
/
MatrixSynapse
mirror of https://github.com/matrix-org/synapse
1
0
Fork 0
Code Issues Releases Wiki Activity

Merge branch 'develop' into mv/add-mxid-validation-log

mv/add-mxid-validation-log
Mathieu Velten 2023-08-07 14:10:58 +02:00 committed by GitHub
parent 44f7df09bb ec8499206e
commit 830a29482a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 219 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
changelog.d/15629.feature Normal file
View File

@ -0,0 +1 @@
Scope transaction IDs to devices (implement [MSC3970](https://github.com/matrix-org/matrix-spec-proposals/pull/3970)).

1
changelog.d/16046.bugfix Normal file
View File

@ -0,0 +1 @@
Fix deletion in dehydrated devices v2.

12
poetry.lock generated
View File

@ -1610,13 +1610,13 @@ files = [
[[package]]
name = "phonenumbers"
version = "8.13.14"
version = "8.13.18"
description = "Python version of Google's common library for parsing, formatting, storing and validating international phone numbers."
optional = false
python-versions = "*"
files = [
{file = "phonenumbers-8.13.14-py2.py3-none-any.whl", hash = "sha256:a4b20b6ba7dd402728f5cc8e86e1f29b1a873af45f5381dbee7e3083af497ff6"},
{file = "phonenumbers-8.13.14.tar.gz", hash = "sha256:5fa952b4abf9fccdaf1f130d96114a520c48890d4091b50a064e22c0fdc12dec"},
{file = "phonenumbers-8.13.18-py2.py3-none-any.whl", hash = "sha256:3d802739a22592e4127139349937753dee9b6a20bdd5d56847cd885bdc766b1f"},
{file = "phonenumbers-8.13.18.tar.gz", hash = "sha256:b360c756252805d44b447b5bca6d250cf6bd6c69b6f0f4258f3bfe5ab81bef69"},
]
[[package]]
@ -3052,13 +3052,13 @@ types-urllib3 = "*"
[[package]]
name = "types-setuptools"
version = "68.0.0.0"
version = "68.0.0.3"
description = "Typing stubs for setuptools"
optional = false
python-versions = "*"
files = [
{file = "types-setuptools-68.0.0.0.tar.gz", hash = "sha256:fc958b4123b155ffc069a66d3af5fe6c1f9d0600c35c0c8444b2ab4147112641"},
{file = "types_setuptools-68.0.0.0-py3-none-any.whl", hash = "sha256:cc00e09ba8f535362cbe1ea8b8407d15d14b59c57f4190cceaf61a9e57616446"},
{file = "types-setuptools-68.0.0.3.tar.gz", hash = "sha256:d57ae6076100b5704b3cc869fdefc671e1baf4c2cd6643f84265dfc0b955bf05"},
{file = "types_setuptools-68.0.0.3-py3-none-any.whl", hash = "sha256:fec09e5c18264c5c09351c00be01a34456fb7a88e457abe97401325f84ad9d36"},
]
[[package]]

9
synapse/config/experimental.py
View File

@ -216,12 +216,6 @@ class MSC3861:
("session_lifetime",),
)
if not root.experimental.msc3970_enabled:
raise ConfigError(
"experimental_features.msc3970_enabled must be 'true' when OAuth delegation is enabled",
("experimental_features", "msc3970_enabled"),
)
@attr.s(auto_attribs=True, frozen=True, slots=True)
class MSC3866Config:
@ -397,9 +391,6 @@ class ExperimentalConfig(Config):
"Invalid MSC3861 configuration", ("experimental", "msc3861")
) from exc
# MSC3970: Scope transaction IDs to devices
self.msc3970_enabled = experimental.get("msc3970_enabled", self.msc3861.enabled)
# Check that none of the other config options conflict with MSC3861 when enabled
self.msc3861.check_config_conflicts(self.root)

42
synapse/events/utils.py
View File

@ -394,7 +394,6 @@ def serialize_event(
time_now_ms: int,
*,
config: SerializeEventConfig = _DEFAULT_SERIALIZE_EVENT_CONFIG,
msc3970_enabled: bool = False,
) -> JsonDict:
"""Serialize event for clients
@ -402,8 +401,6 @@ def serialize_event(
e
time_now_ms
config: Event serialization config
msc3970_enabled: Whether MSC3970 is enabled. It changes whether we should
include the `transaction_id` in the event's `unsigned` section.
Returns:
The serialized event dictionary.
@ -429,38 +426,46 @@ def serialize_event(
e.unsigned["redacted_because"],
time_now_ms,
config=config,
msc3970_enabled=msc3970_enabled,
)
# If we have a txn_id saved in the internal_metadata, we should include it in the
# unsigned section of the event if it was sent by the same session as the one
# requesting the event.
txn_id: Optional[str] = getattr(e.internal_metadata, "txn_id", None)
if txn_id is not None and config.requester is not None:
# For the MSC3970 rules to be applied, we *need* to have the device ID in the
# event internal metadata. Since we were not recording them before, if it hasn't
# been recorded, we fallback to the old behaviour.
if (
txn_id is not None
and config.requester is not None
and config.requester.user.to_string() == e.sender
):
# Some events do not have the device ID stored in the internal metadata,
# this includes old events as well as those created by appservice, guests,
# or with tokens minted with the admin API. For those events, fallback
# to using the access token instead.
event_device_id: Optional[str] = getattr(e.internal_metadata, "device_id", None)
if msc3970_enabled and event_device_id is not None:
if event_device_id is not None:
if event_device_id == config.requester.device_id:
d["unsigned"]["transaction_id"] = txn_id
else:
# The pre-MSC3970 behaviour is to only include the transaction ID if the
# event was sent from the same access token. For regular users, we can use
# the access token ID to determine this. For guests, we can't, but since
# each guest only has one access token, we can just check that the event was
# sent by the same user as the one requesting the event.
# Fallback behaviour: only include the transaction ID if the event
# was sent from the same access token.
#
# For regular users, the access token ID can be used to determine this.
# This includes access tokens minted with the admin API.
#
# For guests and appservice users, we can't check the access token ID
# so assume it is the same session.
event_token_id: Optional[int] = getattr(
e.internal_metadata, "token_id", None
)
if config.requester.user.to_string() == e.sender and (
if (
(
event_token_id is not None
and config.requester.access_token_id is not None
and event_token_id == config.requester.access_token_id
)
or config.requester.is_guest
or config.requester.app_service
):
d["unsigned"]["transaction_id"] = txn_id
@ -504,9 +509,6 @@ class EventClientSerializer:
clients.
"""
def __init__(self, *, msc3970_enabled: bool = False):
self._msc3970_enabled = msc3970_enabled
def serialize_event(
self,
event: Union[JsonDict, EventBase],
@ -531,9 +533,7 @@ class EventClientSerializer:
if not isinstance(event, EventBase):
return event
serialized_event = serialize_event(
event, time_now, config=config, msc3970_enabled=self._msc3970_enabled
)
serialized_event = serialize_event(event, time_now, config=config)
# Check if there are any bundled aggregations to include with the event.
if bundle_aggregations:

16
synapse/handlers/device.py
View File

@ -722,6 +722,22 @@ class DeviceHandler(DeviceWorkerHandler):
return {"success": True}
async def delete_dehydrated_device(self, user_id: str, device_id: str) -> None:
"""
Delete a stored dehydrated device.
Args:
user_id: the user_id to delete the device from
device_id: id of the dehydrated device to delete
"""
success = await self.store.remove_dehydrated_device(user_id, device_id)
if not success:
raise errors.NotFoundError()
await self.delete_devices(user_id, [device_id])
await self.store.delete_e2e_keys_by_device(user_id=user_id, device_id=device_id)
@wrap_as_background_process("_handle_new_device_update_async")
async def _handle_new_device_update_async(self) -> None:
"""Called when we have a new local device list update that we need to

12
synapse/handlers/message.py
View File

@ -561,8 +561,6 @@ class EventCreationHandler:
expiry_ms=30 * 60 * 1000,
)
self._msc3970_enabled = hs.config.experimental.msc3970_enabled
async def create_event(
self,
requester: Requester,
@ -897,9 +895,8 @@ class EventCreationHandler:
"""
existing_event_id = None
if self._msc3970_enabled and requester.device_id:
# When MSC3970 is enabled, we lookup for events sent by the same device first,
# and fallback to the old behaviour if none were found.
# According to the spec, transactions are scoped to a user's device ID.
if requester.device_id:
existing_event_id = (
await self.store.get_event_id_from_transaction_id_and_device_id(
room_id,
@ -911,8 +908,9 @@ class EventCreationHandler:
if existing_event_id:
return existing_event_id
# Pre-MSC3970, we looked up for events that were sent by the same session by
# using the access token ID.
# Some requsters don't have device IDs (appservice, guests, and access
# tokens minted with the admin API), fallback to checking the access token
# ID, which should be close enough.
if requester.access_token_id:
existing_event_id = (
await self.store.get_event_id_from_transaction_id_and_token_id(

14
synapse/rest/client/devices.py
View File

@ -513,10 +513,8 @@ class DehydratedDeviceV2Servlet(RestServlet):
if dehydrated_device is not None:
(device_id, device_data) = dehydrated_device
result = await self.device_handler.rehydrate_device(
requester.user.to_string(),
self.auth.get_access_token_from_request(request),
device_id,
await self.device_handler.delete_dehydrated_device(
requester.user.to_string(), device_id
)
result = {"device_id": device_id}
@ -538,6 +536,14 @@ class DehydratedDeviceV2Servlet(RestServlet):
requester = await self.auth.get_user_by_req(request)
user_id = requester.user.to_string()
old_dehydrated_device = await self.device_handler.get_dehydrated_device(user_id)
# if an old device exists, delete it before creating a new one
if old_dehydrated_device:
await self.device_handler.delete_dehydrated_device(
user_id, old_dehydrated_device[0]
)
device_info = submission.dict()
if "device_keys" not in device_info.keys():
raise SynapseError(

12
synapse/rest/client/transactions.py
View File

@ -50,8 +50,6 @@ class HttpTransactionCache:
# for at *LEAST* 30 mins, and at *MOST* 60 mins.
self.cleaner = self.clock.looping_call(self._cleanup, CLEANUP_PERIOD_MS)
self._msc3970_enabled = hs.config.experimental.msc3970_enabled
def _get_transaction_key(self, request: IRequest, requester: Requester) -> Hashable:
"""A helper function which returns a transaction key that can be used
with TransactionCache for idempotent requests.
@ -78,18 +76,20 @@ class HttpTransactionCache:
elif requester.app_service is not None:
return (path, "appservice", requester.app_service.id)
# With MSC3970, we use the user ID and device ID as the transaction key
elif self._msc3970_enabled:
# Use the user ID and device ID as the transaction key.
elif requester.device_id:
assert requester.user, "Requester must have a user"
assert requester.device_id, "Requester must have a device_id"
return (path, "user", requester.user, requester.device_id)
# Otherwise, the pre-MSC3970 behaviour is to use the access token ID
# Some requsters don't have device IDs, these are mostly handled above
# (appservice and guest users), but does not cover access tokens minted
# by the admin API. Use the access token ID instead.
else:
assert (
requester.access_token_id is not None
), "Requester must have an access_token_id"
return (path, "user", requester.access_token_id)
return (path, "user_admin", requester.access_token_id)
def fetch_or_execute_request(
self,

4
synapse/server.py
View File

@ -785,9 +785,7 @@ class HomeServer(metaclass=abc.ABCMeta):
@cache_in_self
def get_event_client_serializer(self) -> EventClientSerializer:
return EventClientSerializer(
msc3970_enabled=self.config.experimental.msc3970_enabled
)
return EventClientSerializer()
@cache_in_self
def get_password_policy_handler(self) -> PasswordPolicyHandler:

15
synapse/storage/databases/main/events.py
View File

@ -127,8 +127,6 @@ class PersistEventsStore:
self._backfill_id_gen: AbstractStreamIdGenerator = self.store._backfill_id_gen
self._stream_id_gen: AbstractStreamIdGenerator = self.store._stream_id_gen
self._msc3970_enabled = hs.config.experimental.msc3970_enabled
@trace
async def _persist_events_and_state_updates(
self,
@ -1012,9 +1010,11 @@ class PersistEventsStore:
)
)
# Pre-MSC3970, we rely on the access_token_id to scope the txn_id for events.
# Since this is an experimental flag, we still store the mapping even if the
# flag is disabled.
# Synapse usually relies on the device_id to scope transactions for events,
# except for users without device IDs (appservice, guests, and access
# tokens minted with the admin API) which use the access token ID instead.
#
# TODO https://github.com/matrix-org/synapse/issues/16042
if to_insert_token_id:
self.db_pool.simple_insert_many_txn(
txn,
@ -1030,10 +1030,7 @@ class PersistEventsStore:
values=to_insert_token_id,
)
# With MSC3970, we rely on the device_id instead to scope the txn_id for events.
# We're only inserting if MSC3970 is *enabled*, because else the pre-MSC3970
# behaviour would allow for a UNIQUE constraint violation on this table
if to_insert_device_id and self._msc3970_enabled:
if to_insert_device_id:
self.db_pool.simple_insert_many_txn(
txn,
table="event_txn_id_device_id",

5
synapse/storage/schema/__init__.py
View File

@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
SCHEMA_VERSION = 79 # remember to update the list below when updating
SCHEMA_VERSION = 80 # remember to update the list below when updating
"""Represents the expectations made by the codebase about the database schema
This should be incremented whenever the codebase changes its requirements on the
@ -110,6 +110,9 @@ Changes in SCHEMA_VERSION = 78
Changes in SCHEMA_VERSION = 79
- Add tables to handle in DB read-write locks.
- Add some mitigations for a painful race between foreground and background updates, cf #15677.
Changes in SCHEMA_VERSION = 80
- The event_txn_id_device_id is always written to for new events.
"""

7
synapse/types/__init__.py
View File

@ -121,11 +121,12 @@ class Requester:
Attributes:
user: id of the user making the request
access_token_id: *ID* of the access token used for this
request, or None if it came via the appservice API or similar
access_token_id: *ID* of the access token used for this request, or
None for appservices, guests, and tokens generated by the admin API
is_guest: True if the user making this request is a guest user
shadow_banned: True if the user making this request has been shadow-banned.
device_id: device_id which was set at authentication time
device_id: device_id which was set at authentication time, or
None for appservices, guests, and tokens generated by the admin API
app_service: the AS requesting on behalf of the user
authenticated_entity: The entity that authenticated when making the request.
This is different to the user_id when an admin user or the server is

139
tests/rest/client/test_devices.py
View File

@ -379,4 +379,141 @@ class DehydratedDeviceTestCase(unittest.HomeserverTestCase):
access_token=token,
shorthand=False,
)
self.assertEqual(channel.code, 404)
self.assertEqual(channel.code, 401)
@unittest.override_config(
{"experimental_features": {"msc2697_enabled": False, "msc3814_enabled": True}}
)
def test_msc3814_dehydrated_device_delete_works(self) -> None:
user = self.register_user("mikey", "pass")
token = self.login(user, "pass", device_id="device1")
content: JsonDict = {
"device_data": {
"algorithm": "m.dehydration.v1.olm",
},
"device_id": "device2",
"initial_device_display_name": "foo bar",
"device_keys": {
"user_id": "@mikey:test",
"device_id": "device2",
"valid_until_ts": "80",
"algorithms": [
"m.olm.curve25519-aes-sha2",
],
"keys": {
"<algorithm>:<device_id>": "<key_base64>",
},
"signatures": {
"<user_id>": {"<algorithm>:<device_id>": "<signature_base64>"}
},
},
}
channel = self.make_request(
"PUT",
"_matrix/client/unstable/org.matrix.msc3814.v1/dehydrated_device",
content=content,
access_token=token,
shorthand=False,
)
self.assertEqual(channel.code, 200)
device_id = channel.json_body.get("device_id")
assert device_id is not None
self.assertIsInstance(device_id, str)
self.assertEqual("device2", device_id)
# ensure that keys were uploaded and available
channel = self.make_request(
"POST",
"/_matrix/client/r0/keys/query",
{
"device_keys": {
user: ["device2"],
},
},
token,
)
self.assertEqual(
channel.json_body["device_keys"][user]["device2"]["keys"],
{
"<algorithm>:<device_id>": "<key_base64>",
},
)
# delete the dehydrated device
channel = self.make_request(
"DELETE",
"_matrix/client/unstable/org.matrix.msc3814.v1/dehydrated_device",
access_token=token,
shorthand=False,
)
self.assertEqual(channel.code, 200)
# ensure that keys are no longer available for deleted device
channel = self.make_request(
"POST",
"/_matrix/client/r0/keys/query",
{
"device_keys": {
user: ["device2"],
},
},
token,
)
self.assertEqual(channel.json_body["device_keys"], {"@mikey:test": {}})
# check that an old device is deleted when user PUTs a new device
# First, create a device
content["device_id"] = "device3"
content["device_keys"]["device_id"] = "device3"
channel = self.make_request(
"PUT",
"_matrix/client/unstable/org.matrix.msc3814.v1/dehydrated_device",
content=content,
access_token=token,
shorthand=False,
)
self.assertEqual(channel.code, 200)
device_id = channel.json_body.get("device_id")
assert device_id is not None
self.assertIsInstance(device_id, str)
self.assertEqual("device3", device_id)
# create a second device without deleting first device
content["device_id"] = "device4"
content["device_keys"]["device_id"] = "device4"
channel = self.make_request(
"PUT",
"_matrix/client/unstable/org.matrix.msc3814.v1/dehydrated_device",
content=content,
access_token=token,
shorthand=False,
)
self.assertEqual(channel.code, 200)
device_id = channel.json_body.get("device_id")
assert device_id is not None
self.assertIsInstance(device_id, str)
self.assertEqual("device4", device_id)
# check that the second device that was created is what is returned when we GET
channel = self.make_request(
"GET",
"_matrix/client/unstable/org.matrix.msc3814.v1/dehydrated_device",
access_token=token,
shorthand=False,
)
self.assertEqual(channel.code, 200)
returned_device_id = channel.json_body["device_id"]
self.assertEqual(returned_device_id, "device4")
# and that if we query the keys for the first device they are not there
channel = self.make_request(
"POST",
"/_matrix/client/r0/keys/query",
{
"device_keys": {
user: ["device3"],
},
},
token,
)
self.assertEqual(channel.json_body["device_keys"], {"@mikey:test": {}})