OpenCloud
/
MatrixSynapse
mirror of https://github.com/matrix-org/synapse
1
0
Fork 0
Code Issues Releases Wiki Activity

Support filtering the /messages API by relation type (MSC3874). (#14148) 

Gated behind an experimental configuration flag.
pull/14207/head
Patrick Cloke 2022-10-17 11:32:11 -04:00 committed by GitHub
parent 6b24235142
commit 4283bd1cf9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 212 additions and 177 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
changelog.d/14148.feature Normal file
View File

@ -0,0 +1 @@
Experimental support for [MSC3874](https://github.com/matrix-org/matrix-spec-proposals/pull/3874).

27
synapse/api/filtering.py
View File

@ -36,7 +36,7 @@ from jsonschema import FormatChecker
from synapse.api.constants import EduTypes, EventContentFields
from synapse.api.errors import SynapseError
from synapse.api.presence import UserPresenceState
from synapse.events import EventBase
from synapse.events import EventBase, relation_from_event
from synapse.types import JsonDict, RoomID, UserID
if TYPE_CHECKING:
@ -53,6 +53,12 @@ FILTER_SCHEMA = {
# check types are valid event types
"types": {"type": "array", "items": {"type": "string"}},
"not_types": {"type": "array", "items": {"type": "string"}},
# MSC3874, filtering /messages.
"org.matrix.msc3874.rel_types": {"type": "array", "items": {"type": "string"}},
"org.matrix.msc3874.not_rel_types": {
"type": "array",
"items": {"type": "string"},
},
},
}
@ -334,8 +340,15 @@ class Filter:
self.labels = filter_json.get("org.matrix.labels", None)
self.not_labels = filter_json.get("org.matrix.not_labels", [])
self.related_by_senders = self.filter_json.get("related_by_senders", None)
self.related_by_rel_types = self.filter_json.get("related_by_rel_types", None)
self.related_by_senders = filter_json.get("related_by_senders", None)
self.related_by_rel_types = filter_json.get("related_by_rel_types", None)
# For compatibility with _check_fields.
self.rel_types = None
self.not_rel_types = []
if hs.config.experimental.msc3874_enabled:
self.rel_types = filter_json.get("org.matrix.msc3874.rel_types", None)
self.not_rel_types = filter_json.get("org.matrix.msc3874.not_rel_types", [])
def filters_all_types(self) -> bool:
return "*" in self.not_types
@ -386,11 +399,19 @@ class Filter:
# check if there is a string url field in the content for filtering purposes
labels = content.get(EventContentFields.LABELS, [])
# Check if the event has a relation.
rel_type = None
if isinstance(event, EventBase):
relation = relation_from_event(event)
if relation:
rel_type = relation.rel_type
field_matchers = {
"rooms": lambda v: room_id == v,
"senders": lambda v: sender == v,
"types": lambda v: _matches_wildcard(ev_type, v),
"labels": lambda v: v in labels,
"rel_types": lambda v: rel_type == v,
}
result = self._check_fields(field_matchers)

3
synapse/config/experimental.py
View File

@ -117,3 +117,6 @@ class ExperimentalConfig(Config):
self.msc3882_token_timeout = self.parse_duration(
experimental.get("msc3882_token_timeout", "5m")
)
# MSC3874: Filtering /messages with rel_types / not_rel_types.
self.msc3874_enabled: bool = experimental.get("msc3874_enabled", False)

2
synapse/rest/client/versions.py
View File

@ -114,6 +114,8 @@ class VersionsRestServlet(RestServlet):
"org.matrix.msc3882": self.config.experimental.msc3882_enabled,
# Adds support for remotely enabling/disabling pushers, as per MSC3881
"org.matrix.msc3881": self.config.experimental.msc3881_enabled,
# Adds support for filtering /messages by event relation.
"org.matrix.msc3874": self.config.experimental.msc3874_enabled,
},
},
)

29
synapse/storage/databases/main/stream.py
View File

@ -357,6 +357,24 @@ def filter_to_clause(event_filter: Optional[Filter]) -> Tuple[str, List[str]]:
)
args.extend(event_filter.related_by_rel_types)
if event_filter.rel_types:
clauses.append(
"(%s)"
% " OR ".join(
"event_relation.relation_type = ?" for _ in event_filter.rel_types
)
)
args.extend(event_filter.rel_types)
if event_filter.not_rel_types:
clauses.append(
"((%s) OR event_relation.relation_type IS NULL)"
% " AND ".join(
"event_relation.relation_type != ?" for _ in event_filter.not_rel_types
)
)
args.extend(event_filter.not_rel_types)
return " AND ".join(clauses), args
@ -1278,8 +1296,8 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore):
# Multiple labels could cause the same event to appear multiple times.
needs_distinct = True
# If there is a filter on relation_senders and relation_types join to the
# relations table.
# If there is a relation_senders and relation_types filter join to the
# relations table to get events related to the current event.
if event_filter and (
event_filter.related_by_senders or event_filter.related_by_rel_types
):
@ -1294,6 +1312,13 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore):
LEFT JOIN events AS related_event ON (relation.event_id = related_event.event_id)
"""
# If there is a not_rel_types filter join to the relations table to get
# the event's relation information.
if event_filter and (event_filter.rel_types or event_filter.not_rel_types):
join_clause += """
LEFT JOIN event_relations AS event_relation USING (event_id)
"""
if needs_distinct:
select_keywords += " DISTINCT"

63
tests/api/test_filtering.py
View File

@ -35,6 +35,8 @@ def MockEvent(**kwargs):
kwargs["event_id"] = "fake_event_id"
if "type" not in kwargs:
kwargs["type"] = "fake_type"
if "content" not in kwargs:
kwargs["content"] = {}
return make_event_from_dict(kwargs)
@ -357,6 +359,66 @@ class FilteringTestCase(unittest.HomeserverTestCase):
self.assertTrue(Filter(self.hs, definition)._check(event))
@unittest.override_config({"experimental_features": {"msc3874_enabled": True}})
def test_filter_rel_type(self):
definition = {"org.matrix.msc3874.rel_types": ["m.thread"]}
event = MockEvent(
sender="@foo:bar",
type="m.room.message",
room_id="!secretbase:unknown",
content={},
)
self.assertFalse(Filter(self.hs, definition)._check(event))
event = MockEvent(
sender="@foo:bar",
type="m.room.message",
room_id="!secretbase:unknown",
content={"m.relates_to": {"event_id": "$abc", "rel_type": "m.reference"}},
)
self.assertFalse(Filter(self.hs, definition)._check(event))
event = MockEvent(
sender="@foo:bar",
type="m.room.message",
room_id="!secretbase:unknown",
content={"m.relates_to": {"event_id": "$abc", "rel_type": "m.thread"}},
)
self.assertTrue(Filter(self.hs, definition)._check(event))
@unittest.override_config({"experimental_features": {"msc3874_enabled": True}})
def test_filter_not_rel_type(self):
definition = {"org.matrix.msc3874.not_rel_types": ["m.thread"]}
event = MockEvent(
sender="@foo:bar",
type="m.room.message",
room_id="!secretbase:unknown",
content={"m.relates_to": {"event_id": "$abc", "rel_type": "m.thread"}},
)
self.assertFalse(Filter(self.hs, definition)._check(event))
event = MockEvent(
sender="@foo:bar",
type="m.room.message",
room_id="!secretbase:unknown",
content={},
)
self.assertTrue(Filter(self.hs, definition)._check(event))
event = MockEvent(
sender="@foo:bar",
type="m.room.message",
room_id="!secretbase:unknown",
content={"m.relates_to": {"event_id": "$abc", "rel_type": "m.reference"}},
)
self.assertTrue(Filter(self.hs, definition)._check(event))
def test_filter_presence_match(self):
user_filter_json = {"presence": {"types": ["m.*"]}}
filter_id = self.get_success(
@ -456,7 +518,6 @@ class FilteringTestCase(unittest.HomeserverTestCase):
self.assertEqual(filtered_room_ids, ["!allowed:example.com"])
@unittest.override_config({"experimental_features": {"msc3440_enabled": True}})
def test_filter_relations(self):
events = [
# An event without a relation.

1
tests/rest/client/test_relations.py
View File

@ -1677,7 +1677,6 @@ class RelationRedactionTestCase(BaseRelationsTestCase):
{"chunk": [{"type": "m.reaction", "key": "👍", "count": 1}]},
)
@unittest.override_config({"experimental_features": {"msc3440_enabled": True}})
def test_redact_parent_thread(self) -> None:
"""
Test that thread replies are still available when the root event is redacted.

145
tests/rest/client/test_rooms.py
View File

@ -35,7 +35,6 @@ from synapse.api.constants import (
EventTypes,
Membership,
PublicRoomsFilterFields,
RelationTypes,
RoomTypes,
)
from synapse.api.errors import Codes, HttpResponseException
@ -50,6 +49,7 @@ from synapse.util.stringutils import random_string
from tests import unittest
from tests.http.server._base import make_request_with_cancellation_test
from tests.storage.test_stream import PaginationTestCase
from tests.test_utils import make_awaitable
PATH_PREFIX = b"/_matrix/client/api/v1"
@ -2915,149 +2915,20 @@ class LabelsTestCase(unittest.HomeserverTestCase):
return event_id
class RelationsTestCase(unittest.HomeserverTestCase):
servlets = [
synapse.rest.admin.register_servlets_for_client_rest_resource,
room.register_servlets,
login.register_servlets,
]
def default_config(self) -> Dict[str, Any]:
config = super().default_config()
config["experimental_features"] = {"msc3440_enabled": True}
return config
def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None:
self.user_id = self.register_user("test", "test")
self.tok = self.login("test", "test")
self.room_id = self.helper.create_room_as(self.user_id, tok=self.tok)
self.second_user_id = self.register_user("second", "test")
self.second_tok = self.login("second", "test")
self.helper.join(
room=self.room_id, user=self.second_user_id, tok=self.second_tok
)
self.third_user_id = self.register_user("third", "test")
self.third_tok = self.login("third", "test")
self.helper.join(room=self.room_id, user=self.third_user_id, tok=self.third_tok)
# An initial event with a relation from second user.
res = self.helper.send_event(
room_id=self.room_id,
type=EventTypes.Message,
content={"msgtype": "m.text", "body": "Message 1"},
tok=self.tok,
)
self.event_id_1 = res["event_id"]
self.helper.send_event(
room_id=self.room_id,
type="m.reaction",
content={
"m.relates_to": {
"rel_type": RelationTypes.ANNOTATION,
"event_id": self.event_id_1,
"key": "👍",
}
},
tok=self.second_tok,
)
# Another event with a relation from third user.
res = self.helper.send_event(
room_id=self.room_id,
type=EventTypes.Message,
content={"msgtype": "m.text", "body": "Message 2"},
tok=self.tok,
)
self.event_id_2 = res["event_id"]
self.helper.send_event(
room_id=self.room_id,
type="m.reaction",
content={
"m.relates_to": {
"rel_type": RelationTypes.REFERENCE,
"event_id": self.event_id_2,
}
},
tok=self.third_tok,
)
# An event with no relations.
self.helper.send_event(
room_id=self.room_id,
type=EventTypes.Message,
content={"msgtype": "m.text", "body": "No relations"},
tok=self.tok,
)
def _filter_messages(self, filter: JsonDict) -> List[JsonDict]:
class RelationsTestCase(PaginationTestCase):
def _filter_messages(self, filter: JsonDict) -> List[str]:
"""Make a request to /messages with a filter, returns the chunk of events."""
from_token = self.get_success(
self.from_token.to_string(self.hs.get_datastores().main)
)
channel = self.make_request(
"GET",
"/rooms/%s/messages?filter=%s&dir=b" % (self.room_id, json.dumps(filter)),
f"/rooms/{self.room_id}/messages?filter={json.dumps(filter)}&dir=f&from={from_token}",
access_token=self.tok,
)
self.assertEqual(channel.code, HTTPStatus.OK, channel.result)
return channel.json_body["chunk"]
def test_filter_relation_senders(self) -> None:
# Messages which second user reacted to.
filter = {"related_by_senders": [self.second_user_id]}
chunk = self._filter_messages(filter)
self.assertEqual(len(chunk), 1, chunk)
self.assertEqual(chunk[0]["event_id"], self.event_id_1)
# Messages which third user reacted to.
filter = {"related_by_senders": [self.third_user_id]}
chunk = self._filter_messages(filter)
self.assertEqual(len(chunk), 1, chunk)
self.assertEqual(chunk[0]["event_id"], self.event_id_2)
# Messages which either user reacted to.
filter = {"related_by_senders": [self.second_user_id, self.third_user_id]}
chunk = self._filter_messages(filter)
self.assertEqual(len(chunk), 2, chunk)
self.assertCountEqual(
[c["event_id"] for c in chunk], [self.event_id_1, self.event_id_2]
)
def test_filter_relation_type(self) -> None:
# Messages which have annotations.
filter = {"related_by_rel_types": [RelationTypes.ANNOTATION]}
chunk = self._filter_messages(filter)
self.assertEqual(len(chunk), 1, chunk)
self.assertEqual(chunk[0]["event_id"], self.event_id_1)
# Messages which have references.
filter = {"related_by_rel_types": [RelationTypes.REFERENCE]}
chunk = self._filter_messages(filter)
self.assertEqual(len(chunk), 1, chunk)
self.assertEqual(chunk[0]["event_id"], self.event_id_2)
# Messages which have either annotations or references.
filter = {
"related_by_rel_types": [
RelationTypes.ANNOTATION,
RelationTypes.REFERENCE,
]
}
chunk = self._filter_messages(filter)
self.assertEqual(len(chunk), 2, chunk)
self.assertCountEqual(
[c["event_id"] for c in chunk], [self.event_id_1, self.event_id_2]
)
def test_filter_relation_senders_and_type(self) -> None:
# Messages which second user reacted to.
filter = {
"related_by_senders": [self.second_user_id],
"related_by_rel_types": [RelationTypes.ANNOTATION],
}
chunk = self._filter_messages(filter)
self.assertEqual(len(chunk), 1, chunk)
self.assertEqual(chunk[0]["event_id"], self.event_id_1)
return [ev["event_id"] for ev in channel.json_body["chunk"]]
class ContextTestCase(unittest.HomeserverTestCase):

118
tests/storage/test_stream.py
View File

@ -16,7 +16,6 @@ from typing import List
from synapse.api.constants import EventTypes, RelationTypes
from synapse.api.filtering import Filter
from synapse.events import EventBase
from synapse.rest import admin
from synapse.rest.client import login, room
from synapse.types import JsonDict
@ -40,7 +39,7 @@ class PaginationTestCase(HomeserverTestCase):
def default_config(self):
config = super().default_config()
config["experimental_features"] = {"msc3440_enabled": True}
config["experimental_features"] = {"msc3874_enabled": True}
return config
def prepare(self, reactor, clock, homeserver):
@ -58,6 +57,11 @@ class PaginationTestCase(HomeserverTestCase):
self.third_tok = self.login("third", "test")
self.helper.join(room=self.room_id, user=self.third_user_id, tok=self.third_tok)
# Store a token which is after all the room creation events.
self.from_token = self.get_success(
self.hs.get_event_sources().get_current_token_for_pagination(self.room_id)
)
# An initial event with a relation from second user.
res = self.helper.send_event(
room_id=self.room_id,
@ -66,7 +70,7 @@ class PaginationTestCase(HomeserverTestCase):
tok=self.tok,
)
self.event_id_1 = res["event_id"]
self.helper.send_event(
res = self.helper.send_event(
room_id=self.room_id,
type="m.reaction",
content={
@ -78,6 +82,7 @@ class PaginationTestCase(HomeserverTestCase):
},
tok=self.second_tok,
)
self.event_id_annotation = res["event_id"]
# Another event with a relation from third user.
res = self.helper.send_event(
@ -87,7 +92,7 @@ class PaginationTestCase(HomeserverTestCase):
tok=self.tok,
)
self.event_id_2 = res["event_id"]
self.helper.send_event(
res = self.helper.send_event(
room_id=self.room_id,
type="m.reaction",
content={
@ -98,68 +103,59 @@ class PaginationTestCase(HomeserverTestCase):
},
tok=self.third_tok,
)
self.event_id_reference = res["event_id"]
# An event with no relations.
self.helper.send_event(
res = self.helper.send_event(
room_id=self.room_id,
type=EventTypes.Message,
content={"msgtype": "m.text", "body": "No relations"},
tok=self.tok,
)
self.event_id_none = res["event_id"]
def _filter_messages(self, filter: JsonDict) -> List[EventBase]:
def _filter_messages(self, filter: JsonDict) -> List[str]:
"""Make a request to /messages with a filter, returns the chunk of events."""
from_token = self.get_success(
self.hs.get_event_sources().get_current_token_for_pagination(self.room_id)
)
events, next_key = self.get_success(
self.hs.get_datastores().main.paginate_room_events(
room_id=self.room_id,
from_key=from_token.room_key,
from_key=self.from_token.room_key,
to_key=None,
direction="b",
direction="f",
limit=10,
event_filter=Filter(self.hs, filter),
)
)
return events
return [ev.event_id for ev in events]
def test_filter_relation_senders(self):
# Messages which second user reacted to.
filter = {"related_by_senders": [self.second_user_id]}
chunk = self._filter_messages(filter)
self.assertEqual(len(chunk), 1, chunk)
self.assertEqual(chunk[0].event_id, self.event_id_1)
self.assertEqual(chunk, [self.event_id_1])
# Messages which third user reacted to.
filter = {"related_by_senders": [self.third_user_id]}
chunk = self._filter_messages(filter)
self.assertEqual(len(chunk), 1, chunk)
self.assertEqual(chunk[0].event_id, self.event_id_2)
self.assertEqual(chunk, [self.event_id_2])
# Messages which either user reacted to.
filter = {"related_by_senders": [self.second_user_id, self.third_user_id]}
chunk = self._filter_messages(filter)
self.assertEqual(len(chunk), 2, chunk)
self.assertCountEqual(
[c.event_id for c in chunk], [self.event_id_1, self.event_id_2]
)
self.assertCountEqual(chunk, [self.event_id_1, self.event_id_2])
def test_filter_relation_type(self):
# Messages which have annotations.
filter = {"related_by_rel_types": [RelationTypes.ANNOTATION]}
chunk = self._filter_messages(filter)
self.assertEqual(len(chunk), 1, chunk)
self.assertEqual(chunk[0].event_id, self.event_id_1)
self.assertEqual(chunk, [self.event_id_1])
# Messages which have references.
filter = {"related_by_rel_types": [RelationTypes.REFERENCE]}
chunk = self._filter_messages(filter)
self.assertEqual(len(chunk), 1, chunk)
self.assertEqual(chunk[0].event_id, self.event_id_2)
self.assertEqual(chunk, [self.event_id_2])
# Messages which have either annotations or references.
filter = {
@ -169,10 +165,7 @@ class PaginationTestCase(HomeserverTestCase):
]
}
chunk = self._filter_messages(filter)
self.assertEqual(len(chunk), 2, chunk)
self.assertCountEqual(
[c.event_id for c in chunk], [self.event_id_1, self.event_id_2]
)
self.assertCountEqual(chunk, [self.event_id_1, self.event_id_2])
def test_filter_relation_senders_and_type(self):
# Messages which second user reacted to.
@ -181,8 +174,7 @@ class PaginationTestCase(HomeserverTestCase):
"related_by_rel_types": [RelationTypes.ANNOTATION],
}
chunk = self._filter_messages(filter)
self.assertEqual(len(chunk), 1, chunk)
self.assertEqual(chunk[0].event_id, self.event_id_1)
self.assertEqual(chunk, [self.event_id_1])
def test_duplicate_relation(self):
"""An event should only be returned once if there are multiple relations to it."""
@ -201,5 +193,65 @@ class PaginationTestCase(HomeserverTestCase):
filter = {"related_by_senders": [self.second_user_id]}
chunk = self._filter_messages(filter)
self.assertEqual(len(chunk), 1, chunk)
self.assertEqual(chunk[0].event_id, self.event_id_1)
self.assertEqual(chunk, [self.event_id_1])
def test_filter_rel_types(self) -> None:
# Messages which are annotations.
filter = {"org.matrix.msc3874.rel_types": [RelationTypes.ANNOTATION]}
chunk = self._filter_messages(filter)
self.assertEqual(chunk, [self.event_id_annotation])
# Messages which are references.
filter = {"org.matrix.msc3874.rel_types": [RelationTypes.REFERENCE]}
chunk = self._filter_messages(filter)
self.assertEqual(chunk, [self.event_id_reference])
# Messages which are either annotations or references.
filter = {
"org.matrix.msc3874.rel_types": [
RelationTypes.ANNOTATION,
RelationTypes.REFERENCE,
]
}
chunk = self._filter_messages(filter)
self.assertCountEqual(
chunk,
[self.event_id_annotation, self.event_id_reference],
)
def test_filter_not_rel_types(self) -> None:
# Messages which are not annotations.
filter = {"org.matrix.msc3874.not_rel_types": [RelationTypes.ANNOTATION]}
chunk = self._filter_messages(filter)
self.assertEqual(
chunk,
[
self.event_id_1,
self.event_id_2,
self.event_id_reference,
self.event_id_none,
],
)
# Messages which are not references.
filter = {"org.matrix.msc3874.not_rel_types": [RelationTypes.REFERENCE]}
chunk = self._filter_messages(filter)
self.assertEqual(
chunk,
[
self.event_id_1,
self.event_id_annotation,
self.event_id_2,
self.event_id_none,
],
)
# Messages which are neither annotations or references.
filter = {
"org.matrix.msc3874.not_rel_types": [
RelationTypes.ANNOTATION,
RelationTypes.REFERENCE,
]
}
chunk = self._filter_messages(filter)
self.assertEqual(chunk, [self.event_id_1, self.event_id_2, self.event_id_none])