diff --git a/src/MatrixClientPeg.ts b/src/MatrixClientPeg.ts index 358a4f3dec..85a76ee4e1 100644 --- a/src/MatrixClientPeg.ts +++ b/src/MatrixClientPeg.ts @@ -249,6 +249,7 @@ class MatrixClientPegClass implements IMatrixClientPeg { this.matrixClient, proxyUrl || this.matrixClient.baseUrl, ); + SlidingSyncManager.instance.startSpidering(100, 50); // 100 rooms at a time, 50ms apart } // Connect the matrix client to the dispatcher and setting handlers diff --git a/src/SlidingSyncManager.ts b/src/SlidingSyncManager.ts index c41e6a78e3..3e79267ee2 100644 --- a/src/SlidingSyncManager.ts +++ b/src/SlidingSyncManager.ts @@ -52,7 +52,7 @@ import { SlidingSync, } from 'matrix-js-sdk/src/sliding-sync'; import { logger } from "matrix-js-sdk/src/logger"; -import { IDeferred, defer } from 'matrix-js-sdk/src/utils'; +import { IDeferred, defer, sleep } from 'matrix-js-sdk/src/utils'; // how long to long poll for const SLIDING_SYNC_TIMEOUT_MS = 20 * 1000; @@ -109,6 +109,9 @@ export class SlidingSyncManager { public configure(client: MatrixClient, proxyUrl: string): SlidingSync { this.client = client; this.listIdToIndex = {}; + DEFAULT_ROOM_SUBSCRIPTION_INFO.include_old_rooms.required_state.push( + [EventType.RoomMember, client.getUserId()], + ); this.slidingSync = new SlidingSync( proxyUrl, [], DEFAULT_ROOM_SUBSCRIPTION_INFO, client, SLIDING_SYNC_TIMEOUT_MS, ); @@ -262,4 +265,60 @@ export class SlidingSyncManager { } return roomId; } + + /** + * Retrieve all rooms on the user's account. Used for pre-populating the local search cache. + * Retrieval is gradual over time. + * @param batchSize The number of rooms to return in each request. + * @param gapBetweenRequestsMs The number of milliseconds to wait between requests. + */ + public async startSpidering(batchSize: number, gapBetweenRequestsMs: number) { + await sleep(gapBetweenRequestsMs); // wait a bit as this is called on first render so let's let things load + const listIndex = this.getOrAllocateListIndex(SlidingSyncManager.ListSearch); + let startIndex = batchSize; + let hasMore = true; + let firstTime = true; + while (hasMore) { + const endIndex = startIndex + batchSize-1; + try { + const ranges = [[0, batchSize-1], [startIndex, endIndex]]; + if (firstTime) { + await this.slidingSync.setList(listIndex, { + // e.g [0,19] [20,39] then [0,19] [40,59]. We keep [0,20] constantly to ensure + // any changes to the list whilst spidering are caught. + ranges: ranges, + sort: [ + "by_recency", // this list isn't shown on the UI so just sorting by timestamp is enough + ], + timeline_limit: 0, // we only care about the room details, not messages in the room + required_state: [ + [EventType.RoomJoinRules, ""], // the public icon on the room list + [EventType.RoomAvatar, ""], // any room avatar + [EventType.RoomTombstone, ""], // lets JS SDK hide rooms which are dead + [EventType.RoomEncryption, ""], // lets rooms be configured for E2EE correctly + [EventType.RoomCreate, ""], // for isSpaceRoom checks + [EventType.RoomMember, this.client.getUserId()!], // lets the client calculate that we are in fact in the room + ], + // we don't include_old_rooms here in an effort to reduce the impact of spidering all rooms + // on the user's account. This means some data in the search dialog results may be inaccurate + // e.g membership of space, but this will be corrected when the user clicks on the room + // as the direct room subscription does include old room iterations. + filters: { // we get spaces via a different list, so filter them out + not_room_types: ["m.space"], + }, + }); + } else { + await this.slidingSync.setListRanges(listIndex, ranges); + } + // gradually request more over time + await sleep(gapBetweenRequestsMs); + } catch (err) { + // do nothing, as we reject only when we get interrupted but that's fine as the next + // request will include our data + } + hasMore = (endIndex+1) < this.slidingSync.getListData(listIndex)?.joinedCount; + startIndex += batchSize; + firstTime = false; + } + } } diff --git a/src/components/views/dialogs/spotlight/SpotlightDialog.tsx b/src/components/views/dialogs/spotlight/SpotlightDialog.tsx index 6703afc087..f000d3bf4b 100644 --- a/src/components/views/dialogs/spotlight/SpotlightDialog.tsx +++ b/src/components/views/dialogs/spotlight/SpotlightDialog.tsx @@ -91,7 +91,6 @@ import { RoomResultContextMenus } from "./RoomResultContextMenus"; import { RoomContextDetails } from "../../rooms/RoomContextDetails"; import { TooltipOption } from "./TooltipOption"; import { isLocalRoom } from "../../../../utils/localRoom/isLocalRoom"; -import { useSlidingSyncRoomSearch } from "../../../../hooks/useSlidingSyncRoomSearch"; import { shouldShowFeedback } from "../../../../utils/Feedback"; import RoomAvatar from "../../avatars/RoomAvatar"; @@ -342,43 +341,26 @@ const SpotlightDialog: React.FC = ({ initialText = "", initialFilter = n searchProfileInfo, searchParams, ); - const isSlidingSyncEnabled = SettingsStore.getValue("feature_sliding_sync"); - let { - loading: slidingSyncRoomSearchLoading, - rooms: slidingSyncRooms, - search: searchRoomsServerside, - } = useSlidingSyncRoomSearch(); - useDebouncedCallback(isSlidingSyncEnabled, searchRoomsServerside, searchParams); - if (!isSlidingSyncEnabled) { - slidingSyncRoomSearchLoading = false; - } const possibleResults = useMemo( () => { const userResults: IMemberResult[] = []; - let roomResults: IRoomResult[]; - let alreadyAddedUserIds: Set; - if (isSlidingSyncEnabled) { - // use the rooms sliding sync returned as the server has already worked it out for us - roomResults = slidingSyncRooms.map(toRoomResult); - } else { - roomResults = findVisibleRooms(cli).map(toRoomResult); - // If we already have a DM with the user we're looking for, we will - // show that DM instead of the user themselves - alreadyAddedUserIds = roomResults.reduce((userIds, result) => { - const userId = DMRoomMap.shared().getUserIdForRoomId(result.room.roomId); - if (!userId) return userIds; - if (result.room.getJoinedMemberCount() > 2) return userIds; - userIds.add(userId); - return userIds; - }, new Set()); - for (const user of [...findVisibleRoomMembers(cli), ...users]) { - // Make sure we don't have any user more than once - if (alreadyAddedUserIds.has(user.userId)) continue; - alreadyAddedUserIds.add(user.userId); + const roomResults = findVisibleRooms(cli).map(toRoomResult); + // If we already have a DM with the user we're looking for, we will + // show that DM instead of the user themselves + const alreadyAddedUserIds = roomResults.reduce((userIds, result) => { + const userId = DMRoomMap.shared().getUserIdForRoomId(result.room.roomId); + if (!userId) return userIds; + if (result.room.getJoinedMemberCount() > 2) return userIds; + userIds.add(userId); + return userIds; + }, new Set()); + for (const user of [...findVisibleRoomMembers(cli), ...users]) { + // Make sure we don't have any user more than once + if (alreadyAddedUserIds.has(user.userId)) continue; + alreadyAddedUserIds.add(user.userId); - userResults.push(toMemberResult(user)); - } + userResults.push(toMemberResult(user)); } return [ @@ -402,7 +384,7 @@ const SpotlightDialog: React.FC = ({ initialText = "", initialFilter = n ...publicRooms.map(toPublicRoomResult), ].filter(result => filter === null || result.filter.includes(filter)); }, - [cli, users, profile, publicRooms, slidingSyncRooms, isSlidingSyncEnabled, filter], + [cli, users, profile, publicRooms, filter], ); const results = useMemo>(() => { @@ -421,13 +403,10 @@ const SpotlightDialog: React.FC = ({ initialText = "", initialFilter = n possibleResults.forEach(entry => { if (isRoomResult(entry)) { - // sliding sync gives the correct rooms in the list so we don't need to filter - if (!isSlidingSyncEnabled) { - if (!entry.room.normalizedName?.includes(normalizedQuery) && - !entry.room.getCanonicalAlias()?.toLowerCase().includes(lcQuery) && - !entry.query?.some(q => q.includes(lcQuery)) - ) return; // bail, does not match query - } + if (!entry.room.normalizedName?.includes(normalizedQuery) && + !entry.room.getCanonicalAlias()?.toLowerCase().includes(lcQuery) && + !entry.query?.some(q => q.includes(lcQuery)) + ) return; // bail, does not match query } else if (isMemberResult(entry)) { if (!entry.query?.some(q => q.includes(lcQuery))) return; // bail, does not match query } else if (isPublicRoomResult(entry)) { @@ -478,7 +457,7 @@ const SpotlightDialog: React.FC = ({ initialText = "", initialFilter = n } return results; - }, [trimmedQuery, filter, cli, possibleResults, isSlidingSyncEnabled, memberComparator]); + }, [trimmedQuery, filter, cli, possibleResults, memberComparator]); const numResults = sum(Object.values(results).map(it => it.length)); useWebSearchMetrics(numResults, query.length, true); @@ -1236,7 +1215,7 @@ const SpotlightDialog: React.FC = ({ initialText = "", initialFilter = n aria-label={_t("Search")} aria-describedby="mx_SpotlightDialog_keyboardPrompt" /> - { (publicRoomsLoading || peopleLoading || profileLoading || slidingSyncRoomSearchLoading) && ( + { (publicRoomsLoading || peopleLoading || profileLoading) && ( ) } diff --git a/test/SlidingSyncManager-test.ts b/test/SlidingSyncManager-test.ts new file mode 100644 index 0000000000..40da54a7d1 --- /dev/null +++ b/test/SlidingSyncManager-test.ts @@ -0,0 +1,114 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { SlidingSync } from 'matrix-js-sdk/src/sliding-sync'; +import { mocked } from 'jest-mock'; + +import { SlidingSyncManager } from '../src/SlidingSyncManager'; +import { stubClient } from './test-utils'; + +jest.mock('matrix-js-sdk/src/sliding-sync'); +const MockSlidingSync = >SlidingSync; + +describe('SlidingSyncManager', () => { + let manager: SlidingSyncManager; + let slidingSync: SlidingSync; + + beforeEach(() => { + slidingSync = new MockSlidingSync(); + manager = new SlidingSyncManager(); + manager.configure(stubClient(), "invalid"); + manager.slidingSync = slidingSync; + }); + + describe("startSpidering", () => { + it("requests in batchSizes", async () => { + const gapMs = 1; + const batchSize = 10; + mocked(slidingSync.setList).mockResolvedValue("yep"); + mocked(slidingSync.setListRanges).mockResolvedValue("yep"); + mocked(slidingSync.getListData).mockImplementation((i) => { + return { + joinedCount: 64, + roomIndexToRoomId: {}, + }; + }); + await manager.startSpidering(batchSize, gapMs); + // we expect calls for 10,19 -> 20,29 -> 30,39 -> 40,49 -> 50,59 -> 60,69 + const wantWindows = [ + [10, 19], [20, 29], [30, 39], [40, 49], [50, 59], [60, 69], + ]; + expect(slidingSync.getListData).toBeCalledTimes(wantWindows.length); + expect(slidingSync.setList).toBeCalledTimes(1); + expect(slidingSync.setListRanges).toBeCalledTimes(wantWindows.length-1); + wantWindows.forEach((range, i) => { + if (i === 0) { + expect(slidingSync.setList).toBeCalledWith( + manager.getOrAllocateListIndex(SlidingSyncManager.ListSearch), + expect.objectContaining({ + ranges: [[0, batchSize-1], range], + }), + ); + return; + } + expect(slidingSync.setListRanges).toBeCalledWith( + manager.getOrAllocateListIndex(SlidingSyncManager.ListSearch), + [[0, batchSize-1], range], + ); + }); + }); + it("handles accounts with zero rooms", async () => { + const gapMs = 1; + const batchSize = 10; + mocked(slidingSync.setList).mockResolvedValue("yep"); + mocked(slidingSync.getListData).mockImplementation((i) => { + return { + joinedCount: 0, + roomIndexToRoomId: {}, + }; + }); + await manager.startSpidering(batchSize, gapMs); + expect(slidingSync.getListData).toBeCalledTimes(1); + expect(slidingSync.setList).toBeCalledTimes(1); + expect(slidingSync.setList).toBeCalledWith( + manager.getOrAllocateListIndex(SlidingSyncManager.ListSearch), + expect.objectContaining({ + ranges: [[0, batchSize-1], [batchSize, batchSize+batchSize-1]], + }), + ); + }); + it("continues even when setList rejects", async () => { + const gapMs = 1; + const batchSize = 10; + mocked(slidingSync.setList).mockRejectedValue("narp"); + mocked(slidingSync.getListData).mockImplementation((i) => { + return { + joinedCount: 0, + roomIndexToRoomId: {}, + }; + }); + await manager.startSpidering(batchSize, gapMs); + expect(slidingSync.getListData).toBeCalledTimes(1); + expect(slidingSync.setList).toBeCalledTimes(1); + expect(slidingSync.setList).toBeCalledWith( + manager.getOrAllocateListIndex(SlidingSyncManager.ListSearch), + expect.objectContaining({ + ranges: [[0, batchSize-1], [batchSize, batchSize+batchSize-1]], + }), + ); + }); + }); +});