OpenCloud
/
riot-web
mirror of https://github.com/vector-im/riot-web
1
0
Fork 0
Code Issues Releases Wiki Activity

sliding-sync: spider all rooms on the user's account for search (#9514) 

* sliding-sync: spider all rooms on the user's account for search

On startup, slowly accumulate room metadata for all rooms on the
user's account. This is so we can populate the local search cache
with enough data for it to function, obviating the need to have
separate code paths for sliding sync searches.

This will allow spotlight search to work with slow/no network
connectivity, though clicking on the room will still require a
round trip.

This is an explicit request from @ara4n to improve the snapiness
of room searches, despite the unbounded bandwidth costs requesting
all N rooms on the user's account.

* Comments and tweak defaults

* Review comments; remove SS search code

* bugfix: use setListRanges once the list has been set up

If we don't, then we send needless extra data and can cause
bugs because setList will wipe the index->room_id map, which
trips up SlidingRoomListStore.
pull/28788/head^2
kegsay 2022-11-01 10:27:03 +00:00 committed by GitHub
parent 253129e6f2
commit dcf497d013
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 197 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/MatrixClientPeg.ts
View File

@ -249,6 +249,7 @@ class MatrixClientPegClass implements IMatrixClientPeg {
this.matrixClient,
proxyUrl || this.matrixClient.baseUrl,
);
SlidingSyncManager.instance.startSpidering(100, 50); // 100 rooms at a time, 50ms apart
}
// Connect the matrix client to the dispatcher and setting handlers

61
src/SlidingSyncManager.ts
View File

@ -52,7 +52,7 @@ import {
SlidingSync,
} from 'matrix-js-sdk/src/sliding-sync';
import { logger } from "matrix-js-sdk/src/logger";
import { IDeferred, defer } from 'matrix-js-sdk/src/utils';
import { IDeferred, defer, sleep } from 'matrix-js-sdk/src/utils';
// how long to long poll for
const SLIDING_SYNC_TIMEOUT_MS = 20 * 1000;
@ -109,6 +109,9 @@ export class SlidingSyncManager {
public configure(client: MatrixClient, proxyUrl: string): SlidingSync {
this.client = client;
this.listIdToIndex = {};
DEFAULT_ROOM_SUBSCRIPTION_INFO.include_old_rooms.required_state.push(
[EventType.RoomMember, client.getUserId()],
);
this.slidingSync = new SlidingSync(
proxyUrl, [], DEFAULT_ROOM_SUBSCRIPTION_INFO, client, SLIDING_SYNC_TIMEOUT_MS,
);
@ -262,4 +265,60 @@ export class SlidingSyncManager {
}
return roomId;
}
/**
* Retrieve all rooms on the user's account. Used for pre-populating the local search cache.
* Retrieval is gradual over time.
* @param batchSize The number of rooms to return in each request.
* @param gapBetweenRequestsMs The number of milliseconds to wait between requests.
*/
public async startSpidering(batchSize: number, gapBetweenRequestsMs: number) {
await sleep(gapBetweenRequestsMs); // wait a bit as this is called on first render so let's let things load
const listIndex = this.getOrAllocateListIndex(SlidingSyncManager.ListSearch);
let startIndex = batchSize;
let hasMore = true;
let firstTime = true;
while (hasMore) {
const endIndex = startIndex + batchSize-1;
try {
const ranges = [[0, batchSize-1], [startIndex, endIndex]];
if (firstTime) {
await this.slidingSync.setList(listIndex, {
// e.g [0,19] [20,39] then [0,19] [40,59]. We keep [0,20] constantly to ensure
// any changes to the list whilst spidering are caught.
ranges: ranges,
sort: [
"by_recency", // this list isn't shown on the UI so just sorting by timestamp is enough
],
timeline_limit: 0, // we only care about the room details, not messages in the room
required_state: [
[EventType.RoomJoinRules, ""], // the public icon on the room list
[EventType.RoomAvatar, ""], // any room avatar
[EventType.RoomTombstone, ""], // lets JS SDK hide rooms which are dead
[EventType.RoomEncryption, ""], // lets rooms be configured for E2EE correctly
[EventType.RoomCreate, ""], // for isSpaceRoom checks
[EventType.RoomMember, this.client.getUserId()!], // lets the client calculate that we are in fact in the room
],
// we don't include_old_rooms here in an effort to reduce the impact of spidering all rooms
// on the user's account. This means some data in the search dialog results may be inaccurate
// e.g membership of space, but this will be corrected when the user clicks on the room
// as the direct room subscription does include old room iterations.
filters: { // we get spaces via a different list, so filter them out
not_room_types: ["m.space"],
},
});
} else {
await this.slidingSync.setListRanges(listIndex, ranges);
}
// gradually request more over time
await sleep(gapBetweenRequestsMs);
} catch (err) {
// do nothing, as we reject only when we get interrupted but that's fine as the next
// request will include our data
}
hasMore = (endIndex+1) < this.slidingSync.getListData(listIndex)?.joinedCount;
startIndex += batchSize;
firstTime = false;
}
}
}

65
src/components/views/dialogs/spotlight/SpotlightDialog.tsx
View File

@ -91,7 +91,6 @@ import { RoomResultContextMenus } from "./RoomResultContextMenus";
import { RoomContextDetails } from "../../rooms/RoomContextDetails";
import { TooltipOption } from "./TooltipOption";
import { isLocalRoom } from "../../../../utils/localRoom/isLocalRoom";
import { useSlidingSyncRoomSearch } from "../../../../hooks/useSlidingSyncRoomSearch";
import { shouldShowFeedback } from "../../../../utils/Feedback";
import RoomAvatar from "../../avatars/RoomAvatar";
@ -342,43 +341,26 @@ const SpotlightDialog: React.FC<IProps> = ({ initialText = "", initialFilter = n
searchProfileInfo,
searchParams,
);
const isSlidingSyncEnabled = SettingsStore.getValue("feature_sliding_sync");
let {
loading: slidingSyncRoomSearchLoading,
rooms: slidingSyncRooms,
search: searchRoomsServerside,
} = useSlidingSyncRoomSearch();
useDebouncedCallback(isSlidingSyncEnabled, searchRoomsServerside, searchParams);
if (!isSlidingSyncEnabled) {
slidingSyncRoomSearchLoading = false;
}
const possibleResults = useMemo<Result[]>(
() => {
const userResults: IMemberResult[] = [];
let roomResults: IRoomResult[];
let alreadyAddedUserIds: Set<string>;
if (isSlidingSyncEnabled) {
// use the rooms sliding sync returned as the server has already worked it out for us
roomResults = slidingSyncRooms.map(toRoomResult);
} else {
roomResults = findVisibleRooms(cli).map(toRoomResult);
// If we already have a DM with the user we're looking for, we will
// show that DM instead of the user themselves
alreadyAddedUserIds = roomResults.reduce((userIds, result) => {
const userId = DMRoomMap.shared().getUserIdForRoomId(result.room.roomId);
if (!userId) return userIds;
if (result.room.getJoinedMemberCount() > 2) return userIds;
userIds.add(userId);
return userIds;
}, new Set<string>());
for (const user of [...findVisibleRoomMembers(cli), ...users]) {
// Make sure we don't have any user more than once
if (alreadyAddedUserIds.has(user.userId)) continue;
alreadyAddedUserIds.add(user.userId);
const roomResults = findVisibleRooms(cli).map(toRoomResult);
// If we already have a DM with the user we're looking for, we will
// show that DM instead of the user themselves
const alreadyAddedUserIds = roomResults.reduce((userIds, result) => {
const userId = DMRoomMap.shared().getUserIdForRoomId(result.room.roomId);
if (!userId) return userIds;
if (result.room.getJoinedMemberCount() > 2) return userIds;
userIds.add(userId);
return userIds;
}, new Set<string>());
for (const user of [...findVisibleRoomMembers(cli), ...users]) {
// Make sure we don't have any user more than once
if (alreadyAddedUserIds.has(user.userId)) continue;
alreadyAddedUserIds.add(user.userId);
userResults.push(toMemberResult(user));
}
userResults.push(toMemberResult(user));
}
return [
@ -402,7 +384,7 @@ const SpotlightDialog: React.FC<IProps> = ({ initialText = "", initialFilter = n
...publicRooms.map(toPublicRoomResult),
].filter(result => filter === null || result.filter.includes(filter));
},
[cli, users, profile, publicRooms, slidingSyncRooms, isSlidingSyncEnabled, filter],
[cli, users, profile, publicRooms, filter],
);
const results = useMemo<Record<Section, Result[]>>(() => {
@ -421,13 +403,10 @@ const SpotlightDialog: React.FC<IProps> = ({ initialText = "", initialFilter = n
possibleResults.forEach(entry => {
if (isRoomResult(entry)) {
// sliding sync gives the correct rooms in the list so we don't need to filter
if (!isSlidingSyncEnabled) {
if (!entry.room.normalizedName?.includes(normalizedQuery) &&
!entry.room.getCanonicalAlias()?.toLowerCase().includes(lcQuery) &&
!entry.query?.some(q => q.includes(lcQuery))
) return; // bail, does not match query
}
if (!entry.room.normalizedName?.includes(normalizedQuery) &&
!entry.room.getCanonicalAlias()?.toLowerCase().includes(lcQuery) &&
!entry.query?.some(q => q.includes(lcQuery))
) return; // bail, does not match query
} else if (isMemberResult(entry)) {
if (!entry.query?.some(q => q.includes(lcQuery))) return; // bail, does not match query
} else if (isPublicRoomResult(entry)) {
@ -478,7 +457,7 @@ const SpotlightDialog: React.FC<IProps> = ({ initialText = "", initialFilter = n
}
return results;
}, [trimmedQuery, filter, cli, possibleResults, isSlidingSyncEnabled, memberComparator]);
}, [trimmedQuery, filter, cli, possibleResults, memberComparator]);
const numResults = sum(Object.values(results).map(it => it.length));
useWebSearchMetrics(numResults, query.length, true);
@ -1236,7 +1215,7 @@ const SpotlightDialog: React.FC<IProps> = ({ initialText = "", initialFilter = n
aria-label={_t("Search")}
aria-describedby="mx_SpotlightDialog_keyboardPrompt"
/>
{ (publicRoomsLoading || peopleLoading || profileLoading || slidingSyncRoomSearchLoading) && (
{ (publicRoomsLoading || peopleLoading || profileLoading) && (
<Spinner w={24} h={24} />
) }
</div>

114
test/SlidingSyncManager-test.ts Normal file
View File

@ -0,0 +1,114 @@
/*
Copyright 2022 The Matrix.org Foundation C.I.C.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
import { SlidingSync } from 'matrix-js-sdk/src/sliding-sync';
import { mocked } from 'jest-mock';
import { SlidingSyncManager } from '../src/SlidingSyncManager';
import { stubClient } from './test-utils';
jest.mock('matrix-js-sdk/src/sliding-sync');
const MockSlidingSync = <jest.Mock<SlidingSync>><unknown>SlidingSync;
describe('SlidingSyncManager', () => {
let manager: SlidingSyncManager;
let slidingSync: SlidingSync;
beforeEach(() => {
slidingSync = new MockSlidingSync();
manager = new SlidingSyncManager();
manager.configure(stubClient(), "invalid");
manager.slidingSync = slidingSync;
});
describe("startSpidering", () => {
it("requests in batchSizes", async () => {
const gapMs = 1;
const batchSize = 10;
mocked(slidingSync.setList).mockResolvedValue("yep");
mocked(slidingSync.setListRanges).mockResolvedValue("yep");
mocked(slidingSync.getListData).mockImplementation((i) => {
return {
joinedCount: 64,
roomIndexToRoomId: {},
};
});
await manager.startSpidering(batchSize, gapMs);
// we expect calls for 10,19 -> 20,29 -> 30,39 -> 40,49 -> 50,59 -> 60,69
const wantWindows = [
[10, 19], [20, 29], [30, 39], [40, 49], [50, 59], [60, 69],
];
expect(slidingSync.getListData).toBeCalledTimes(wantWindows.length);
expect(slidingSync.setList).toBeCalledTimes(1);
expect(slidingSync.setListRanges).toBeCalledTimes(wantWindows.length-1);
wantWindows.forEach((range, i) => {
if (i === 0) {
expect(slidingSync.setList).toBeCalledWith(
manager.getOrAllocateListIndex(SlidingSyncManager.ListSearch),
expect.objectContaining({
ranges: [[0, batchSize-1], range],
}),
);
return;
}
expect(slidingSync.setListRanges).toBeCalledWith(
manager.getOrAllocateListIndex(SlidingSyncManager.ListSearch),
[[0, batchSize-1], range],
);
});
});
it("handles accounts with zero rooms", async () => {
const gapMs = 1;
const batchSize = 10;
mocked(slidingSync.setList).mockResolvedValue("yep");
mocked(slidingSync.getListData).mockImplementation((i) => {
return {
joinedCount: 0,
roomIndexToRoomId: {},
};
});
await manager.startSpidering(batchSize, gapMs);
expect(slidingSync.getListData).toBeCalledTimes(1);
expect(slidingSync.setList).toBeCalledTimes(1);
expect(slidingSync.setList).toBeCalledWith(
manager.getOrAllocateListIndex(SlidingSyncManager.ListSearch),
expect.objectContaining({
ranges: [[0, batchSize-1], [batchSize, batchSize+batchSize-1]],
}),
);
});
it("continues even when setList rejects", async () => {
const gapMs = 1;
const batchSize = 10;
mocked(slidingSync.setList).mockRejectedValue("narp");
mocked(slidingSync.getListData).mockImplementation((i) => {
return {
joinedCount: 0,
roomIndexToRoomId: {},
};
});
await manager.startSpidering(batchSize, gapMs);
expect(slidingSync.getListData).toBeCalledTimes(1);
expect(slidingSync.setList).toBeCalledTimes(1);
expect(slidingSync.setList).toBeCalledWith(
manager.getOrAllocateListIndex(SlidingSyncManager.ListSearch),
expect.objectContaining({
ranges: [[0, batchSize-1], [batchSize, batchSize+batchSize-1]],
}),
);
});
});
});