sliding sync: add lazy-loading member support (#9530)

* sliding sync: add lazy-loading member support

Also swap to `$ME` constants when referring to our own member event.

* Hook into existing LL logic when showing the MemberList

* Linting

* Use consts in js sdk not react sdk

* Add jest tests

* linting

* Store the room in the test

* Fix up getRoom impl

* Add MemberListStore

* Use the right context in MemberList tests

* Fix RightPanel-test

* Always return members even if we lazy load

* Add MemberListStore tests

* Additional tests

src/SlidingSyncManager.ts

@@ -49,6 +49,9 @@ import { EventType } from 'matrix-js-sdk/src/@types/event';
 import {
     MSC3575Filter,
     MSC3575List,
+    MSC3575_STATE_KEY_LAZY,
+    MSC3575_STATE_KEY_ME,
+    MSC3575_WILDCARD,
     SlidingSync,
 } from 'matrix-js-sdk/src/sliding-sync';
 import { logger } from "matrix-js-sdk/src/logger";
@@ -60,19 +63,35 @@ const SLIDING_SYNC_TIMEOUT_MS = 20 * 1000;
 // the things to fetch when a user clicks on a room
 const DEFAULT_ROOM_SUBSCRIPTION_INFO = {
     timeline_limit: 50,
-    required_state: [
+    // missing required_state which will change depending on the kind of room
-        ["*", "*"], // all events
-    ],
     include_old_rooms: {
         timeline_limit: 0,
         required_state: [ // state needed to handle space navigation and tombstone chains
             [EventType.RoomCreate, ""],
             [EventType.RoomTombstone, ""],
-            [EventType.SpaceChild, "*"],
+            [EventType.SpaceChild, MSC3575_WILDCARD],
-            [EventType.SpaceParent, "*"],
+            [EventType.SpaceParent, MSC3575_WILDCARD],
+            [EventType.RoomMember, MSC3575_STATE_KEY_ME],
         ],
     },
 };
+// lazy load room members so rooms like Matrix HQ don't take forever to load
+const UNENCRYPTED_SUBSCRIPTION_NAME = "unencrypted";
+const UNENCRYPTED_SUBSCRIPTION = Object.assign({
+    required_state: [
+        [MSC3575_WILDCARD, MSC3575_WILDCARD], // all events
+        [EventType.RoomMember, MSC3575_STATE_KEY_ME], // except for m.room.members, get our own membership
+        [EventType.RoomMember, MSC3575_STATE_KEY_LAZY], // ...and lazy load the rest.
+    ],
+}, DEFAULT_ROOM_SUBSCRIPTION_INFO);
+
+// we need all the room members in encrypted rooms because we need to know which users to encrypt
+// messages for.
+const ENCRYPTED_SUBSCRIPTION = Object.assign({
+    required_state: [
+        [MSC3575_WILDCARD, MSC3575_WILDCARD], // all events
+    ],
+}, DEFAULT_ROOM_SUBSCRIPTION_INFO);
 
 export type PartialSlidingSyncRequest = {
     filters?: MSC3575Filter;
@@ -109,12 +128,12 @@ export class SlidingSyncManager {
     public configure(client: MatrixClient, proxyUrl: string): SlidingSync {
         this.client = client;
         this.listIdToIndex = {};
-        DEFAULT_ROOM_SUBSCRIPTION_INFO.include_old_rooms.required_state.push(
+        // by default use the encrypted subscription as that gets everything, which is a safer
-            [EventType.RoomMember, client.getUserId()],
+        // default than potentially missing member events.
-        );
         this.slidingSync = new SlidingSync(
-            proxyUrl, [], DEFAULT_ROOM_SUBSCRIPTION_INFO, client, SLIDING_SYNC_TIMEOUT_MS,
+            proxyUrl, [], ENCRYPTED_SUBSCRIPTION, client, SLIDING_SYNC_TIMEOUT_MS,
         );
+        this.slidingSync.addCustomSubscription(UNENCRYPTED_SUBSCRIPTION_NAME, UNENCRYPTED_SUBSCRIPTION);
         // set the space list
         this.slidingSync.setList(this.getOrAllocateListIndex(SlidingSyncManager.ListSpaces), {
             ranges: [[0, 20]],
@@ -129,18 +148,18 @@ export class SlidingSyncManager {
                 [EventType.RoomTombstone, ""], // lets JS SDK hide rooms which are dead
                 [EventType.RoomEncryption, ""], // lets rooms be configured for E2EE correctly
                 [EventType.RoomCreate, ""], // for isSpaceRoom checks
-                [EventType.SpaceChild, "*"], // all space children
+                [EventType.SpaceChild, MSC3575_WILDCARD], // all space children
-                [EventType.SpaceParent, "*"], // all space parents
+                [EventType.SpaceParent, MSC3575_WILDCARD], // all space parents
-                [EventType.RoomMember, this.client.getUserId()!], // lets the client calculate that we are in fact in the room
+                [EventType.RoomMember, MSC3575_STATE_KEY_ME], // lets the client calculate that we are in fact in the room
             ],
             include_old_rooms: {
                 timeline_limit: 0,
                 required_state: [
                     [EventType.RoomCreate, ""],
                     [EventType.RoomTombstone, ""], // lets JS SDK hide rooms which are dead
-                    [EventType.SpaceChild, "*"], // all space children
+                    [EventType.SpaceChild, MSC3575_WILDCARD], // all space children
-                    [EventType.SpaceParent, "*"], // all space parents
+                    [EventType.SpaceParent, MSC3575_WILDCARD], // all space parents
-                    [EventType.RoomMember, this.client.getUserId()!], // lets the client calculate that we are in fact in the room
+                    [EventType.RoomMember, MSC3575_STATE_KEY_ME], // lets the client calculate that we are in fact in the room
                 ],
             },
             filters: {
@@ -207,16 +226,16 @@ export class SlidingSyncManager {
                     [EventType.RoomTombstone, ""], // lets JS SDK hide rooms which are dead
                     [EventType.RoomEncryption, ""], // lets rooms be configured for E2EE correctly
                     [EventType.RoomCreate, ""], // for isSpaceRoom checks
-                    [EventType.RoomMember, this.client.getUserId()], // lets the client calculate that we are in fact in the room
+                    [EventType.RoomMember, MSC3575_STATE_KEY_ME], // lets the client calculate that we are in fact in the room
                 ],
                 include_old_rooms: {
                     timeline_limit: 0,
                     required_state: [
                         [EventType.RoomCreate, ""],
                         [EventType.RoomTombstone, ""], // lets JS SDK hide rooms which are dead
-                        [EventType.SpaceChild, "*"], // all space children
+                        [EventType.SpaceChild, MSC3575_WILDCARD], // all space children
-                        [EventType.SpaceParent, "*"], // all space parents
+                        [EventType.SpaceParent, MSC3575_WILDCARD], // all space parents
-                        [EventType.RoomMember, this.client.getUserId()!], // lets the client calculate that we are in fact in the room
+                        [EventType.RoomMember, MSC3575_STATE_KEY_ME], // lets the client calculate that we are in fact in the room
                     ],
                 },
             };
@@ -252,9 +271,21 @@ export class SlidingSyncManager {
         } else {
             subscriptions.delete(roomId);
         }
-        logger.log("SlidingSync setRoomVisible:", roomId, visible);
+        const room = this.client.getRoom(roomId);
+        let shouldLazyLoad = !this.client.isRoomEncrypted(roomId);
+        if (!room) {
+            // default to safety: request all state if we can't work it out. This can happen if you
+            // refresh the app whilst viewing a room: we call setRoomVisible before we know anything
+            // about the room.
+            shouldLazyLoad = false;
+        }
+        logger.log("SlidingSync setRoomVisible:", roomId, visible, "shouldLazyLoad:", shouldLazyLoad);
+        if (shouldLazyLoad) {
+            // lazy load this room
+            this.slidingSync.useCustomSubscription(roomId, UNENCRYPTED_SUBSCRIPTION_NAME);
+        }
         const p = this.slidingSync.modifyRoomSubscriptions(subscriptions);
-        if (this.client.getRoom(roomId)) {
+        if (room) {
             return roomId; // we have data already for this room, show immediately e.g it's in a list
         }
         try {
@@ -297,7 +328,7 @@ export class SlidingSyncManager {
                             [EventType.RoomTombstone, ""], // lets JS SDK hide rooms which are dead
                             [EventType.RoomEncryption, ""], // lets rooms be configured for E2EE correctly
                             [EventType.RoomCreate, ""], // for isSpaceRoom checks
-                            [EventType.RoomMember, this.client.getUserId()!], // lets the client calculate that we are in fact in the room
+                            [EventType.RoomMember, MSC3575_STATE_KEY_ME], // lets the client calculate that we are in fact in the room
                         ],
                         // we don't include_old_rooms here in an effort to reduce the impact of spidering all rooms
                         // on the user's account. This means some data in the search dialog results may be inaccurate

src/components/views/rooms/MemberList.tsx

@@ -29,14 +29,12 @@ import { ClientEvent } from "matrix-js-sdk/src/client";
 import { EventType } from "matrix-js-sdk/src/@types/event";
 
 import { _t } from '../../../languageHandler';
-import SdkConfig from '../../../SdkConfig';
 import dis from '../../../dispatcher/dispatcher';
 import { isValid3pidInvite } from "../../../RoomInvite";
 import { MatrixClientPeg } from "../../../MatrixClientPeg";
 import BaseCard from "../right_panel/BaseCard";
 import RoomAvatar from "../avatars/RoomAvatar";
 import RoomName from "../elements/RoomName";
-import SettingsStore from "../../../settings/SettingsStore";
 import TruncatedList from '../elements/TruncatedList';
 import Spinner from "../elements/Spinner";
 import SearchBox from "../../structures/SearchBox";
@@ -47,15 +45,12 @@ import BaseAvatar from '../avatars/BaseAvatar';
 import { shouldShowComponent } from "../../../customisations/helpers/UIComponents";
 import { UIComponent } from "../../../settings/UIFeature";
 import PosthogTrackers from "../../../PosthogTrackers";
+import { SDKContext } from '../../../contexts/SDKContext';
 
 const INITIAL_LOAD_NUM_MEMBERS = 30;
 const INITIAL_LOAD_NUM_INVITED = 5;
 const SHOW_MORE_INCREMENT = 100;
 
-// Regex applied to filter our punctuation in member names before applying sort, to fuzzy it a little
-// matches all ASCII punctuation: !"#$%&'()*+,-./:;<=>?@[\]^_`{|}~
-const SORT_REGEX = /[\x21-\x2F\x3A-\x40\x5B-\x60\x7B-\x7E]+/g;
-
 interface IProps {
     roomId: string;
     searchQuery: string;
@@ -65,7 +60,6 @@ interface IProps {
 
 interface IState {
     loading: boolean;
-    members: Array<RoomMember>;
     filteredJoinedMembers: Array<RoomMember>;
     filteredInvitedMembers: Array<RoomMember | MatrixEvent>;
     canInvite: boolean;
@@ -76,35 +70,16 @@ interface IState {
 export default class MemberList extends React.Component<IProps, IState> {
     private showPresence = true;
     private mounted = false;
-    private collator: Intl.Collator;
-    private sortNames = new Map<RoomMember, string>(); // RoomMember -> sortName
 
-    constructor(props) {
+    static contextType = SDKContext;
+    public context!: React.ContextType<typeof SDKContext>;
+
+    constructor(props: IProps, context: React.ContextType<typeof SDKContext>) {
         super(props);
-
+        this.state = this.getMembersState([], []);
-        const cli = MatrixClientPeg.get();
+        this.showPresence = context.memberListStore.isPresenceEnabled();
-        if (cli.hasLazyLoadMembersEnabled()) {
-            // show an empty list
-            this.state = this.getMembersState([]);
-        } else {
-            this.state = this.getMembersState(this.roomMembers());
-        }
-
-        cli.on(ClientEvent.Room, this.onRoom); // invites & joining after peek
-        const enablePresenceByHsUrl = SdkConfig.get("enable_presence_by_hs_url");
-        const hsUrl = MatrixClientPeg.get().baseUrl;
-        this.showPresence = enablePresenceByHsUrl?.[hsUrl] ?? true;
-    }
-
-    public componentDidMount() {
-        const cli = MatrixClientPeg.get();
         this.mounted = true;
-        if (cli.hasLazyLoadMembersEnabled()) {
+        this.listenForMembersChanges();
-            this.showMembersAccordingToMembershipWithLL();
-            cli.on(RoomEvent.MyMembership, this.onMyMembership);
-        } else {
-            this.listenForMembersChanges();
-        }
     }
 
     private listenForMembersChanges(): void {
@@ -118,6 +93,12 @@ export default class MemberList extends React.Component<IProps, IState> {
         cli.on(UserEvent.LastPresenceTs, this.onUserPresenceChange);
         cli.on(UserEvent.Presence, this.onUserPresenceChange);
         cli.on(UserEvent.CurrentlyActive, this.onUserPresenceChange);
+        cli.on(ClientEvent.Room, this.onRoom); // invites & joining after peek
+        cli.on(RoomEvent.MyMembership, this.onMyMembership);
+    }
+
+    componentDidMount(): void {
+        this.updateListNow(true);
     }
 
     public componentWillUnmount() {
@@ -138,33 +119,6 @@ export default class MemberList extends React.Component<IProps, IState> {
         this.updateList.cancel();
     }
 
-    /**
-     * If lazy loading is enabled, either:
-     * show a spinner and load the members if the user is joined,
-     * or show the members available so far if the user is invited
-     */
-    private async showMembersAccordingToMembershipWithLL(): Promise<void> {
-        const cli = MatrixClientPeg.get();
-        if (cli.hasLazyLoadMembersEnabled()) {
-            const cli = MatrixClientPeg.get();
-            const room = cli.getRoom(this.props.roomId);
-            const membership = room && room.getMyMembership();
-            if (membership === "join") {
-                this.setState({ loading: true });
-                try {
-                    await room.loadMembersIfNeeded();
-                } catch (ex) {/* already logged in RoomView */}
-                if (this.mounted) {
-                    this.setState(this.getMembersState(this.roomMembers()));
-                    this.listenForMembersChanges();
-                }
-            } else {
-                // show the members we already have loaded
-                this.setState(this.getMembersState(this.roomMembers()));
-            }
-        }
-    }
-
     private get canInvite(): boolean {
         const cli = MatrixClientPeg.get();
         const room = cli.getRoom(this.props.roomId);
@@ -175,14 +129,11 @@ export default class MemberList extends React.Component<IProps, IState> {
         );
     }
 
-    private getMembersState(members: Array<RoomMember>): IState {
+    private getMembersState(invitedMembers: Array<RoomMember>, joinedMembers: Array<RoomMember>): IState {
-        // set the state after determining showPresence to make sure it's
-        // taken into account while rendering
         return {
             loading: false,
-            members: members,
+            filteredJoinedMembers: joinedMembers,
-            filteredJoinedMembers: this.filterMembers(members, 'join', this.props.searchQuery),
+            filteredInvitedMembers: invitedMembers,
-            filteredInvitedMembers: this.filterMembers(members, 'invite', this.props.searchQuery),
             canInvite: this.canInvite,
 
             // ideally we'd size this to the page height, but
@@ -209,12 +160,13 @@ export default class MemberList extends React.Component<IProps, IState> {
         // We listen for room events because when we accept an invite
         // we need to wait till the room is fully populated with state
         // before refreshing the member list else we get a stale list.
-        this.showMembersAccordingToMembershipWithLL();
+        this.updateListNow(true);
     };
 
     private onMyMembership = (room: Room, membership: string, oldMembership: string): void => {
-        if (room.roomId === this.props.roomId && membership === "join") {
+        if (room.roomId === this.props.roomId && membership === "join" && oldMembership !== "join") {
-            this.showMembersAccordingToMembershipWithLL();
+            // we just joined the room, load the member list
+            this.updateListNow(true);
         }
     };
 
@@ -239,61 +191,29 @@ export default class MemberList extends React.Component<IProps, IState> {
     };
 
     private updateList = throttle(() => {
-        this.updateListNow();
+        this.updateListNow(false);
     }, 500, { leading: true, trailing: true });
 
-    private updateListNow(): void {
+    private async updateListNow(showLoadingSpinner: boolean): Promise<void> {
-        const members = this.roomMembers();
+        if (!this.mounted) {
-
+            return;
+        }
+        if (showLoadingSpinner) {
+            this.setState({ loading: true });
+        }
+        const { joined, invited } = await this.context.memberListStore.loadMemberList(
+            this.props.roomId, this.props.searchQuery,
+        );
+        if (!this.mounted) {
+            return;
+        }
         this.setState({
             loading: false,
-            members: members,
+            filteredJoinedMembers: joined,
-            filteredJoinedMembers: this.filterMembers(members, 'join', this.props.searchQuery),
+            filteredInvitedMembers: invited,
-            filteredInvitedMembers: this.filterMembers(members, 'invite', this.props.searchQuery),
         });
     }
 
-    private getMembersWithUser(): Array<RoomMember> {
-        if (!this.props.roomId) return [];
-        const cli = MatrixClientPeg.get();
-        const room = cli.getRoom(this.props.roomId);
-        if (!room) return [];
-
-        const allMembers = Object.values(room.currentState.members);
-
-        allMembers.forEach((member) => {
-            // work around a race where you might have a room member object
-            // before the user object exists. This may or may not cause
-            // https://github.com/vector-im/vector-web/issues/186
-            if (!member.user) {
-                member.user = cli.getUser(member.userId);
-            }
-
-            this.sortNames.set(
-                member,
-                (member.name[0] === '@' ? member.name.slice(1) : member.name).replace(SORT_REGEX, ""),
-            );
-
-            // XXX: this user may have no lastPresenceTs value!
-            // the right solution here is to fix the race rather than leave it as 0
-        });
-
-        return allMembers;
-    }
-
-    private roomMembers(): Array<RoomMember> {
-        const allMembers = this.getMembersWithUser();
-        const filteredAndSortedMembers = allMembers.filter((m) => {
-            return (
-                m.membership === 'join' || m.membership === 'invite'
-            );
-        });
-        const language = SettingsStore.getValue("language");
-        this.collator = new Intl.Collator(language, { sensitivity: 'base', ignorePunctuation: false });
-        filteredAndSortedMembers.sort(this.memberSort);
-        return filteredAndSortedMembers;
-    }
-
     private createOverflowTileJoined = (overflowCount: number, totalCount: number): JSX.Element => {
         return this.createOverflowTile(overflowCount, totalCount, this.showMoreJoinedMemberList);
     };
@@ -357,59 +277,14 @@ export default class MemberList extends React.Component<IProps, IState> {
         }
     }
 
-    // returns negative if a comes before b,
+    public componentDidUpdate(prevProps: Readonly<IProps>, prevState: Readonly<IState>, snapshot?: any): void {
-    // returns 0 if a and b are equivalent in ordering
+        if (prevProps.searchQuery !== this.props.searchQuery) {
-    // returns positive if a comes after b.
+            this.updateListNow(false);
-    private memberSort = (memberA: RoomMember, memberB: RoomMember): number => {
-        // order by presence, with "active now" first.
-        // ...and then by power level
-        // ...and then by last active
-        // ...and then alphabetically.
-        // We could tiebreak instead by "last recently spoken in this room" if we wanted to.
-
-        const userA = memberA.user;
-        const userB = memberB.user;
-
-        if (!userA && !userB) return 0;
-        if (userA && !userB) return -1;
-        if (!userA && userB) return 1;
-
-        // First by presence
-        if (this.showPresence) {
-            const convertPresence = (p) => p === 'unavailable' ? 'online' : p;
-            const presenceIndex = p => {
-                const order = ['active', 'online', 'offline'];
-                const idx = order.indexOf(convertPresence(p));
-                return idx === -1 ? order.length : idx; // unknown states at the end
-            };
-
-            const idxA = presenceIndex(userA.currentlyActive ? 'active' : userA.presence);
-            const idxB = presenceIndex(userB.currentlyActive ? 'active' : userB.presence);
-            if (idxA !== idxB) {
-                return idxA - idxB;
-            }
         }
-
+    }
-        // Second by power level
-        if (memberA.powerLevel !== memberB.powerLevel) {
-            return memberB.powerLevel - memberA.powerLevel;
-        }
-
-        // Third by last active
-        if (this.showPresence && userA.getLastActiveTs() !== userB.getLastActiveTs()) {
-            return userB.getLastActiveTs() - userA.getLastActiveTs();
-        }
-
-        // Fourth by name (alphabetical)
-        return this.collator.compare(this.sortNames.get(memberA), this.sortNames.get(memberB));
-    };
 
     private onSearchQueryChanged = (searchQuery: string): void => {
         this.props.onSearchQueryChanged(searchQuery);
-        this.setState({
-            filteredJoinedMembers: this.filterMembers(this.state.members, 'join', searchQuery),
-            filteredInvitedMembers: this.filterMembers(this.state.members, 'invite', searchQuery),
-        });
     };
 
     private onPending3pidInviteClick = (inviteEvent: MatrixEvent): void => {
@@ -419,22 +294,6 @@ export default class MemberList extends React.Component<IProps, IState> {
         });
     };
 
-    private filterMembers(members: Array<RoomMember>, membership: string, query?: string): Array<RoomMember> {
-        return members.filter((m) => {
-            if (query) {
-                query = query.toLowerCase();
-                const matchesName = m.name.toLowerCase().indexOf(query) !== -1;
-                const matchesId = m.userId.toLowerCase().indexOf(query) !== -1;
-
-                if (!matchesName && !matchesId) {
-                    return false;
-                }
-            }
-
-            return m.membership === membership;
-        });
-    }
-
     private getPending3PidInvites(): Array<MatrixEvent> {
         // include 3pid invites (m.room.third_party_invite) state events.
         // The HS may have already converted these into m.room.member invites so

src/contexts/SDKContext.ts

@@ -21,6 +21,7 @@ import defaultDispatcher from "../dispatcher/dispatcher";
 import LegacyCallHandler from "../LegacyCallHandler";
 import { PosthogAnalytics } from "../PosthogAnalytics";
 import { SlidingSyncManager } from "../SlidingSyncManager";
+import { MemberListStore } from "../stores/MemberListStore";
 import { RoomNotificationStateStore } from "../stores/notifications/RoomNotificationStateStore";
 import RightPanelStore from "../stores/right-panel/RightPanelStore";
 import { RoomViewStore } from "../stores/RoomViewStore";
@@ -54,6 +55,7 @@ export class SdkContextClass {
 
     // All protected fields to make it easier to derive test stores
     protected _WidgetPermissionStore?: WidgetPermissionStore;
+    protected _MemberListStore?: MemberListStore;
     protected _RightPanelStore?: RightPanelStore;
     protected _RoomNotificationStateStore?: RoomNotificationStateStore;
     protected _RoomViewStore?: RoomViewStore;
@@ -125,6 +127,12 @@ export class SdkContextClass {
         }
         return this._PosthogAnalytics;
     }
+    public get memberListStore(): MemberListStore {
+        if (!this._MemberListStore) {
+            this._MemberListStore = new MemberListStore(this);
+        }
+        return this._MemberListStore;
+    }
     public get slidingSyncManager(): SlidingSyncManager {
         if (!this._SlidingSyncManager) {
             this._SlidingSyncManager = SlidingSyncManager.instance;

src/stores/MemberListStore.ts

@@ -0,0 +1,261 @@
+/*
+Copyright 2022 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { Room, RoomMember } from "matrix-js-sdk/src/matrix";
+
+import SettingsStore from "../settings/SettingsStore";
+import { SdkContextClass } from "../contexts/SDKContext";
+import SdkConfig from "../SdkConfig";
+
+// Regex applied to filter our punctuation in member names before applying sort, to fuzzy it a little
+// matches all ASCII punctuation: !"#$%&'()*+,-./:;<=>?@[\]^_`{|}~
+const SORT_REGEX = /[\x21-\x2F\x3A-\x40\x5B-\x60\x7B-\x7E]+/g;
+
+/**
+ * A class for storing application state for MemberList.
+ */
+export class MemberListStore {
+    // cache of Display Name -> name to sort based on. This strips out special symbols like @.
+    private readonly sortNames = new Map<string, string>();
+    // list of room IDs that have been lazy loaded
+    private readonly loadedRooms = new Set<string>;
+
+    private collator?: Intl.Collator;
+
+    public constructor(
+        private readonly stores: SdkContextClass,
+    ) {
+    }
+
+    /**
+     * Load the member list. Call this whenever the list may have changed.
+     * @param roomId The room to load the member list in
+     * @param searchQuery Optional search query to filter the list.
+     * @returns A list of filtered and sorted room members, grouped by membership.
+     */
+    public async loadMemberList(
+        roomId: string, searchQuery?: string,
+    ): Promise<Record<"joined" | "invited", RoomMember[]>> {
+        if (!this.stores.client) {
+            return {
+                joined: [],
+                invited: [],
+            };
+        }
+        const language = SettingsStore.getValue("language");
+        this.collator = new Intl.Collator(language, { sensitivity: 'base', ignorePunctuation: false });
+        const members = await this.loadMembers(roomId);
+        // Filter then sort as it's more efficient than sorting tons of members we will just filter out later.
+        // Also sort each group, as there's no point comparing invited/joined users when they aren't in the same list!
+        const membersByMembership = this.filterMembers(members, searchQuery);
+        membersByMembership.joined.sort((a: RoomMember, b: RoomMember) => {
+            return this.sortMembers(a, b);
+        });
+        membersByMembership.invited.sort((a: RoomMember, b: RoomMember) => {
+            return this.sortMembers(a, b);
+        });
+        return {
+            joined: membersByMembership.joined,
+            invited: membersByMembership.invited,
+        };
+    }
+
+    private async loadMembers(roomId: string): Promise<Array<RoomMember>> {
+        const room = this.stores.client!.getRoom(roomId);
+        if (!room) {
+            return [];
+        }
+
+        if (!this.isLazyLoadingEnabled(roomId) || this.loadedRooms.has(roomId)) {
+            // nice and easy, we must already have all the members so just return them.
+            return this.loadMembersInRoom(room);
+        }
+        // lazy loading is enabled. There are two kinds of lazy loading:
+        // - With storage: most members are in indexedDB, we just need a small delta via /members.
+        //   Valid for normal sync in normal windows.
+        // - Without storage: nothing in indexedDB, we need to load all via /members. Valid for
+        //   Sliding Sync and incognito windows (non-Sliding Sync).
+        if (!this.isLazyMemberStorageEnabled()) {
+            // pull straight from the server. Don't use a since token as we don't have earlier deltas
+            // accumulated.
+            room.currentState.markOutOfBandMembersStarted();
+            const response = await this.stores.client!.members(roomId, undefined, "leave");
+            const memberEvents = response.chunk.map(this.stores.client!.getEventMapper());
+            room.currentState.setOutOfBandMembers(memberEvents);
+        } else {
+            // load using traditional lazy loading
+            try {
+                await room.loadMembersIfNeeded();
+            } catch (ex) {/* already logged in RoomView */}
+        }
+        // remember that we have loaded the members so we don't hit /members all the time. We
+        // will forget this on refresh which is fine as we only store the data in-memory.
+        this.loadedRooms.add(roomId);
+        return this.loadMembersInRoom(room);
+    }
+
+    private loadMembersInRoom(room: Room): Array<RoomMember> {
+        const allMembers = Object.values(room.currentState.members);
+        allMembers.forEach((member) => {
+            // work around a race where you might have a room member object
+            // before the user object exists. This may or may not cause
+            // https://github.com/vector-im/vector-web/issues/186
+            if (!member.user) {
+                member.user = this.stores.client!.getUser(member.userId) || undefined;
+            }
+            // XXX: this user may have no lastPresenceTs value!
+            // the right solution here is to fix the race rather than leave it as 0
+        });
+        return allMembers;
+    }
+
+    /**
+     * Check if this room should be lazy loaded. Lazy loading means fetching the member list in
+     * a delayed or incremental fashion. It means the `Room` object doesn't have all the members.
+     * @param roomId The room to check if lazy loading is enabled
+     * @returns True if enabled
+     */
+    private isLazyLoadingEnabled(roomId: string): boolean {
+        if (SettingsStore.getValue("feature_sliding_sync")) {
+            // only unencrypted rooms use lazy loading
+            return !this.stores.client!.isRoomEncrypted(roomId);
+        }
+        return this.stores.client!.hasLazyLoadMembersEnabled();
+    }
+
+    /**
+     * Check if lazy member storage is supported.
+     * @returns True if there is storage for lazy loading members
+     */
+    private isLazyMemberStorageEnabled(): boolean {
+        if (SettingsStore.getValue("feature_sliding_sync")) {
+            return false;
+        }
+        return this.stores.client!.hasLazyLoadMembersEnabled();
+    }
+
+    public isPresenceEnabled(): boolean {
+        if (!this.stores.client) {
+            return true;
+        }
+        const enablePresenceByHsUrl = SdkConfig.get("enable_presence_by_hs_url");
+        return enablePresenceByHsUrl?.[this.stores.client!.baseUrl] ?? true;
+    }
+
+    /**
+     * Filter out members based on an optional search query. Groups by membership state.
+     * @param members The list of members to filter.
+     * @param query The textual query to filter based on.
+     * @returns An object with a list of joined and invited users respectively.
+     */
+    private filterMembers(
+        members: Array<RoomMember>, query?: string,
+    ): Record<"joined" | "invited", RoomMember[]> {
+        const result: Record<"joined" | "invited", RoomMember[]> = {
+            joined: [],
+            invited: [],
+        };
+        members.forEach((m) => {
+            if (m.membership !== "join" && m.membership !== "invite") {
+                return; // bail early for left/banned users
+            }
+            if (query) {
+                query = query.toLowerCase();
+                const matchesName = m.name.toLowerCase().includes(query);
+                const matchesId = m.userId.toLowerCase().includes(query);
+                if (!matchesName && !matchesId) {
+                    return;
+                }
+            }
+            switch (m.membership) {
+                case "join":
+                    result.joined.push(m);
+                    break;
+                case "invite":
+                    result.invited.push(m);
+                    break;
+            }
+        });
+        return result;
+    }
+
+    /**
+     * Sort algorithm for room members.
+     * @param memberA
+     * @param memberB
+     * @returns Negative if A comes before B, 0 if A and B are equivalent, Positive is A comes after B.
+     */
+    private sortMembers(memberA: RoomMember, memberB: RoomMember): number {
+        // order by presence, with "active now" first.
+        // ...and then by power level
+        // ...and then by last active
+        // ...and then alphabetically.
+        // We could tiebreak instead by "last recently spoken in this room" if we wanted to.
+
+        const userA = memberA.user;
+        const userB = memberB.user;
+
+        if (!userA && !userB) return 0;
+        if (userA && !userB) return -1;
+        if (!userA && userB) return 1;
+
+        const showPresence = this.isPresenceEnabled();
+
+        // First by presence
+        if (showPresence) {
+            const convertPresence = (p: string): string => p === 'unavailable' ? 'online' : p;
+            const presenceIndex = (p: string): number => {
+                const order = ['active', 'online', 'offline'];
+                const idx = order.indexOf(convertPresence(p));
+                return idx === -1 ? order.length : idx; // unknown states at the end
+            };
+
+            const idxA = presenceIndex(userA!.currentlyActive ? 'active' : userA!.presence);
+            const idxB = presenceIndex(userB!.currentlyActive ? 'active' : userB!.presence);
+            if (idxA !== idxB) {
+                return idxA - idxB;
+            }
+        }
+
+        // Second by power level
+        if (memberA.powerLevel !== memberB.powerLevel) {
+            return memberB.powerLevel - memberA.powerLevel;
+        }
+
+        // Third by last active
+        if (showPresence && userA!.getLastActiveTs() !== userB!.getLastActiveTs()) {
+            return userB!.getLastActiveTs() - userA!.getLastActiveTs();
+        }
+
+        // Fourth by name (alphabetical)
+        return this.collator!.compare(this.canonicalisedName(memberA.name), this.canonicalisedName(memberB.name));
+    }
+
+    /**
+     * Calculate the canonicalised name for the input name.
+     * @param name The member display name
+     * @returns The name to sort on
+     */
+    private canonicalisedName(name: string): string {
+        let result = this.sortNames.get(name);
+        if (result) {
+            return result;
+        }
+        result = (name[0] === '@' ? name.slice(1) : name).replace(SORT_REGEX, "");
+        this.sortNames.set(name, result);
+        return result;
+    }
+}

test/SlidingSyncManager-test.ts

@@ -16,6 +16,7 @@ limitations under the License.
 
 import { SlidingSync } from 'matrix-js-sdk/src/sliding-sync';
 import { mocked } from 'jest-mock';
+import { MatrixClient, MatrixEvent, Room } from 'matrix-js-sdk/src/matrix';
 
 import { SlidingSyncManager } from '../src/SlidingSyncManager';
 import { stubClient } from './test-utils';
@@ -26,14 +27,57 @@ const MockSlidingSync = <jest.Mock<SlidingSync>><unknown>SlidingSync;
 describe('SlidingSyncManager', () => {
     let manager: SlidingSyncManager;
     let slidingSync: SlidingSync;
+    let client: MatrixClient;
 
     beforeEach(() => {
         slidingSync = new MockSlidingSync();
         manager = new SlidingSyncManager();
-        manager.configure(stubClient(), "invalid");
+        client = stubClient();
+        // by default the client has no rooms: stubClient magically makes rooms annoyingly.
+        mocked(client.getRoom).mockReturnValue(null);
+        manager.configure(client, "invalid");
         manager.slidingSync = slidingSync;
     });
 
+    describe("setRoomVisible", () => {
+        it("adds a subscription for the room", async () => {
+            const roomId = "!room:id";
+            const subs = new Set<string>();
+            mocked(slidingSync.getRoomSubscriptions).mockReturnValue(subs);
+            mocked(slidingSync.modifyRoomSubscriptions).mockResolvedValue("yep");
+            await manager.setRoomVisible(roomId, true);
+            expect(slidingSync.modifyRoomSubscriptions).toBeCalledWith(new Set<string>([roomId]));
+        });
+        it("adds a custom subscription for a lazy-loadable room", async () => {
+            const roomId = "!lazy:id";
+            const room = new Room(roomId, client, client.getUserId());
+            room.getLiveTimeline().initialiseState([
+                new MatrixEvent({
+                    type: "m.room.create",
+                    state_key: "",
+                    event_id: "$abc123",
+                    sender: client.getUserId(),
+                    content: {
+                        creator: client.getUserId(),
+                    },
+                }),
+            ]);
+            mocked(client.getRoom).mockImplementation((r: string): Room => {
+                if (roomId === r) {
+                    return room;
+                }
+                return null;
+            });
+            const subs = new Set<string>();
+            mocked(slidingSync.getRoomSubscriptions).mockReturnValue(subs);
+            mocked(slidingSync.modifyRoomSubscriptions).mockResolvedValue("yep");
+            await manager.setRoomVisible(roomId, true);
+            expect(slidingSync.modifyRoomSubscriptions).toBeCalledWith(new Set<string>([roomId]));
+            // we aren't prescriptive about what the sub name is.
+            expect(slidingSync.useCustomSubscription).toBeCalledWith(roomId, expect.anything());
+        });
+    });
+
     describe("startSpidering", () => {
         it("requests in batchSizes", async () => {
             const gapMs = 1;

test/components/structures/RightPanel-test.tsx

@@ -24,7 +24,7 @@ import { MatrixClient } from "matrix-js-sdk/src/client";
 import _RightPanel from "../../../src/components/structures/RightPanel";
 import { MatrixClientPeg } from "../../../src/MatrixClientPeg";
 import ResizeNotifier from "../../../src/utils/ResizeNotifier";
-import { stubClient, wrapInMatrixClientContext, mkRoom } from "../../test-utils";
+import { stubClient, wrapInMatrixClientContext, mkRoom, wrapInSdkContext } from "../../test-utils";
 import { Action } from "../../../src/dispatcher/actions";
 import dis from "../../../src/dispatcher/dispatcher";
 import DMRoomMap from "../../../src/utils/DMRoomMap";
@@ -35,17 +35,23 @@ import { UPDATE_EVENT } from "../../../src/stores/AsyncStore";
 import { WidgetLayoutStore } from "../../../src/stores/widgets/WidgetLayoutStore";
 import RoomSummaryCard from "../../../src/components/views/right_panel/RoomSummaryCard";
 import MemberList from "../../../src/components/views/rooms/MemberList";
+import { SdkContextClass } from "../../../src/contexts/SDKContext";
 
-const RightPanel = wrapInMatrixClientContext(_RightPanel);
+const RightPanelBase = wrapInMatrixClientContext(_RightPanel);
 
 describe("RightPanel", () => {
     const resizeNotifier = new ResizeNotifier();
 
     let cli: MockedObject<MatrixClient>;
+    let context: SdkContextClass;
+    let RightPanel: React.ComponentType<React.ComponentProps<typeof RightPanelBase>>;
     beforeEach(() => {
         stubClient();
         cli = mocked(MatrixClientPeg.get());
         DMRoomMap.makeShared();
+        context = new SdkContextClass();
+        context.client = cli;
+        RightPanel = wrapInSdkContext(RightPanelBase, context);
     });
 
     afterEach(async () => {

test/components/views/rooms/MemberList-test.tsx

@@ -1,5 +1,6 @@
 /*
 Copyright 2021 Šimon Brandner <simon.bra.ag@gmail.com>
+Copyright 2022 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -26,7 +27,8 @@ import { MatrixClientPeg } from '../../../../src/MatrixClientPeg';
 import * as TestUtils from '../../../test-utils';
 import MemberList from "../../../../src/components/views/rooms/MemberList";
 import MemberTile from '../../../../src/components/views/rooms/MemberTile';
-import MatrixClientContext from "../../../../src/contexts/MatrixClientContext";
+import { SDKContext } from '../../../../src/contexts/SDKContext';
+import { TestSdkContext } from '../../../TestSdkContext';
 
 function generateRoomId() {
     return '!' + Math.random().toString().slice(2, 10) + ':domain';
@@ -116,9 +118,11 @@ describe('MemberList', () => {
         const gatherWrappedRef = (r) => {
             memberList = r;
         };
+        const context = new TestSdkContext();
+        context.client = client;
         root = ReactDOM.render(
             (
-                <MatrixClientContext.Provider value={client}>
+                <SDKContext.Provider value={context}>
                     <MemberList
                         searchQuery=""
                         onClose={jest.fn()}
@@ -126,7 +130,7 @@ describe('MemberList', () => {
                         roomId={memberListRoom.roomId}
                         ref={gatherWrappedRef}
                     />
-                </MatrixClientContext.Provider>
+                </SDKContext.Provider>
             ),
             parentDiv,
         );

test/stores/MemberListStore-test.ts

@@ -0,0 +1,235 @@
+/*
+Copyright 2022 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { mocked } from "jest-mock";
+import { EventType, IContent, MatrixClient, MatrixEvent, Room } from "matrix-js-sdk/src/matrix";
+
+import SdkConfig from "../../src/SdkConfig";
+import SettingsStore from "../../src/settings/SettingsStore";
+import { MemberListStore } from "../../src/stores/MemberListStore";
+import { stubClient } from "../test-utils";
+import { TestSdkContext } from "../TestSdkContext";
+
+describe("MemberListStore", () => {
+    const alice = "@alice:bar";
+    const bob = "@bob:bar";
+    const charlie = "@charlie:bar";
+    const roomId = "!foo:bar";
+    let store: MemberListStore;
+    let client: MatrixClient;
+    let room: Room;
+
+    beforeEach(() => {
+        const context = new TestSdkContext();
+        client = stubClient();
+        client.baseUrl = "https://invalid.base.url.here";
+        context.client = client;
+        store = new MemberListStore(context);
+        // alice is joined to the room.
+        room = new Room(roomId, client, client.getUserId()!);
+        room.currentState.setStateEvents([
+            new MatrixEvent({
+                type: EventType.RoomCreate,
+                state_key: "",
+                content: {
+                    creator: alice,
+                },
+                sender: alice,
+                room_id: roomId,
+                event_id: "$1",
+            }),
+            new MatrixEvent({
+                type: EventType.RoomMember,
+                state_key: alice,
+                content: {
+                    membership: "join",
+                },
+                sender: alice,
+                room_id: roomId,
+                event_id: "$2",
+            }),
+        ]);
+        room.recalculate();
+        mocked(client.getRoom).mockImplementation((r: string): Room | null => {
+            if (r === roomId) {
+                return room;
+            }
+            return null;
+        });
+        SdkConfig.put({
+            enable_presence_by_hs_url: {
+                [client.baseUrl]: false,
+            },
+        });
+    });
+
+    it("loads members in a room", async () => {
+        addMember(room, bob, "invite");
+        addMember(room, charlie, "leave");
+
+        const { invited, joined } = await store.loadMemberList(roomId);
+        expect(invited).toEqual([room.getMember(bob)]);
+        expect(joined).toEqual([room.getMember(alice)]);
+    });
+
+    it("fails gracefully for invalid rooms", async () => {
+        const { invited, joined } = await store.loadMemberList("!idontexist:bar");
+        expect(invited).toEqual([]);
+        expect(joined).toEqual([]);
+    });
+
+    it("sorts by power level", async () => {
+        addMember(room, bob, "join");
+        addMember(room, charlie, "join");
+        setPowerLevels(room, {
+            users: {
+                [alice]: 100,
+                [charlie]: 50,
+            },
+            users_default: 10,
+        });
+
+        const { invited, joined } = await store.loadMemberList(roomId);
+        expect(invited).toEqual([]);
+        expect(joined).toEqual([room.getMember(alice), room.getMember(charlie), room.getMember(bob)]);
+    });
+
+    it("sorts by name if power level is equal", async () => {
+        const doris = "@doris:bar";
+        addMember(room, bob, "join");
+        addMember(room, charlie, "join");
+        setPowerLevels(room, {
+            users_default: 10,
+        });
+
+        let { invited, joined } = await store.loadMemberList(roomId);
+        expect(invited).toEqual([]);
+        expect(joined).toEqual([room.getMember(alice), room.getMember(bob), room.getMember(charlie)]);
+
+        // Ensure it sorts by display name if they are set
+        addMember(room, doris, "join", "AAAAA");
+        ({ invited, joined } = await store.loadMemberList(roomId));
+        expect(invited).toEqual([]);
+        expect(joined).toEqual(
+            [room.getMember(doris), room.getMember(alice), room.getMember(bob), room.getMember(charlie)],
+        );
+    });
+
+    it("filters based on a search query", async () => {
+        const mice = "@mice:bar";
+        const zorro = "@zorro:bar";
+        addMember(room, bob, "join");
+        addMember(room, mice, "join");
+
+        let { invited, joined } = await store.loadMemberList(roomId, "ice");
+        expect(invited).toEqual([]);
+        expect(joined).toEqual([room.getMember(alice), room.getMember(mice)]);
+
+        // Ensure it filters by display name if they are set
+        addMember(room, zorro, "join", "ice ice baby");
+        ({ invited, joined } = await store.loadMemberList(roomId, "ice"));
+        expect(invited).toEqual([]);
+        expect(joined).toEqual([room.getMember(alice), room.getMember(zorro), room.getMember(mice)]);
+    });
+
+    describe("lazy loading", () => {
+        beforeEach(() => {
+            mocked(client.hasLazyLoadMembersEnabled).mockReturnValue(true);
+            room.loadMembersIfNeeded = jest.fn();
+            mocked(room.loadMembersIfNeeded).mockResolvedValue(true);
+        });
+
+        it("calls Room.loadMembersIfNeeded once when enabled", async () => {
+            let { joined } = await store.loadMemberList(roomId);
+            expect(joined).toEqual([room.getMember(alice)]);
+            expect(room.loadMembersIfNeeded).toHaveBeenCalledTimes(1);
+            ({ joined } = await store.loadMemberList(roomId));
+            expect(joined).toEqual([room.getMember(alice)]);
+            expect(room.loadMembersIfNeeded).toHaveBeenCalledTimes(1);
+        });
+    });
+
+    describe("sliding sync", () => {
+        beforeEach(() => {
+            jest.spyOn(SettingsStore, 'getValue').mockImplementation((settingName, roomId, value) => {
+                return settingName === "feature_sliding_sync"; // this is enabled, everything else is disabled.
+            });
+            client.members = jest.fn();
+        });
+
+        it("calls /members when lazy loading", async () => {
+            mocked(client.members).mockResolvedValue({
+                chunk: [
+                    {
+                        type: EventType.RoomMember,
+                        state_key: bob,
+                        content: {
+                            membership: "join",
+                            displayname: "Bob",
+                        },
+                        sender: bob,
+                        room_id: room.roomId,
+                        event_id: "$" + Math.random(),
+                        origin_server_ts: 2,
+                    },
+                ],
+            });
+            const { joined } = await store.loadMemberList(roomId);
+            expect(joined).toEqual([room.getMember(alice), room.getMember(bob)]);
+            expect(client.members).toHaveBeenCalled();
+        });
+
+        it("does not use lazy loading on encrypted rooms", async () => {
+            client.isRoomEncrypted = jest.fn();
+            mocked(client.isRoomEncrypted).mockReturnValue(true);
+
+            const { joined } = await store.loadMemberList(roomId);
+            expect(joined).toEqual([room.getMember(alice)]);
+            expect(client.members).not.toHaveBeenCalled();
+        });
+    });
+});
+
+function addEventToRoom(room: Room, ev: MatrixEvent) {
+    room.getLiveTimeline().addEvent(ev, {
+        toStartOfTimeline: false,
+    });
+}
+
+function setPowerLevels(room: Room, pl: IContent) {
+    addEventToRoom(room, new MatrixEvent({
+        type: EventType.RoomPowerLevels,
+        state_key: "",
+        content: pl,
+        sender: room.getCreator()!,
+        room_id: room.roomId,
+        event_id: "$" + Math.random(),
+    }));
+}
+
+function addMember(room: Room, userId: string, membership: string, displayName?: string) {
+    addEventToRoom(room, new MatrixEvent({
+        type: EventType.RoomMember,
+        state_key: userId,
+        content: {
+            membership: membership,
+            displayname: displayName,
+        },
+        sender: userId,
+        room_id: room.roomId,
+        event_id: "$" + Math.random(),
+    }));
+}