Fix end-to-end tests for end-to-end encryption verification

Fixes https://github.com/vector-im/riot-web/issues/13226

This isn't the fastest route, but it is a predictable route for the happy path we probably want to test. For example, Alice will already be staring at the DM and could easily accept the verification there, but we probably want to make sure that the toast is present and does the right thing. Similarly, neither of them need to verify that there's green shields everywhere, they should be implied, however an explicit check follows a real user's gaze.

src/components/views/right_panel/UserInfo.js

@@ -1308,8 +1308,9 @@ const BasicUserInfo = ({room, member, groupId, devices, isRoomEncrypted}) => {
         useHasCrossSigningKeys(cli, member, canVerify, setUpdating );
 
     if (canVerify) {
+        // Note: mx_UserInfo_verifyButton is for the end-to-end tests
         verifyButton = (
-            <AccessibleButton className="mx_UserInfo_field" onClick={() => {
+            <AccessibleButton className="mx_UserInfo_field mx_UserInfo_verifyButton" onClick={() => {
                 if (hasCrossSigningKeys) {
                     verifyUser(member);
                 } else {

src/components/views/right_panel/VerificationPanel.js

@@ -123,10 +123,17 @@ export default class VerificationPanel extends React.PureComponent {
             const sasLabel = showQR ?
                 _t("If you can't scan the code above, verify by comparing unique emoji.") :
                 _t("Verify by comparing unique emoji.");
+
+            // Note: mx_VerificationPanel_verifyByEmojiButton is for the end-to-end tests
             sasBlock = <div className="mx_UserInfo_container">
                 <h3>{_t("Verify by emoji")}</h3>
                 <p>{sasLabel}</p>
-                <AccessibleButton disabled={disabled} kind="primary" className="mx_UserInfo_wideButton" onClick={this._startSAS}>
+                <AccessibleButton
+                    disabled={disabled}
+                    kind="primary"
+                    className="mx_UserInfo_wideButton mx_VerificationPanel_verifyByEmojiButton"
+                    onClick={this._startSAS}
+                >
                     {_t("Verify by emoji")}
                 </AccessibleButton>
             </div>;

test/end-to-end-tests/src/scenarios/directory.js

@@ -20,7 +20,7 @@ const join = require('../usecases/join');
 const sendMessage = require('../usecases/send-message');
 const {receiveMessage} = require('../usecases/timeline');
 const {createRoom} = require('../usecases/create-room');
-const changeRoomSettings = require('../usecases/room-settings');
+const {changeRoomSettings} = require('../usecases/room-settings');
 
 module.exports = async function roomDirectoryScenarios(alice, bob) {
     console.log(" creating a public room and join through directory:");

test/end-to-end-tests/src/scenarios/e2e-encryption.js

@@ -1,6 +1,6 @@
 /*
 Copyright 2018 New Vector Ltd
-Copyright 2019 The Matrix.org Foundation C.I.C.
+Copyright 2019, 2020 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -15,42 +15,33 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-// TODO: Update test for cross signing
+const sendMessage = require('../usecases/send-message');
-// https://github.com/vector-im/riot-web/issues/13226
+const acceptInvite = require('../usecases/accept-invite');
+const invite = require('../usecases/invite');
+const {receiveMessage} = require('../usecases/timeline');
+const {createDm} = require('../usecases/create-room');
+const {checkRoomSettings} = require('../usecases/room-settings');
+const {startSasVerifcation, acceptSasVerification} = require('../usecases/verify');
+const assert = require('assert');
 
-module.exports = async function() {
+module.exports = async function e2eEncryptionScenarios(alice, bob) {
-    console.log(" this is supposed to be an e2e test, but it's broken");
+    console.log(" creating an e2e encrypted DM and join through invite:");
+    await createDm(bob, ['@alice:localhost']);
+    await checkRoomSettings(bob, {encryption: true}); // for sanity, should be e2e-by-default
+    await acceptInvite(alice, 'bob');
+    // do sas verifcation
+    bob.log.step(`starts SAS verification with ${alice.username}`);
+    const bobSasPromise = startSasVerifcation(bob, alice.username);
+    const aliceSasPromise = acceptSasVerification(alice, bob.username);
+    // wait in parallel, so they don't deadlock on each other
+    // the logs get a bit messy here, but that's fine enough for debugging (hopefully)
+    const [bobSas, aliceSas] = await Promise.all([bobSasPromise, aliceSasPromise]);
+    assert.deepEqual(bobSas, aliceSas);
+    bob.log.done(`done (match for ${bobSas.join(", ")})`);
+    const aliceMessage = "Guess what I just heard?!";
+    await sendMessage(alice, aliceMessage);
+    await receiveMessage(bob, {sender: "alice", body: aliceMessage, encrypted: true});
+    const bobMessage = "You've got to tell me!";
+    await sendMessage(bob, bobMessage);
+    await receiveMessage(alice, {sender: "bob", body: bobMessage, encrypted: true});
 };
-
-// const sendMessage = require('../usecases/send-message');
-// const acceptInvite = require('../usecases/accept-invite');
-// const invite = require('../usecases/invite');
-// const {receiveMessage} = require('../usecases/timeline');
-// const {createRoom} = require('../usecases/create-room');
-// const changeRoomSettings = require('../usecases/room-settings');
-// const {startSasVerifcation, acceptSasVerification} = require('../usecases/verify');
-// const assert = require('assert');
-//
-// module.exports = async function e2eEncryptionScenarios(alice, bob) {
-//     console.log(" creating an e2e encrypted room and join through invite:");
-//     const room = "secrets";
-//     await createRoom(bob, room);
-//     await changeRoomSettings(bob, {encryption: true});
-//     // await cancelKeyBackup(bob);
-//     await invite(bob, "@alice:localhost");
-//     await acceptInvite(alice, room);
-//     // do sas verifcation
-//     bob.log.step(`starts SAS verification with ${alice.username}`);
-//     const bobSasPromise = startSasVerifcation(bob, alice.username);
-//     const aliceSasPromise = acceptSasVerification(alice, bob.username);
-//     // wait in parallel, so they don't deadlock on each other
-//     const [bobSas, aliceSas] = await Promise.all([bobSasPromise, aliceSasPromise]);
-//     assert.deepEqual(bobSas, aliceSas);
-//     bob.log.done(`done (match for ${bobSas.join(", ")})`);
-//     const aliceMessage = "Guess what I just heard?!";
-//     await sendMessage(alice, aliceMessage);
-//     await receiveMessage(bob, {sender: "alice", body: aliceMessage, encrypted: true});
-//     const bobMessage = "You've got to tell me!";
-//     await sendMessage(bob, bobMessage);
-//     await receiveMessage(alice, {sender: "bob", body: bobMessage, encrypted: true});
-// };

test/end-to-end-tests/src/scenarios/lazy-loading.js

@@ -25,7 +25,7 @@ const {
 } = require('../usecases/timeline');
 const {createRoom} = require('../usecases/create-room');
 const {getMembersInMemberlist} = require('../usecases/memberlist');
-const changeRoomSettings = require('../usecases/room-settings');
+const {changeRoomSettings} = require('../usecases/room-settings');
 const assert = require('assert');
 
 module.exports = async function lazyLoadingScenarios(alice, bob, charlies) {

test/end-to-end-tests/src/session.js

@@ -75,6 +75,10 @@ module.exports = class RiotSession {
         return this.getElementProperty(field, 'outerHTML');
     }
 
+    isChecked(field) {
+        return this.getElementProperty(field, 'checked');
+    }
+
     consoleLogs() {
         return this.consoleLog.buffer;
     }

test/end-to-end-tests/src/usecases/create-room.js

@@ -27,7 +27,7 @@ async function createRoom(session, roomName, encrypted=false) {
     const roomListHeaderLabels = await Promise.all(roomListHeaders.map(h => session.innerText(h)));
     const roomsIndex = roomListHeaderLabels.findIndex(l => l.toLowerCase().includes("rooms"));
     if (roomsIndex === -1) {
-        throw new Error("could not find room list section that contains rooms in header");
+        throw new Error("could not find room list section that contains 'rooms' in header");
     }
     const roomsHeader = roomListHeaders[roomsIndex];
     const addRoomButton = await roomsHeader.$(".mx_RoomSubList_addRoom");
@@ -48,4 +48,39 @@ async function createRoom(session, roomName, encrypted=false) {
     session.log.done();
 }
 
-module.exports = {openRoomDirectory, createRoom};
+async function createDm(session, invitees) {
+    session.log.step(`creates DM with ${JSON.stringify(invitees)}`);
+
+    const roomListHeaders = await session.queryAll('.mx_RoomSubList_labelContainer');
+    const roomListHeaderLabels = await Promise.all(roomListHeaders.map(h => session.innerText(h)));
+    const dmsIndex = roomListHeaderLabels.findIndex(l => l.toLowerCase().includes('direct messages'));
+    if (dmsIndex === -1) {
+        throw new Error("could not find room list section that contains 'direct messages' in header");
+    }
+    const dmsHeader = roomListHeaders[dmsIndex];
+    const addRoomButton = await dmsHeader.$(".mx_RoomSubList_addRoom");
+    await addRoomButton.click();
+
+    const inviteesEditor = await session.query('.mx_InviteDialog_editor textarea');
+    for (const target of invitees) {
+        await session.replaceInputText(inviteesEditor, target);
+        await session.delay(1000); // give it a moment to figure out a suggestion
+        // find the suggestion and accept it
+        const suggestions = await session.queryAll('.mx_InviteDialog_roomTile_userId');
+        const suggestionTexts = await Promise.all(suggestions.map(s => session.innerText(s)));
+        const suggestionIndex = suggestionTexts.indexOf(target);
+        if (suggestionIndex === -1) {
+            throw new Error(`failed to find a suggestion in the DM dialog to invite ${target} with`);
+        }
+        await suggestions[suggestionIndex].click();
+    }
+
+    // press the go button and hope for the best
+    const goButton = await session.query('.mx_InviteDialog_goButton');
+    await goButton.click();
+
+    await session.query('.mx_MessageComposer');
+    session.log.done();
+}
+
+module.exports = {openRoomDirectory, createRoom, createDm};

test/end-to-end-tests/src/usecases/room-settings.js

@@ -30,18 +30,102 @@ async function setSettingsToggle(session, toggle, enabled) {
     }
 }
 
-module.exports = async function changeRoomSettings(session, settings) {
+async function checkSettingsToggle(session, toggle, shouldBeEnabled) {
-    session.log.startGroup(`changes the room settings`);
+    const className = await session.getElementProperty(toggle, "className");
+    const checked = className.includes("mx_ToggleSwitch_on");
+    if (checked === shouldBeEnabled) {
+        session.log.done('set as expected');
+    } else {
+        // other logs in the area should give more context as to what this means.
+        throw new Error("settings toggle value didn't match expectation");
+    }
+}
+
+async function findTabs(session) {
     /// XXX delay is needed here, possibly because the header is being rerendered
     /// click doesn't do anything otherwise
     await session.delay(1000);
     const settingsButton = await session.query(".mx_RoomHeader .mx_AccessibleButton[title=Settings]");
     await settingsButton.click();
+
     //find tabs
     const tabButtons = await session.queryAll(".mx_RoomSettingsDialog .mx_TabbedView_tabLabel");
     const tabLabels = await Promise.all(tabButtons.map(t => session.innerText(t)));
     const securityTabButton = tabButtons[tabLabels.findIndex(l => l.toLowerCase().includes("security"))];
 
+    return {securityTabButton};
+}
+
+async function checkRoomSettings(session, expectedSettings) {
+    session.log.startGroup(`checks the room settings`);
+
+    const {securityTabButton} = await findTabs(session);
+    const generalSwitches = await session.queryAll(".mx_RoomSettingsDialog .mx_ToggleSwitch");
+    const isDirectory = generalSwitches[0];
+
+    if (typeof expectedSettings.directory === 'boolean') {
+        session.log.step(`checks directory listing is ${expectedSettings.directory}`);
+        await checkSettingsToggle(session, isDirectory, expectedSettings.directory);
+    }
+
+    if (expectedSettings.alias) {
+        session.log.step(`checks for local alias of ${expectedSettings.alias}`);
+        const summary = await session.query(".mx_RoomSettingsDialog .mx_AliasSettings summary");
+        await summary.click();
+        const localAliases = await session.query('.mx_RoomSettingsDialog .mx_AliasSettings .mx_EditableItem_item');
+        const localAliasTexts = await Promise.all(localAliases.map(a => session.innerText(a)));
+        if (localAliasTexts.find(a => a.includes(expectedSettings.alias))) {
+            session.log.done("present");
+        } else {
+            throw new Error(`could not find local alias ${expectedSettings.alias}`);
+        }
+    }
+
+    securityTabButton.click();
+    await session.delay(500);
+    const securitySwitches = await session.queryAll(".mx_RoomSettingsDialog .mx_ToggleSwitch");
+    const e2eEncryptionToggle = securitySwitches[0];
+
+    if (typeof expectedSettings.encryption === "boolean") {
+        session.log.step(`checks room e2e encryption is ${expectedSettings.encryption}`);
+        await checkSettingsToggle(session, e2eEncryptionToggle, expectedSettings.encryption);
+    }
+
+    if (expectedSettings.visibility) {
+        session.log.step(`checks visibility is ${expectedSettings.visibility}`);
+        const radios = await session.queryAll(".mx_RoomSettingsDialog input[type=radio]");
+        assert.equal(radios.length, 7);
+        const inviteOnly = radios[0];
+        const publicNoGuests = radios[1];
+        const publicWithGuests = radios[2];
+
+        let expectedRadio = null;
+        if (expectedSettings.visibility === "invite_only") {
+            expectedRadio = inviteOnly;
+        } else if (expectedSettings.visibility === "public_no_guests") {
+            expectedRadio = publicNoGuests;
+        } else if (expectedSettings.visibility === "public_with_guests") {
+            expectedRadio = publicWithGuests;
+        } else {
+            throw new Error(`unrecognized room visibility setting: ${expectedSettings.visibility}`);
+        }
+        if (await session.isChecked(expectedRadio)) {
+            session.log.done();
+        } else {
+            throw new Error("room visibility is not as expected");
+        }
+    }
+
+    const closeButton = await session.query(".mx_RoomSettingsDialog .mx_Dialog_cancelButton");
+    await closeButton.click();
+
+    session.log.endGroup();
+}
+
+async function changeRoomSettings(session, settings) {
+    session.log.startGroup(`changes the room settings`);
+
+    const {securityTabButton} = await findTabs(session);
     const generalSwitches = await session.queryAll(".mx_RoomSettingsDialog .mx_ToggleSwitch");
     const isDirectory = generalSwitches[0];
 
@@ -100,4 +184,6 @@ module.exports = async function changeRoomSettings(session, settings) {
     await closeButton.click();
 
     session.log.endGroup();
-};
+}
+
+module.exports = {checkRoomSettings, changeRoomSettings};

test/end-to-end-tests/src/usecases/verify.js

@@ -25,10 +25,19 @@ async function assertVerified(session) {
 }
 
 async function startVerification(session, name) {
+    session.log.step("opens their opponent's profile and starts verification");
     await openMemberInfo(session, name);
     // click verify in member info
-    const firstVerifyButton = await session.query(".mx_MemberDeviceInfo_verify");
+    const firstVerifyButton = await session.query(".mx_UserInfo_verifyButton");
     await firstVerifyButton.click();
+
+    // wait for the animation to finish
+    await session.delay(1000);
+
+    // click 'start verification'
+    const startVerifyButton = await session.query('.mx_UserInfo_container .mx_AccessibleButton_kind_primary');
+    await startVerifyButton.click();
+    session.log.done();
 }
 
 async function getSasCodes(session) {
@@ -38,33 +47,73 @@ async function getSasCodes(session) {
     return sasLabels;
 }
 
-module.exports.startSasVerifcation = async function(session, name) {
+async function doSasVerification(session) {
-    await startVerification(session, name);
+    session.log.step("hunts for the emoji to yell at their opponent");
-    // expect "Verify device" dialog and click "Begin Verification"
-    await assertDialog(session, "Verify device");
-    // click "Begin Verification"
-    await acceptDialog(session);
     const sasCodes = await getSasCodes(session);
-    // click "Verify"
+    session.log.done(sasCodes);
-    await acceptDialog(session);
+
-    await assertVerified(session);
+    // Assume they match
-    // click "Got it" when verification is done
+    session.log.step("assumes the emoji match");
-    await acceptDialog(session);
+    const matchButton = await session.query(".mx_VerificationShowSas .mx_AccessibleButton_kind_primary");
+    await matchButton.click();
+    session.log.done();
+
+    // Wait for a big green shield (universal sign that it worked)
+    session.log.step("waits for a green shield");
+    await session.query(".mx_VerificationPanel_verified_section .mx_E2EIcon_verified");
+    session.log.done();
+
+    // Click 'Got It'
+    session.log.step("confirms the green shield");
+    const doneButton = await session.query(".mx_VerificationPanel_verified_section .mx_AccessibleButton_kind_primary");
+    await doneButton.click();
+    session.log.done();
+
+    // Wait a bit for the animation
+    session.log.step("confirms their opponent has a green shield");
+    await session.delay(1000);
+
+    // Verify that we now have a green shield in their name (proving it still works)
+    await session.query('.mx_UserInfo_profile .mx_E2EIcon_verified');
+    session.log.done();
+
+    return sasCodes;
+}
+
+module.exports.startSasVerifcation = async function(session, name) {
+    session.log.startGroup("starts verification");
+    await startVerification(session, name);
+
+    // expect to be waiting (the presence of a spinner is a good thing)
+    await session.query('.mx_UserInfo_container .mx_EncryptionInfo_spinner');
+
+    const sasCodes = await doSasVerification(session);
+    session.log.endGroup();
     return sasCodes;
 };
 
 module.exports.acceptSasVerification = async function(session, name) {
-    await assertDialog(session, "Incoming Verification Request");
+    session.log.startGroup("accepts verification");
-    const opponentLabelElement = await session.query(".mx_IncomingSasDialog_opponentProfile h2");
+    const requestToast = await session.query('.mx_Toast_icon_verification');
-    const opponentLabel = await session.innerText(opponentLabelElement);
+
-    assert(opponentLabel, name);
+    // verify the toast is for verification
-    // click "Continue" button
+    const toastHeader = await requestToast.$("h2");
-    await acceptDialog(session);
+    const toastHeaderText = await session.innerText(toastHeader);
-    const sasCodes = await getSasCodes(session);
+    assert.equal(toastHeaderText, 'Verification Request');
-    // click "Verify"
+    const toastDescription = await requestToast.$(".mx_Toast_description");
-    await acceptDialog(session);
+    const toastDescText = await session.innerText(toastDescription);
-    await assertVerified(session);
+    assert.equal(toastDescText.startsWith(name), true,
-    // click "Got it" when verification is done
+        `verification opponent mismatch: expected to start with '${name}', got '${toastDescText}'`);
-    await acceptDialog(session);
+
+    // accept the verification
+    const acceptButton = await requestToast.$(".mx_AccessibleButton_kind_primary");
+    await acceptButton.click();
+
+    // find the emoji button
+    const startEmojiButton = await session.query(".mx_VerificationPanel_verifyByEmojiButton");
+    await startEmojiButton.click();
+
+    const sasCodes = await doSasVerification(session);
+    session.log.endGroup();
     return sasCodes;
 };