Merge pull request #3827 from matrix-org/travis/sm-e2e-tests

Enable end-to-end tests for sourcemaps (+Windows instructions)

.buildkite/pipeline.yaml

@@ -54,77 +54,50 @@ steps:
       - docker#v3.0.1:
           image: "node:12"
 
-#  - label: ":chains: End-to-End Tests"
-#    agents:
-#      # We use a xlarge sized instance instead of the normal small ones because
-#      # e2e tests otherwise take +-8min
-#      queue: "xlarge"
-#    command:
-#      # TODO: Remove hacky chmod for BuildKite
-#      - "echo '--- Setup'"
-#      - "chmod +x ./scripts/ci/*.sh"
-#      - "chmod +x ./scripts/*"
-#      - "echo '--- Install js-sdk'"
-#      - "./scripts/ci/install-deps.sh"
-#      - "./scripts/ci/end-to-end-tests.sh"
-#    plugins:
-#      - docker#v3.0.1:
-#          image: "matrixdotorg/riotweb-ci-e2etests-env:latest"
-#          propagate-environment: true
-#
-#  - label: ":karma: Tests"
-#    agents:
-#      # We use a medium sized instance instead of the normal small ones because
-#      # webpack loves to gorge itself on resources.
-#      queue: "medium"
-#    command:
-#      # Install chrome
-#      - "echo '--- Installing Chrome'"
-#      - "wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add -"
-#      - "sh -c 'echo \"deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main\" >> /etc/apt/sources.list.d/google.list'"
-#      - "apt-get update"
-#      - "apt-get install -y google-chrome-stable"
-#      # Run tests
-#      # TODO: Remove hacky chmod for BuildKite
-#      - "chmod +x ./scripts/ci/*.sh"
-#      - "chmod +x ./scripts/*"
-#      - "echo '--- Installing Dependencies'"
-#      - "./scripts/ci/install-deps.sh"
-#      - "echo '+++ Running Tests'"
-#      - "./scripts/ci/unit-tests.sh"
-#    env:
-#      CHROME_BIN: "/usr/bin/google-chrome-stable"
-#    plugins:
-#      - docker#v3.0.1:
-#          image: "node:10"
-#          propagate-environment: true
-#
-#  - label: "🔧 Riot Tests"
-#    agents:
-#      # We use a medium sized instance instead of the normal small ones because
-#      # webpack loves to gorge itself on resources.
-#      queue: "medium"
-#    command:
-#      # Install chrome
-#      - "echo '--- Installing Chrome'"
-#      - "wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add -"
-#      - "sh -c 'echo \"deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main\" >> /etc/apt/sources.list.d/google.list'"
-#      - "apt-get update"
-#      - "apt-get install -y google-chrome-stable"
-#      # Run tests
-#      # TODO: Remove hacky chmod for BuildKite
-#      - "chmod +x ./scripts/ci/*.sh"
-#      - "chmod +x ./scripts/*"
-#      - "echo '--- Installing Dependencies'"
-#      - "./scripts/ci/install-deps.sh"
-#      - "echo '+++ Running Tests'"
-#      - "./scripts/ci/riot-unit-tests.sh"
-#    env:
-#      CHROME_BIN: "/usr/bin/google-chrome-stable"
-#    plugins:
-#      - docker#v3.0.1:
-#          image: "node:10"
-#          propagate-environment: true
+  - label: ":chains: End-to-End Tests"
+    agents:
+      # We use a xlarge sized instance instead of the normal small ones because
+      # e2e tests otherwise take +-8min
+      queue: "xlarge"
+    command:
+      # TODO: Remove hacky chmod for BuildKite
+      - "echo '--- Setup'"
+      - "chmod +x ./scripts/ci/*.sh"
+      - "chmod +x ./scripts/*"
+      - "echo '--- Install js-sdk'"
+      - "./scripts/ci/install-deps.sh"
+      - "./scripts/ci/end-to-end-tests.sh"
+    plugins:
+      - docker#v3.0.1:
+          image: "matrixdotorg/riotweb-ci-e2etests-env:latest"
+          propagate-environment: true
+
+  - label: "🔧 Riot Tests"
+    agents:
+      # We use a medium sized instance instead of the normal small ones because
+      # webpack loves to gorge itself on resources.
+      queue: "medium"
+    command:
+      # Install chrome
+      - "echo '--- Installing Chrome'"
+      - "wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add -"
+      - "sh -c 'echo \"deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main\" >> /etc/apt/sources.list.d/google.list'"
+      - "apt-get update"
+      - "apt-get install -y google-chrome-stable"
+      # Run tests
+      # TODO: Remove hacky chmod for BuildKite
+      - "chmod +x ./scripts/ci/*.sh"
+      - "chmod +x ./scripts/*"
+      - "echo '--- Installing Dependencies'"
+      - "./scripts/ci/install-deps.sh"
+      - "echo '+++ Running Tests'"
+      - "./scripts/ci/riot-unit-tests.sh"
+    env:
+      CHROME_BIN: "/usr/bin/google-chrome-stable"
+    plugins:
+      - docker#v3.0.1:
+          image: "node:10"
+          propagate-environment: true
 
   - label: "🌐 i18n"
     command:

test/end-to-end-tests/Windows.md

@@ -0,0 +1,39 @@
+# Running the end-to-end tests on Windows
+
+Windows is not the best platform to run the tests on, but if you have to, enable Windows Subsystem for Linux (WSL)
+and start following these steps to get going:
+
+1. Navigate to your working directory (`cd /mnt/c/users/travisr/whatever/matrix-react-sdk` for example).
+2. Run `sudo apt-get install unzip python3 virtualenv dos2unix`
+3. Run `dos2unix ./test/end-to-end-tests/*.sh ./test/end-to-end-tests/synapse/*.sh ./test/end-to-end-tests/riot/*.sh`
+4. Install NodeJS for ubuntu: 
+   ```bash
+   curl -sL https://deb.nodesource.com/setup_10.x | sudo -E bash -
+   sudo apt-get update
+   sudo apt-get install nodejs
+   ```
+5. Start Riot on Windows through `yarn start`
+6. While that builds... Run:
+   ```bash
+   sudo apt-get install x11-apps
+   wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb
+   sudo dpkg -i google-chrome-stable_current_amd64.deb
+   sudo apt -f install
+   ```
+7. Run: 
+   ```bash
+   cd ./test/end-to-end-tests
+   ./synapse/install.sh
+   ./run.sh --riot-url http://localhost:8080 --no-sandbox
+   ```
+
+Note that using `yarn test:e2e` probably won't work for you. You might also have to use the config.json from the
+`riot/config-template` directory in order to actually succeed at the tests.
+
+Also note that you'll have to use `--no-sandbox` otherwise Chrome will complain that there's no sandbox available. You
+could probably fix this with enough effort, or you could run a headless Chrome in the WSL container without a sandbox.
+
+
+Reference material that isn't fully represented in the steps above (but snippets have been borrowed):
+* https://virtualizationreview.com/articles/2017/02/08/graphical-programs-on-windows-subsystem-on-linux.aspx
+* https://gist.github.com/drexler/d70ab957f964dbef1153d46bd853c775

test/end-to-end-tests/src/logbuffer.js

@@ -15,7 +15,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-export default class LogBuffer {
+module.exports = class LogBuffer {
     constructor(page, eventName, eventMapper, reduceAsync=false, initialValue = "") {
         this.buffer = initialValue;
         page.on(eventName, (arg) => {

test/end-to-end-tests/src/logger.js

@@ -15,7 +15,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-export default class Logger {
+module.exports = class Logger {
     constructor(username) {
         this.indent = 0;
         this.username = username;

test/end-to-end-tests/src/rest/consent.js

@@ -19,7 +19,7 @@ const request = require('request-promise-native');
 const cheerio = require('cheerio');
 const url = require("url");
 
-export async function approveConsent(consentUrl) {
+module.exports.approveConsent = async function(consentUrl) {
     const body = await request.get(consentUrl);
     const doc = cheerio.load(body);
     const v = doc("input[name=v]").val();
@@ -28,4 +28,4 @@ export async function approveConsent(consentUrl) {
     const formAction = doc("form").attr("action");
     const absAction = url.resolve(consentUrl, formAction);
     await request.post(absAction).form({v, u, h});
-}
+};

test/end-to-end-tests/src/rest/creator.js

@@ -32,7 +32,7 @@ function execAsync(command, options) {
     });
 }
 
-export default class RestSessionCreator {
+module.exports = class RestSessionCreator {
     constructor(synapseSubdir, hsUrl, cwd) {
         this.synapseSubdir = synapseSubdir;
         this.hsUrl = hsUrl;
@@ -72,12 +72,12 @@ export default class RestSessionCreator {
 
     async _authenticate(username, password) {
         const requestBody = {
-          "type": "m.login.password",
-          "identifier": {
-            "type": "m.id.user",
-            "user": username,
-          },
-          "password": password,
+            "type": "m.login.password",
+            "identifier": {
+                "type": "m.id.user",
+                "user": username,
+            },
+            "password": password,
         };
         const url = `${this.hsUrl}/_matrix/client/r0/login`;
         const responseBody = await request.post({url, json: true, body: requestBody});

test/end-to-end-tests/src/rest/multi.js

@@ -17,7 +17,7 @@ limitations under the License.
 
 const Logger = require('../logger');
 
-export default class RestMultiSession {
+module.exports = class RestMultiSession {
     constructor(sessions, groupName) {
         this.log = new Logger(groupName);
         this.sessions = sessions;

test/end-to-end-tests/src/rest/room.js

@@ -18,7 +18,7 @@ limitations under the License.
 const uuidv4 = require('uuid/v4');
 
 /* no pun intented */
-export default class RestRoom {
+module.exports = class RestRoom {
     constructor(session, roomId, log) {
         this.session = session;
         this._roomId = roomId;

test/end-to-end-tests/src/rest/session.js

@@ -19,7 +19,7 @@ const Logger = require('../logger');
 const RestRoom = require('./room');
 const {approveConsent} = require('./consent');
 
-export default class RestSession {
+module.exports = class RestSession {
     constructor(credentials) {
         this.log = new Logger(credentials.userId);
         this._credentials = credentials;

test/end-to-end-tests/src/scenarios/directory.js

@@ -22,7 +22,7 @@ const {receiveMessage} = require('../usecases/timeline');
 const {createRoom} = require('../usecases/create-room');
 const changeRoomSettings = require('../usecases/room-settings');
 
-export default async function roomDirectoryScenarios(alice, bob) {
+module.exports = async function roomDirectoryScenarios(alice, bob) {
     console.log(" creating a public room and join through directory:");
     const room = 'test';
     await createRoom(alice, room);

test/end-to-end-tests/src/scenarios/e2e-encryption.js

@@ -24,7 +24,7 @@ const changeRoomSettings = require('../usecases/room-settings');
 const {startSasVerifcation, acceptSasVerification} = require('../usecases/verify');
 const assert = require('assert');
 
-export default async function e2eEncryptionScenarios(alice, bob) {
+module.exports = async function e2eEncryptionScenarios(alice, bob) {
     console.log(" creating an e2e encrypted room and join through invite:");
     const room = "secrets";
     await createRoom(bob, room);

test/end-to-end-tests/src/scenarios/lazy-loading.js

@@ -28,7 +28,7 @@ const {getMembersInMemberlist} = require('../usecases/memberlist');
 const changeRoomSettings = require('../usecases/room-settings');
 const assert = require('assert');
 
-export default async function lazyLoadingScenarios(alice, bob, charlies) {
+module.exports = async function lazyLoadingScenarios(alice, bob, charlies) {
     console.log(" creating a room for lazy loading member scenarios:");
     const charly1to5 = charlies.slice("charly-1..5", 0, 5);
     const charly6to10 = charlies.slice("charly-6..10", 5);

test/end-to-end-tests/src/session.js

@@ -22,7 +22,7 @@ const {delay} = require('./util');
 
 const DEFAULT_TIMEOUT = 20000;
 
-export default class RiotSession {
+module.exports = class RiotSession {
     constructor(browser, page, username, riotserver, hsUrl) {
         this.browser = browser;
         this.page = page;

test/end-to-end-tests/src/usecases/accept-invite.js

@@ -15,7 +15,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-export default async function acceptInvite(session, name) {
+module.exports = async function acceptInvite(session, name) {
     session.log.step(`accepts "${name}" invite`);
     //TODO: brittle selector
     const invitesHandles = await session.queryAll('.mx_RoomTile_name.mx_RoomTile_invite');
@@ -24,7 +24,7 @@ export default async function acceptInvite(session, name) {
         return {inviteHandle, text};
     }));
     const inviteHandle = invitesWithText.find(({inviteHandle, text}) => {
-    return text.trim() === name;
+        return text.trim() === name;
     }).inviteHandle;
 
     await inviteHandle.click();

test/end-to-end-tests/src/usecases/create-room.js

@@ -15,12 +15,12 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-export async function openRoomDirectory(session) {
+async function openRoomDirectory(session) {
     const roomDirectoryButton = await session.query('.mx_LeftPanel_explore .mx_AccessibleButton');
     await roomDirectoryButton.click();
 }
 
-export async function createRoom(session, roomName) {
+async function createRoom(session, roomName) {
     session.log.step(`creates room "${roomName}"`);
 
     const roomListHeaders = await session.queryAll('.mx_RoomSubList_labelContainer');
@@ -43,3 +43,5 @@ export async function createRoom(session, roomName) {
     await session.query('.mx_MessageComposer');
     session.log.done();
 }
+
+module.exports = {openRoomDirectory, createRoom};

test/end-to-end-tests/src/usecases/dialog.js

@@ -17,20 +17,20 @@ limitations under the License.
 
 const assert = require('assert');
 
-export async function assertDialog(session, expectedTitle) {
+async function assertDialog(session, expectedTitle) {
     const titleElement = await session.query(".mx_Dialog .mx_Dialog_title");
     const dialogHeader = await session.innerText(titleElement);
     assert(dialogHeader, expectedTitle);
 }
 
-export async function acceptDialog(session, expectedTitle) {
+async function acceptDialog(session, expectedTitle) {
     const foundDialog = await acceptDialogMaybe(session, expectedTitle);
     if (!foundDialog) {
         throw new Error("could not find a dialog");
     }
 }
 
-export async function acceptDialogMaybe(session, expectedTitle) {
+async function acceptDialogMaybe(session, expectedTitle) {
     let primaryButton = null;
     try {
         primaryButton = await session.query(".mx_Dialog .mx_Dialog_primary");
@@ -43,3 +43,9 @@ export async function acceptDialogMaybe(session, expectedTitle) {
     await primaryButton.click();
     return true;
 }
+
+module.exports = {
+    assertDialog,
+    acceptDialog,
+    acceptDialogMaybe,
+};

test/end-to-end-tests/src/usecases/invite.js

@@ -15,7 +15,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-export default async function invite(session, userId) {
+module.exports = async function invite(session, userId) {
     session.log.step(`invites "${userId}" to room`);
     await session.delay(1000);
     const memberPanelButton = await session.query(".mx_RightPanel_membersButton");

test/end-to-end-tests/src/usecases/join.js

@@ -17,7 +17,7 @@ limitations under the License.
 
 const {openRoomDirectory} = require('./create-room');
 
-export default async function join(session, roomName) {
+module.exports = async function join(session, roomName) {
     session.log.step(`joins room "${roomName}"`);
     await openRoomDirectory(session);
     const roomInput = await session.query('.mx_DirectorySearchBox input');

test/end-to-end-tests/src/usecases/memberlist.js

@@ -17,7 +17,7 @@ limitations under the License.
 
 const assert = require('assert');
 
-export async function openMemberInfo(session, name) {
+async function openMemberInfo(session, name) {
     const membersAndNames = await getMembersInMemberlist(session);
     const matchingLabel = membersAndNames.filter((m) => {
         return m.displayName === name;
@@ -25,7 +25,9 @@ export async function openMemberInfo(session, name) {
     await matchingLabel.click();
 }
 
-export async function verifyDeviceForUser(session, name, expectedDevice) {
+module.exports.openMemberInfo = openMemberInfo;
+
+module.exports.verifyDeviceForUser = async function(session, name, expectedDevice) {
     session.log.step(`verifies e2e device for ${name}`);
     const membersAndNames = await getMembersInMemberlist(session);
     const matchingLabel = membersAndNames.filter((m) => {
@@ -58,9 +60,9 @@ export async function verifyDeviceForUser(session, name, expectedDevice) {
     const closeMemberInfo = await session.query(".mx_MemberInfo_cancel");
     await closeMemberInfo.click();
     session.log.done();
-}
+};
 
-export async function getMembersInMemberlist(session) {
+async function getMembersInMemberlist(session) {
     const memberPanelButton = await session.query(".mx_RightPanel_membersButton");
     try {
         await session.query(".mx_RightPanel_headerButton_highlight", 500);
@@ -77,3 +79,5 @@ export async function getMembersInMemberlist(session) {
         return {label: el, displayName: await session.innerText(el)};
     }));
 }
+
+module.exports.getMembersInMemberlist = getMembersInMemberlist;

test/end-to-end-tests/src/usecases/room-settings.js

@@ -30,7 +30,7 @@ async function setSettingsToggle(session, toggle, enabled) {
     }
 }
 
-export default async function changeRoomSettings(session, settings) {
+module.exports = async function changeRoomSettings(session, settings) {
     session.log.startGroup(`changes the room settings`);
     /// XXX delay is needed here, possibly because the header is being rerendered
     /// click doesn't do anything otherwise

test/end-to-end-tests/src/usecases/settings.js

@@ -29,7 +29,7 @@ async function openSettings(session, section) {
     }
 }
 
-export async function enableLazyLoading(session) {
+module.exports.enableLazyLoading = async function(session) {
     session.log.step(`enables lazy loading of members in the lab settings`);
     const settingsButton = await session.query('.mx_BottomLeftMenu_settings');
     await settingsButton.click();
@@ -39,9 +39,9 @@ export async function enableLazyLoading(session) {
     const closeButton = await session.query(".mx_RoomHeader_cancelButton");
     await closeButton.click();
     session.log.done();
-}
+};
 
-export async function getE2EDeviceFromSettings(session) {
+module.exports.getE2EDeviceFromSettings = async function(session) {
     session.log.step(`gets e2e device/key from settings`);
     await openSettings(session, "security");
     const deviceAndKey = await session.queryAll(".mx_SettingsTab_section .mx_SecurityUserSettingsTab_deviceInfo code");
@@ -52,4 +52,4 @@ export async function getE2EDeviceFromSettings(session) {
     await closeButton.click();
     session.log.done();
     return {id, key};
-}
+};

test/end-to-end-tests/src/usecases/signup.js

@@ -17,7 +17,7 @@ limitations under the License.
 
 const assert = require('assert');
 
-export default async function signup(session, username, password, homeserver) {
+module.exports = async function signup(session, username, password, homeserver) {
     session.log.step("signs up");
     await session.goto(session.url('/#/register'));
     // change the homeserver by clicking the advanced section

test/end-to-end-tests/src/usecases/timeline.js

@@ -17,7 +17,7 @@ limitations under the License.
 
 const assert = require('assert');
 
-export async function scrollToTimelineTop(session) {
+module.exports.scrollToTimelineTop = async function(session) {
     session.log.step(`scrolls to the top of the timeline`);
     await session.page.evaluate(() => {
         return Promise.resolve().then(async () => {
@@ -41,9 +41,9 @@ export async function scrollToTimelineTop(session) {
         });
     });
     session.log.done();
-}
+};
 
-export async function receiveMessage(session, expectedMessage) {
+module.exports.receiveMessage = async function(session, expectedMessage) {
     session.log.step(`receives message "${expectedMessage.body}" from ${expectedMessage.sender}`);
     // wait for a response to come in that contains the message
     // crude, but effective
@@ -67,10 +67,10 @@ export async function receiveMessage(session, expectedMessage) {
     });
     assertMessage(lastMessage, expectedMessage);
     session.log.done();
-}
+};
 
 
-export async function checkTimelineContains(session, expectedMessages, sendersDescription) {
+module.exports.checkTimelineContains = async function(session, expectedMessages, sendersDescription) {
     session.log.step(`checks timeline contains ${expectedMessages.length} ` +
         `given messages${sendersDescription ? ` from ${sendersDescription}`:""}`);
     const eventTiles = await getAllEventTiles(session);
@@ -91,7 +91,7 @@ export async function checkTimelineContains(session, expectedMessages, sendersDe
     expectedMessages.forEach((expectedMessage) => {
         const foundMessage = timelineMessages.find((message) => {
             return message.sender === expectedMessage.sender &&
-                   message.body === expectedMessage.body;
+                message.body === expectedMessage.body;
         });
         try {
             assertMessage(foundMessage, expectedMessage);
@@ -102,7 +102,7 @@ export async function checkTimelineContains(session, expectedMessages, sendersDe
     });
 
     session.log.done();
-}
+};
 
 function assertMessage(foundMessage, expectedMessage) {
     assert(foundMessage, `message ${JSON.stringify(expectedMessage)} not found in timeline`);

test/end-to-end-tests/src/usecases/verify.js

@@ -38,7 +38,7 @@ async function getSasCodes(session) {
     return sasLabels;
 }
 
-export async function startSasVerifcation(session, name) {
+module.exports.startSasVerifcation = async function(session, name) {
     await startVerification(session, name);
     // expect "Verify device" dialog and click "Begin Verification"
     await assertDialog(session, "Verify device");
@@ -51,9 +51,9 @@ export async function startSasVerifcation(session, name) {
     // click "Got it" when verification is done
     await acceptDialog(session);
     return sasCodes;
-}
+};
 
-export async function acceptSasVerification(session, name) {
+module.exports.acceptSasVerification = async function(session, name) {
     await assertDialog(session, "Incoming Verification Request");
     const opponentLabelElement = await session.query(".mx_IncomingSasDialog_opponentProfile h2");
     const opponentLabel = await session.innerText(opponentLabelElement);
@@ -67,4 +67,4 @@ export async function acceptSasVerification(session, name) {
     // click "Got it" when verification is done
     await acceptDialog(session);
     return sasCodes;
-}
+};

test/end-to-end-tests/src/util.js

@@ -15,14 +15,14 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-export function range(start, amount, step = 1) {
+module.exports.range = function(start, amount, step = 1) {
     const r = [];
     for (let i = 0; i < amount; ++i) {
         r.push(start + (i * step));
     }
     return r;
-}
+};
 
-export function delay(ms) {
+module.exports.delay = function(ms) {
     return new Promise((resolve) => setTimeout(resolve, ms));
-}
+};