Merge pull request #26229 from vector-im/johannes/webpack-5

2023-11-20 13:30:36 +01:00 · 2023-11-20 13:30:36 +01:00 · 619f36b82a
parent 4a519916e5 37c1c4e8ae
commit 619f36b82a
10 changed files with 699 additions and 1769 deletions
--- a/jest.config.ts
+++ b/jest.config.ts
@ -40,9 +40,9 @@ const config: Config = {
        "waveWorker\\.min\\.js": "<rootDir>/node_modules/matrix-react-sdk/__mocks__/empty.js",
        "context-filter-polyfill": "<rootDir>/node_modules/matrix-react-sdk/__mocks__/empty.js",
        "FontManager.ts": "<rootDir>/node_modules/matrix-react-sdk/__mocks__/FontManager.js",
-        "workers/(.+)\\.worker\\.ts": "<rootDir>/node_modules/matrix-react-sdk/__mocks__/workerMock.js",
+        "workers/(.+)Factory": "<rootDir>/node_modules/matrix-react-sdk/__mocks__/workerFactoryMock.js",
        "^!!raw-loader!.*": "jest-raw-loader",
-        "RecorderWorklet": "<rootDir>/node_modules/matrix-react-sdk/__mocks__/empty.js",
+        "recorderWorkletFactory": "<rootDir>/node_modules/matrix-react-sdk/__mocks__/empty.js",
        "^fetch-mock$": "<rootDir>/node_modules/fetch-mock",
    },
    transformIgnorePatterns: ["/node_modules/(?!matrix-js-sdk).+$", "/node_modules/(?!matrix-react-sdk).+$"],
--- a/package.json
+++ b/package.json
@ -43,12 +43,12 @@
        "build:modernizr": "modernizr -c .modernizr.json -d src/vector/modernizr.js",
        "build:bundle": "webpack --progress --mode production",
        "build:bundle-stats": "webpack --progress --mode production --json > webpack-stats.json",
-        "build:module_system": "tsc --project ./tsconfig.module_system.json && node ./lib/module_system/scripts/install.js",
+        "build:module_system": "ts-node --project ./tsconfig.module_system.json module_system/scripts/install.ts",
        "dist": "scripts/package.sh",
        "start": "concurrently --kill-others-on-fail --prefix \"{time} [{name}]\" -n modules,res,jitsi \"yarn build:module_system\" \"yarn build:res\" \"yarn build:jitsi\" && concurrently --kill-others-on-fail --prefix \"{time} [{name}]\" -n res,element-js \"yarn start:res\" \"yarn start:js\"",
        "start:https": "concurrently --kill-others-on-fail --prefix \"{time} [{name}]\" -n res,element-js \"yarn start:res\" \"yarn start:js --https\"",
        "start:res": "ts-node scripts/copy-res.ts -w",
-        "start:js": "webpack serve --output-path webapp --mode development",
+        "start:js": "webpack serve --output-path webapp --output-filename=bundles/_dev_/[name].js --output-chunk-filename=bundles/_dev_/[name].js --mode development",
        "lint": "yarn lint:types && yarn lint:js && yarn lint:style",
        "lint:js": "yarn lint:js:src && yarn lint:js:module_system",
        "lint:js:src": "eslint --max-warnings 0 src test && prettier --check .",
@ -103,13 +103,12 @@
        "@babel/runtime": "^7.12.5",
        "@casualbot/jest-sonar-reporter": "2.2.7",
        "@principalstudio/html-webpack-inject-preload": "^1.2.7",
-        "@sentry/webpack-plugin": "^2.0.0",
+        "@sentry/webpack-plugin": "^2.7.1",
        "@svgr/webpack": "^5.5.0",
        "@testing-library/react": "^12.1.5",
        "@types/jest": "^29.0.0",
        "@types/jitsi-meet": "^2.0.2",
        "@types/jsrsasign": "^10.5.4",
-        "@types/loader-utils": "^2.0.4",
        "@types/lodash": "^4.14.197",
        "@types/node": "^16",
        "@types/node-fetch": "^2.6.4",
@ -121,11 +120,13 @@
        "allchange": "^1.0.6",
        "babel-jest": "^29.0.0",
        "babel-loader": "^8.2.2",
+        "buffer": "^6.0.3",
        "chokidar": "^3.5.1",
        "concurrently": "^8.0.0",
-        "copy-webpack-plugin": "^6.0.0",
+        "copy-webpack-plugin": "^11.0.0",
        "cronstrue": "^2.41.0",
        "css-loader": "^4",
+        "css-minimizer-webpack-plugin": "^5.0.1",
        "dotenv": "^16.0.2",
        "eslint": "8.53.0",
        "eslint-config-google": "^0.14.0",
@ -140,21 +141,19 @@
        "fetch-mock": "9.11.0",
        "fetch-mock-jest": "^1.5.1",
        "file-loader": "^6.0.0",
-        "html-webpack-plugin": "^4.5.2",
+        "html-webpack-plugin": "^5.5.3",
        "jest": "^29.0.0",
        "jest-canvas-mock": "2.5.2",
        "jest-environment-jsdom": "^29.0.0",
        "jest-mock": "^29.0.0",
        "jest-raw-loader": "^1.0.1",
-        "loader-utils": "^3.0.0",
        "matrix-mock-request": "^2.5.0",
        "matrix-web-i18n": "^3.1.3",
-        "mini-css-extract-plugin": "^1",
+        "mini-css-extract-plugin": "^2.7.6",
        "minimist": "^1.2.6",
        "mkdirp": "^3.0.0",
        "modernizr": "^3.12.0",
        "node-fetch": "^2.6.7",
-        "optimize-css-assets-webpack-plugin": "^6.0.0",
        "postcss": "^8.4.31",
        "postcss-easings": "^2.0.0",
        "postcss-hexrgba": "2.0.1",
@ -166,25 +165,26 @@
        "postcss-scss": "^4.0.4",
        "postcss-simple-vars": "^5.0.2",
        "prettier": "2.8.8",
+        "process": "^0.11.10",
        "proxy-agent": "^6.3.0",
        "raw-loader": "^4.0.2",
        "rimraf": "^5.0.0",
        "semver": "^7.5.2",
+        "setimmediate": "^1.0.5",
        "string-replace-loader": "3",
        "style-loader": "2",
        "stylelint": "^15.10.1",
        "stylelint-config-standard": "^34.0.0",
        "stylelint-scss": "^5.0.0",
-        "terser-webpack-plugin": "^4.0.0",
+        "terser-webpack-plugin": "^5.3.9",
        "ts-node": "^10.9.1",
        "ts-prune": "^0.10.3",
        "typescript": "5.2.2",
-        "webpack": "^4.47.0",
+        "util": "^0.12.5",
+        "webpack": "^5.89.0",
        "webpack-bundle-analyzer": "^4.8.0",
        "webpack-cli": "^4.10.0",
        "webpack-dev-server": "^4.15.1",
-        "worker-loader": "^3.0.0",
-        "worklet-loader": "^2.0.0",
        "yaml": "^2.3.3"
    },
    "@casualbot/jest-sonar-reporter": {
--- a/recorder-worklet-loader.js
+++ b/recorder-worklet-loader.js
@ -0,0 +1,45 @@
+/*
+Copyright 2023 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Inspired by https://github.com/reklawnos/worklet-loader which doesn't
+// formally support Webpack 5
+
+const SingleEntryPlugin = require("webpack/lib/SingleEntryPlugin");
+
+module.exports = function () {};
+
+module.exports.pitch = function pitch(request) {
+    const cb = this.async();
+    const filename = "recorder.worklet.js";
+
+    const compiler = this._compilation.createChildCompiler("worker", {
+        filename,
+        chunkFilename: `[id].${filename}`,
+        namedChunkFilename: null,
+    });
+
+    new SingleEntryPlugin(this.context, `!!${request}`, "main").apply(compiler);
+
+    compiler.runAsChild((err, entries, compilation) => {
+        if (err) {
+            return cb(err);
+        }
+        if (entries[0]) {
+            return cb(null, `module.exports = __webpack_public_path__ + ${JSON.stringify(entries[0].files[0])};`);
+        }
+        return cb(null, null);
+    });
+};
--- a/scripts/copy-res.ts
+++ b/scripts/copy-res.ts
@ -6,7 +6,7 @@ import parseArgs from "minimist";
 import * as chokidar from "chokidar";
 import * as fs from "node:fs";
 import _ from "lodash";
-import * as loaderUtils from "loader-utils";
+import { util } from "webpack";
 import { Translations } from "matrix-web-i18n";

 const REACT_I18N_BASE_PATH = "node_modules/matrix-react-sdk/src/i18n/strings/";
@ -60,7 +60,7 @@ function prepareLangFile(lang: string, dest: string): [filename: string, json: s

    const json = JSON.stringify(translations, null, 4);
    const jsonBuffer = Buffer.from(json);
-    const digest = loaderUtils.getHashDigest(jsonBuffer, null, "hex", 7);
+    const digest = util.createHash("xxhash64").update(jsonBuffer).digest("hex").slice(0, 7);
    const filename = `${lang}.${digest}.json`;

    return [filename, json];
--- a/src/@types/loader-utils.d.ts
+++ b/src/@types/loader-utils.d.ts
@ -1,26 +0,0 @@
-/*
-Copyright 2023 The Matrix.org Foundation C.I.C.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-import * as LoaderUtils from "loader-utils";
-
-declare module "loader-utils" {
-    export function getHashDigest(
-        buffer: Buffer,
-        hashType: null,
-        digestType: LoaderUtils.DigestType,
-        maxLength: number,
-    ): string;
-}
--- a/src/vector/index.html
+++ b/src/vector/index.html
@ -49,11 +49,10 @@
        <% }
    } %>

-    <% for (var i=0; i < htmlWebpackPlugin.tags.headTags.length; i++) {
-        var tag = htmlWebpackPlugin.tags.headTags[i];
-        var path = tag.attributes && tag.attributes.href;
-        if (path.includes("/Inter/")) { %>
-            <link rel="preload" as="font" href=".<%= path %>" crossorigin="anonymous"/>
+    <% for (const tag of htmlWebpackPlugin.tags.headTags) {
+        let path = tag.attributes && tag.attributes.href;
+        if (path && path.includes("/Inter/")) { %>
+            <link rel="preload" as="font" href="<%= path %>" crossorigin="anonymous"/>
        <% }
    } %>

--- a/src/vector/index.ts
+++ b/src/vector/index.ts
@ -25,6 +25,9 @@ import { extractErrorMessageFromError } from "matrix-react-sdk/src/components/vi
 import { parseQsFromFragment } from "./url_utils";
 import "./modernizr";

+// Make setImmediate available in bundle
+import "setimmediate";
+
 // Require common CSS here; this will make webpack process it into bundle.css.
 // Our own CSS (which is themed) is imported via separate webpack entry points
 // in webpack.config.js
--- a/tsconfig.json
+++ b/tsconfig.json
@ -4,7 +4,7 @@
        "emitDecoratorMetadata": false,
        "resolveJsonModule": true,
        "esModuleInterop": true,
-        "module": "commonjs",
+        "module": "es2022",
        "moduleResolution": "node",
        "target": "es2016",
        "noUnusedLocals": true,
@ -28,6 +28,9 @@
        "./scripts/*.ts"
    ],
    "ts-node": {
-        "files": true
+        "files": true,
+        "moduleTypes": {
+            "*": "cjs"
+        }
    }
 }
--- a/webpack.config.js
+++ b/webpack.config.js
@ -6,7 +6,7 @@ const webpack = require("webpack");
 const HtmlWebpackPlugin = require("html-webpack-plugin");
 const MiniCssExtractPlugin = require("mini-css-extract-plugin");
 const TerserPlugin = require("terser-webpack-plugin");
-const OptimizeCSSAssetsPlugin = require("optimize-css-assets-webpack-plugin");
+const CssMinimizerPlugin = require("css-minimizer-webpack-plugin");
 const HtmlWebpackInjectPreload = require("@principalstudio/html-webpack-inject-preload");
 const { sentryWebpackPlugin } = require("@sentry/webpack-plugin");
 const crypto = require("crypto");
@ -81,7 +81,15 @@ function parseOverridesToReplacements(overrides) {
            // because the input is effectively defined by the person running the build, we don't
            // need to do anything special to protect against regex overrunning, etc.
            new RegExp(oldPath.replace(/\//g, "[\\/\\\\]").replace(/\./g, "\\.")),
-            path.resolve(__dirname, newPath),
+            function (resource) {
+                resource.request = path.resolve(__dirname, newPath);
+                resource.createData.resource = path.resolve(__dirname, newPath);
+                // Starting with Webpack 5 we also need to set the context as otherwise replacing
+                // files in e.g. matrix-react-sdk with files from element-web will try to resolve
+                // them within matrix-react-sdk (https://github.com/webpack/webpack/issues/17716)
+                resource.context = path.dirname(resource.request);
+                resource.createData.context = path.dirname(resource.createData.resource);
+            },
        );
    });
 }
@ -147,14 +155,6 @@ module.exports = (env, argv) => {

        bail: true,

-        node: {
-            // Mock out the NodeFS module: The opus decoder imports this wrongly.
-            fs: "empty",
-            net: "empty",
-            tls: "empty",
-            crypto: "empty",
-        },
-
        entry: {
            bundle: "./src/vector/index.ts",
            mobileguide: "./src/vector/mobile_guide/index.ts",
@ -192,14 +192,13 @@ module.exports = (env, argv) => {
                },
            },

-            // This fixes duplicate files showing up in chrome with sourcemaps enabled.
-            // See https://github.com/webpack/webpack/issues/7128 for more info.
-            namedModules: false,
+            // Readable IDs for better debugging
+            moduleIds: "named",

            // Minification is normally enabled by default for webpack in production mode, but
            // we use a CSS optimizer too and need to manage it ourselves.
            minimize: enableMinification,
-            minimizer: enableMinification ? [new TerserPlugin({}), new OptimizeCSSAssetsPlugin({})] : [],
+            minimizer: enableMinification ? [new TerserPlugin({}), new CssMinimizerPlugin()] : [],

            // Set the value of `process.env.NODE_ENV` for libraries like React
            // See also https://v4.webpack.js.org/configuration/optimization/#optimizationnodeenv
@ -242,6 +241,20 @@ module.exports = (env, argv) => {
                // Define a variable so the i18n stuff can load
                "$webapp": path.resolve(__dirname, "webapp"),
            },
+            fallback: {
+                // Mock out the NodeFS module: The opus decoder imports this wrongly.
+                "fs": false,
+                "net": false,
+                "tls": false,
+                "crypto": false,
+
+                // Polyfill needed by counterpart
+                "util": require.resolve("util/"),
+                // Polyfill needed by matrix-js-sdk/src/crypto
+                "buffer": require.resolve("buffer/"),
+                // Polyfill needed by sentry
+                "process/browser": require.resolve("process/browser"),
+            },
        },

        module: {
@ -268,15 +281,6 @@ module.exports = (env, argv) => {
                        replace: getThemesImports(),
                    },
                },
-                {
-                    test: /\.worker\.ts$/,
-                    loader: "worker-loader",
-                    options: {
-                        // Prevent bundling workers since CSP forbids loading them
-                        // from another origin.
-                        filename: "[hash].worker.js",
-                    },
-                },
                {
                    test: /\.(ts|js)x?$/,
                    include: (f) => {
@ -452,22 +456,17 @@ module.exports = (env, argv) => {
                    },
                },
                {
-                    // Special case the recorder worklet as it can't end up HMR'd, but the worker-loader
-                    // isn't good enough for us. Note that the worklet-loader is listed as "do not use",
-                    // however it seems to work fine for our purposes.
+                    // Ideally we should use the built-in worklet support in Webpack 5 with the syntax
+                    // described in https://github.com/webpack/webpack.js.org/issues/6869. However, this
+                    // doesn't currently appear to work with our public path setup. So we handle this
+                    // with a custom loader instead.
                    test: /RecorderWorklet\.ts$/,
                    type: "javascript/auto",
                    use: [
-                        // executed last -> first, for some reason.
                        {
-                            loader: "worklet-loader",
-                            options: {
-                                // Override name so we know what it is in the output.
-                                name: "recorder-worklet.[hash:7].js",
-                            },
+                            loader: path.resolve("./recorder-worklet-loader.js"),
                        },
                        {
-                            // TS -> JS because the worklet-loader won't do this for us.
                            loader: "babel-loader",
                        },
                    ],
@ -709,6 +708,7 @@ module.exports = (env, argv) => {
                        console.log(`::warning title=Sentry error::${err.message}`);
                    },
                }),
+
            new webpack.EnvironmentPlugin(["VERSION"]),

            new CopyWebpackPlugin({
@ -727,6 +727,13 @@ module.exports = (env, argv) => {
                    "contribute.json",
                ],
            }),
+
+            // Automatically load buffer & process modules as we use them without explicitly
+            // importing them
+            new webpack.ProvidePlugin({
+                Buffer: ["buffer", "Buffer"],
+                process: "process/browser",
+            }),
        ].filter(Boolean),

        output: {
@ -746,6 +753,15 @@ module.exports = (env, argv) => {

        // configuration for the webpack-dev-server
        devServer: {
+            client: {
+                overlay: {
+                    // Only show overlay on build errors as anything more can get annoying quickly
+                    errors: true,
+                    warnings: false,
+                    runtimeErrors: false,
+                },
+            },
+
            static: {
                // Where to serve static assets from
                directory: "./webapp",
--- a/yarn.lock
+++ b/yarn.lock