Merge pull request #5077 from matrix-org/travis/i18n

Replace i18n generation script with something matching our project

package.json

@@ -105,6 +105,7 @@
   "devDependencies": {
     "@babel/cli": "^7.10.5",
     "@babel/core": "^7.10.5",
+    "@babel/parser": "^7.11.0",
     "@babel/plugin-proposal-class-properties": "^7.10.4",
     "@babel/plugin-proposal-decorators": "^7.10.5",
     "@babel/plugin-proposal-export-default-from": "^7.10.4",
@@ -117,6 +118,7 @@
     "@babel/preset-react": "^7.10.4",
     "@babel/preset-typescript": "^7.10.4",
     "@babel/register": "^7.10.5",
+    "@babel/traverse": "^7.11.0",
     "@peculiar/webcrypto": "^1.1.2",
     "@types/classnames": "^2.2.10",
     "@types/counterpart": "^0.18.1",
@@ -145,9 +147,7 @@
     "eslint-plugin-flowtype": "^2.50.3",
     "eslint-plugin-react": "^7.20.3",
     "eslint-plugin-react-hooks": "^2.5.1",
-    "estree-walker": "^0.9.0",
     "file-loader": "^3.0.1",
-    "flow-parser": "0.57.3",
     "glob": "^5.0.15",
     "jest": "^24.9.0",
     "jest-canvas-mock": "^2.2.0",

scripts/gen-i18n.js

@@ -18,7 +18,7 @@ limitations under the License.
 
 /**
  * Regenerates the translations en_EN file by walking the source tree and
- * parsing each file with flow-parser. Emits a JSON file with the
+ * parsing each file with the appropriate parser. Emits a JSON file with the
  * translatable strings mapped to themselves in the order they appeared
  * in the files and grouped by the file they appeared in.
  *
@@ -29,8 +29,8 @@ const path = require('path');
 
 const walk = require('walk');
 
-const flowParser = require('flow-parser');
-const estreeWalker = require('estree-walker');
+const parser = require("@babel/parser");
+const traverse = require("@babel/traverse");
 
 const TRANSLATIONS_FUNCS = ['_t', '_td'];
 
@@ -44,17 +44,9 @@ const OUTPUT_FILE = 'src/i18n/strings/en_EN.json';
 // to a project that's actively maintained.
 const SEARCH_PATHS = ['src', 'res'];
 
-const FLOW_PARSER_OPTS = {
-  esproposal_class_instance_fields: true,
-  esproposal_class_static_fields: true,
-  esproposal_decorators: true,
-  esproposal_export_star_as: true,
-  types: true,
-};
-
 function getObjectValue(obj, key) {
     for (const prop of obj.properties) {
-        if (prop.key.type == 'Identifier' && prop.key.name == key) {
+        if (prop.key.type === 'Identifier' && prop.key.name === key) {
             return prop.value;
         }
     }
@@ -62,11 +54,11 @@ function getObjectValue(obj, key) {
 }
 
 function getTKey(arg) {
-    if (arg.type == 'Literal') {
+    if (arg.type === 'Literal' || arg.type === "StringLiteral") {
         return arg.value;
-    } else if (arg.type == 'BinaryExpression' && arg.operator == '+') {
+    } else if (arg.type === 'BinaryExpression' && arg.operator === '+') {
         return getTKey(arg.left) + getTKey(arg.right);
-    } else if (arg.type == 'TemplateLiteral') {
+    } else if (arg.type === 'TemplateLiteral') {
         return arg.quasis.map((q) => {
             return q.value.raw;
         }).join('');
@@ -110,81 +102,112 @@ function getFormatStrings(str) {
 }
 
 function getTranslationsJs(file) {
-    const tree = flowParser.parse(fs.readFileSync(file, { encoding: 'utf8' }), FLOW_PARSER_OPTS);
+    const contents = fs.readFileSync(file, { encoding: 'utf8' });
 
     const trs = new Set();
 
-    estreeWalker.walk(tree, {
-        enter: function(node, parent) {
-            if (
-                node.type == 'CallExpression' &&
-                TRANSLATIONS_FUNCS.includes(node.callee.name)
-            ) {
-                const tKey = getTKey(node.arguments[0]);
-                // This happens whenever we call _t with non-literals (ie. whenever we've
-                // had to use a _td to compensate) so is expected.
-                if (tKey === null) return;
+    try {
+        const plugins = [
+            // https://babeljs.io/docs/en/babel-parser#plugins
+            "classProperties",
+            "objectRestSpread",
+            "throwExpressions",
+            "exportDefaultFrom",
+            "decorators-legacy",
+        ];
 
-                // check the format string against the args
-                // We only check _t: _td has no args
-                if (node.callee.name === '_t') {
-                    try {
-                        const placeholders = getFormatStrings(tKey);
-                        for (const placeholder of placeholders) {
-                            if (node.arguments.length < 2) {
-                                throw new Error(`Placeholder found ('${placeholder}') but no substitutions given`);
-                            }
-                            const value = getObjectValue(node.arguments[1], placeholder);
-                            if (value === null) {
-                                throw new Error(`No value found for placeholder '${placeholder}'`);
-                            }
-                        }
+        if (file.endsWith(".js") || file.endsWith(".jsx")) {
+            // all JS is assumed to be flow or react
+            plugins.push("flow", "jsx");
+        } else if (file.endsWith(".ts")) {
+            // TS can't use JSX unless it's a TSX file (otherwise angle casts fail)
+            plugins.push("typescript");
+        } else if (file.endsWith(".tsx")) {
+            // When the file is a TSX file though, enable JSX parsing
+            plugins.push("typescript", "jsx");
+        }
 
-                        // Validate tag replacements
-                        if (node.arguments.length > 2) {
-                            const tagMap = node.arguments[2];
-                            for (const prop of tagMap.properties || []) {
-                                if (prop.key.type === 'Literal') {
-                                    const tag = prop.key.value;
-                                    // RegExp same as in src/languageHandler.js
-                                    const regexp = new RegExp(`(<${tag}>(.*?)<\\/${tag}>|<${tag}>|<${tag}\\s*\\/>)`);
-                                    if (!tKey.match(regexp)) {
-                                        throw new Error(`No match for ${regexp} in ${tKey}`);
+        const babelParsed = parser.parse(contents, {
+            allowImportExportEverywhere: true,
+            errorRecovery: true,
+            sourceFilename: file,
+            tokens: true,
+            plugins,
+        });
+        traverse.default(babelParsed, {
+            enter: (p) => {
+                const node = p.node;
+                if (p.isCallExpression() && node.callee && TRANSLATIONS_FUNCS.includes(node.callee.name)) {
+                    const tKey = getTKey(node.arguments[0]);
+
+                    // This happens whenever we call _t with non-literals (ie. whenever we've
+                    // had to use a _td to compensate) so is expected.
+                    if (tKey === null) return;
+
+                    // check the format string against the args
+                    // We only check _t: _td has no args
+                    if (node.callee.name === '_t') {
+                        try {
+                            const placeholders = getFormatStrings(tKey);
+                            for (const placeholder of placeholders) {
+                                if (node.arguments.length < 2) {
+                                    throw new Error(`Placeholder found ('${placeholder}') but no substitutions given`);
+                                }
+                                const value = getObjectValue(node.arguments[1], placeholder);
+                                if (value === null) {
+                                    throw new Error(`No value found for placeholder '${placeholder}'`);
+                                }
+                            }
+
+                            // Validate tag replacements
+                            if (node.arguments.length > 2) {
+                                const tagMap = node.arguments[2];
+                                for (const prop of tagMap.properties || []) {
+                                    if (prop.key.type === 'Literal') {
+                                        const tag = prop.key.value;
+                                        // RegExp same as in src/languageHandler.js
+                                        const regexp = new RegExp(`(<${tag}>(.*?)<\\/${tag}>|<${tag}>|<${tag}\\s*\\/>)`);
+                                        if (!tKey.match(regexp)) {
+                                            throw new Error(`No match for ${regexp} in ${tKey}`);
+                                        }
                                     }
                                 }
                             }
-                        }
 
-                    } catch (e) {
-                        console.log();
-                        console.error(`ERROR: ${file}:${node.loc.start.line} ${tKey}`);
-                        console.error(e);
-                        process.exit(1);
-                    }
-                }
-
-                let isPlural = false;
-                if (node.arguments.length > 1 && node.arguments[1].type == 'ObjectExpression') {
-                    const countVal = getObjectValue(node.arguments[1], 'count');
-                    if (countVal) {
-                        isPlural = true;
-                    }
-                }
-
-                if (isPlural) {
-                    trs.add(tKey + "|other");
-                    const plurals = enPlurals[tKey];
-                    if (plurals) {
-                        for (const pluralType of Object.keys(plurals)) {
-                            trs.add(tKey + "|" + pluralType);
+                        } catch (e) {
+                            console.log();
+                            console.error(`ERROR: ${file}:${node.loc.start.line} ${tKey}`);
+                            console.error(e);
+                            process.exit(1);
                         }
                     }
-                } else {
-                    trs.add(tKey);
+
+                    let isPlural = false;
+                    if (node.arguments.length > 1 && node.arguments[1].type === 'ObjectExpression') {
+                        const countVal = getObjectValue(node.arguments[1], 'count');
+                        if (countVal) {
+                            isPlural = true;
+                        }
+                    }
+
+                    if (isPlural) {
+                        trs.add(tKey + "|other");
+                        const plurals = enPlurals[tKey];
+                        if (plurals) {
+                            for (const pluralType of Object.keys(plurals)) {
+                                trs.add(tKey + "|" + pluralType);
+                            }
+                        }
+                    } else {
+                        trs.add(tKey);
+                    }
                 }
-            }
-        }
-    });
+            },
+        });
+    } catch (e) {
+        console.error(e);
+        process.exit(1);
+    }
 
     return trs;
 }

yarn.lock

@@ -111,6 +111,15 @@
     jsesc "^2.5.1"
     source-map "^0.5.0"
 
+"@babel/generator@^7.11.0":
+  version "7.11.0"
+  resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.11.0.tgz#4b90c78d8c12825024568cbe83ee6c9af193585c"
+  integrity sha512-fEm3Uzw7Mc9Xi//qU20cBKatTfs2aOtKqmvy/Vm7RkJEGFQ4xc9myCfbXxqK//ZS8MR/ciOHw6meGASJuKmDfQ==
+  dependencies:
+    "@babel/types" "^7.11.0"
+    jsesc "^2.5.1"
+    source-map "^0.5.0"
+
 "@babel/helper-annotate-as-pure@^7.10.1":
   version "7.10.1"
   resolved "https://registry.yarnpkg.com/@babel/helper-annotate-as-pure/-/helper-annotate-as-pure-7.10.1.tgz#f6d08acc6f70bbd59b436262553fb2e259a1a268"
@@ -400,6 +409,13 @@
   dependencies:
     "@babel/types" "^7.10.4"
 
+"@babel/helper-split-export-declaration@^7.11.0":
+  version "7.11.0"
+  resolved "https://registry.yarnpkg.com/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.11.0.tgz#f8a491244acf6a676158ac42072911ba83ad099f"
+  integrity sha512-74Vejvp6mHkGE+m+k5vHY93FX2cAtrw1zXrZXRlG4l410Nm9PxfEiVTn1PjDPV5SnmieiueY4AFg2xqhNFuuZg==
+  dependencies:
+    "@babel/types" "^7.11.0"
+
 "@babel/helper-validator-identifier@^7.10.1", "@babel/helper-validator-identifier@^7.10.3":
   version "7.10.3"
   resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.10.3.tgz#60d9847f98c4cea1b279e005fdb7c28be5412d15"
@@ -466,6 +482,11 @@
   resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.10.5.tgz#e7c6bf5a7deff957cec9f04b551e2762909d826b"
   integrity sha512-wfryxy4bE1UivvQKSQDU4/X6dr+i8bctjUjj8Zyt3DQy7NtPizJXT8M52nqpNKL+nq2PW8lxk4ZqLj0fD4B4hQ==
 
+"@babel/parser@^7.11.0":
+  version "7.11.0"
+  resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.11.0.tgz#a9d7e11aead25d3b422d17b2c6502c8dddef6a5d"
+  integrity sha512-qvRvi4oI8xii8NllyEc4MDJjuZiNaRzyb7Y7lup1NqJV8TZHF4O27CcP+72WPn/k1zkgJ6WJfnIbk4jTsVAZHw==
+
 "@babel/plugin-proposal-async-generator-functions@^7.10.4":
   version "7.10.5"
   resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-async-generator-functions/-/plugin-proposal-async-generator-functions-7.10.5.tgz#3491cabf2f7c179ab820606cec27fed15e0e8558"
@@ -1213,6 +1234,21 @@
     globals "^11.1.0"
     lodash "^4.17.19"
 
+"@babel/traverse@^7.11.0":
+  version "7.11.0"
+  resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.11.0.tgz#9b996ce1b98f53f7c3e4175115605d56ed07dd24"
+  integrity sha512-ZB2V+LskoWKNpMq6E5UUCrjtDUh5IOTAyIl0dTjIEoXum/iKWkoIEKIRDnUucO6f+2FzNkE0oD4RLKoPIufDtg==
+  dependencies:
+    "@babel/code-frame" "^7.10.4"
+    "@babel/generator" "^7.11.0"
+    "@babel/helper-function-name" "^7.10.4"
+    "@babel/helper-split-export-declaration" "^7.11.0"
+    "@babel/parser" "^7.11.0"
+    "@babel/types" "^7.11.0"
+    debug "^4.1.0"
+    globals "^11.1.0"
+    lodash "^4.17.19"
+
 "@babel/types@^7.0.0", "@babel/types@^7.10.1", "@babel/types@^7.10.2", "@babel/types@^7.3.0", "@babel/types@^7.4.0", "@babel/types@^7.4.4", "@babel/types@^7.7.0":
   version "7.10.2"
   resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.10.2.tgz#30283be31cad0dbf6fb00bd40641ca0ea675172d"
@@ -1231,6 +1267,15 @@
     lodash "^4.17.19"
     to-fast-properties "^2.0.0"
 
+"@babel/types@^7.11.0":
+  version "7.11.0"
+  resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.11.0.tgz#2ae6bf1ba9ae8c3c43824e5861269871b206e90d"
+  integrity sha512-O53yME4ZZI0jO1EVGtF1ePGl0LHirG4P1ibcD80XyzZcKhcMFeCXmh4Xb1ifGBIV233Qg12x4rBfQgA+tmOukA==
+  dependencies:
+    "@babel/helper-validator-identifier" "^7.10.4"
+    lodash "^4.17.19"
+    to-fast-properties "^2.0.0"
+
 "@cnakazawa/watch@^1.0.3":
   version "1.0.4"
   resolved "https://registry.yarnpkg.com/@cnakazawa/watch/-/watch-1.0.4.tgz#f864ae85004d0fcab6f50be9141c4da368d1656a"
@@ -4178,11 +4223,6 @@ estree-walker@^0.6.1:
   resolved "https://registry.yarnpkg.com/estree-walker/-/estree-walker-0.6.1.tgz#53049143f40c6eb918b23671d1fe3219f3a1b362"
   integrity sha512-SqmZANLWS0mnatqbSfRP5g8OXZC12Fgg1IwNtLsyHDzJizORW4khDfjPqJZsemPWBB2uqykUah5YpQ6epsqC/w==
 
-estree-walker@^0.9.0:
-  version "0.9.0"
-  resolved "https://registry.yarnpkg.com/estree-walker/-/estree-walker-0.9.0.tgz#9116372f09c02fd88fcafb0c04343631012a0aa6"
-  integrity sha512-12U47o7XHUX329+x3FzNVjCx3SHEzMF0nkDv7r/HnBzX/xNTKxajBk6gyygaxrAFtLj39219oMfbtxv4KpaOiA==
-
 esutils@^2.0.2:
   version "2.0.3"
   resolved "https://registry.yarnpkg.com/esutils/-/esutils-2.0.3.tgz#74d2eb4de0b8da1293711910d50775b9b710ef64"
@@ -4525,11 +4565,6 @@ flatted@^2.0.0:
   resolved "https://registry.yarnpkg.com/flatted/-/flatted-2.0.2.tgz#4575b21e2bcee7434aa9be662f4b7b5f9c2b5138"
   integrity sha512-r5wGx7YeOwNWNlCA0wQ86zKyDLMQr+/RB8xy74M4hTphfmjlijTSSXGuH8rnvKZnfT9i+75zmd8jcKdMR4O6jA==
 
-flow-parser@0.57.3:
-  version "0.57.3"
-  resolved "https://registry.yarnpkg.com/flow-parser/-/flow-parser-0.57.3.tgz#b8d241a1b1cbae043afa7976e39f269988d8fe34"
-  integrity sha1-uNJBobHLrgQ6+nl2458mmYjY/jQ=
-
 flush-write-stream@^1.0.0:
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/flush-write-stream/-/flush-write-stream-1.1.1.tgz#8dd7d873a1babc207d94ead0c2e0e44276ebf2e8"