diff --git a/package.json b/package.json index 0808f5aecd..61a9a21815 100644 --- a/package.json +++ b/package.json @@ -105,6 +105,7 @@ "devDependencies": { "@babel/cli": "^7.10.5", "@babel/core": "^7.10.5", + "@babel/parser": "^7.11.0", "@babel/plugin-proposal-class-properties": "^7.10.4", "@babel/plugin-proposal-decorators": "^7.10.5", "@babel/plugin-proposal-export-default-from": "^7.10.4", @@ -117,6 +118,7 @@ "@babel/preset-react": "^7.10.4", "@babel/preset-typescript": "^7.10.4", "@babel/register": "^7.10.5", + "@babel/traverse": "^7.11.0", "@peculiar/webcrypto": "^1.1.2", "@types/classnames": "^2.2.10", "@types/counterpart": "^0.18.1", @@ -145,9 +147,7 @@ "eslint-plugin-flowtype": "^2.50.3", "eslint-plugin-react": "^7.20.3", "eslint-plugin-react-hooks": "^2.5.1", - "estree-walker": "^0.9.0", "file-loader": "^3.0.1", - "flow-parser": "0.57.3", "glob": "^5.0.15", "jest": "^24.9.0", "jest-canvas-mock": "^2.2.0", diff --git a/scripts/gen-i18n.js b/scripts/gen-i18n.js index a1823cdf50..c30ac62e3b 100755 --- a/scripts/gen-i18n.js +++ b/scripts/gen-i18n.js @@ -18,7 +18,7 @@ limitations under the License. /** * Regenerates the translations en_EN file by walking the source tree and - * parsing each file with flow-parser. Emits a JSON file with the + * parsing each file with the appropriate parser. Emits a JSON file with the * translatable strings mapped to themselves in the order they appeared * in the files and grouped by the file they appeared in. * @@ -29,8 +29,8 @@ const path = require('path'); const walk = require('walk'); -const flowParser = require('flow-parser'); -const estreeWalker = require('estree-walker'); +const parser = require("@babel/parser"); +const traverse = require("@babel/traverse"); const TRANSLATIONS_FUNCS = ['_t', '_td']; @@ -44,17 +44,9 @@ const OUTPUT_FILE = 'src/i18n/strings/en_EN.json'; // to a project that's actively maintained. const SEARCH_PATHS = ['src', 'res']; -const FLOW_PARSER_OPTS = { - esproposal_class_instance_fields: true, - esproposal_class_static_fields: true, - esproposal_decorators: true, - esproposal_export_star_as: true, - types: true, -}; - function getObjectValue(obj, key) { for (const prop of obj.properties) { - if (prop.key.type == 'Identifier' && prop.key.name == key) { + if (prop.key.type === 'Identifier' && prop.key.name === key) { return prop.value; } } @@ -62,11 +54,11 @@ function getObjectValue(obj, key) { } function getTKey(arg) { - if (arg.type == 'Literal') { + if (arg.type === 'Literal' || arg.type === "StringLiteral") { return arg.value; - } else if (arg.type == 'BinaryExpression' && arg.operator == '+') { + } else if (arg.type === 'BinaryExpression' && arg.operator === '+') { return getTKey(arg.left) + getTKey(arg.right); - } else if (arg.type == 'TemplateLiteral') { + } else if (arg.type === 'TemplateLiteral') { return arg.quasis.map((q) => { return q.value.raw; }).join(''); @@ -110,81 +102,112 @@ function getFormatStrings(str) { } function getTranslationsJs(file) { - const tree = flowParser.parse(fs.readFileSync(file, { encoding: 'utf8' }), FLOW_PARSER_OPTS); + const contents = fs.readFileSync(file, { encoding: 'utf8' }); const trs = new Set(); - estreeWalker.walk(tree, { - enter: function(node, parent) { - if ( - node.type == 'CallExpression' && - TRANSLATIONS_FUNCS.includes(node.callee.name) - ) { - const tKey = getTKey(node.arguments[0]); - // This happens whenever we call _t with non-literals (ie. whenever we've - // had to use a _td to compensate) so is expected. - if (tKey === null) return; + try { + const plugins = [ + // https://babeljs.io/docs/en/babel-parser#plugins + "classProperties", + "objectRestSpread", + "throwExpressions", + "exportDefaultFrom", + "decorators-legacy", + ]; - // check the format string against the args - // We only check _t: _td has no args - if (node.callee.name === '_t') { - try { - const placeholders = getFormatStrings(tKey); - for (const placeholder of placeholders) { - if (node.arguments.length < 2) { - throw new Error(`Placeholder found ('${placeholder}') but no substitutions given`); - } - const value = getObjectValue(node.arguments[1], placeholder); - if (value === null) { - throw new Error(`No value found for placeholder '${placeholder}'`); - } - } + if (file.endsWith(".js") || file.endsWith(".jsx")) { + // all JS is assumed to be flow or react + plugins.push("flow", "jsx"); + } else if (file.endsWith(".ts")) { + // TS can't use JSX unless it's a TSX file (otherwise angle casts fail) + plugins.push("typescript"); + } else if (file.endsWith(".tsx")) { + // When the file is a TSX file though, enable JSX parsing + plugins.push("typescript", "jsx"); + } - // Validate tag replacements - if (node.arguments.length > 2) { - const tagMap = node.arguments[2]; - for (const prop of tagMap.properties || []) { - if (prop.key.type === 'Literal') { - const tag = prop.key.value; - // RegExp same as in src/languageHandler.js - const regexp = new RegExp(`(<${tag}>(.*?)<\\/${tag}>|<${tag}>|<${tag}\\s*\\/>)`); - if (!tKey.match(regexp)) { - throw new Error(`No match for ${regexp} in ${tKey}`); + const babelParsed = parser.parse(contents, { + allowImportExportEverywhere: true, + errorRecovery: true, + sourceFilename: file, + tokens: true, + plugins, + }); + traverse.default(babelParsed, { + enter: (p) => { + const node = p.node; + if (p.isCallExpression() && node.callee && TRANSLATIONS_FUNCS.includes(node.callee.name)) { + const tKey = getTKey(node.arguments[0]); + + // This happens whenever we call _t with non-literals (ie. whenever we've + // had to use a _td to compensate) so is expected. + if (tKey === null) return; + + // check the format string against the args + // We only check _t: _td has no args + if (node.callee.name === '_t') { + try { + const placeholders = getFormatStrings(tKey); + for (const placeholder of placeholders) { + if (node.arguments.length < 2) { + throw new Error(`Placeholder found ('${placeholder}') but no substitutions given`); + } + const value = getObjectValue(node.arguments[1], placeholder); + if (value === null) { + throw new Error(`No value found for placeholder '${placeholder}'`); + } + } + + // Validate tag replacements + if (node.arguments.length > 2) { + const tagMap = node.arguments[2]; + for (const prop of tagMap.properties || []) { + if (prop.key.type === 'Literal') { + const tag = prop.key.value; + // RegExp same as in src/languageHandler.js + const regexp = new RegExp(`(<${tag}>(.*?)<\\/${tag}>|<${tag}>|<${tag}\\s*\\/>)`); + if (!tKey.match(regexp)) { + throw new Error(`No match for ${regexp} in ${tKey}`); + } } } } - } - } catch (e) { - console.log(); - console.error(`ERROR: ${file}:${node.loc.start.line} ${tKey}`); - console.error(e); - process.exit(1); - } - } - - let isPlural = false; - if (node.arguments.length > 1 && node.arguments[1].type == 'ObjectExpression') { - const countVal = getObjectValue(node.arguments[1], 'count'); - if (countVal) { - isPlural = true; - } - } - - if (isPlural) { - trs.add(tKey + "|other"); - const plurals = enPlurals[tKey]; - if (plurals) { - for (const pluralType of Object.keys(plurals)) { - trs.add(tKey + "|" + pluralType); + } catch (e) { + console.log(); + console.error(`ERROR: ${file}:${node.loc.start.line} ${tKey}`); + console.error(e); + process.exit(1); } } - } else { - trs.add(tKey); + + let isPlural = false; + if (node.arguments.length > 1 && node.arguments[1].type === 'ObjectExpression') { + const countVal = getObjectValue(node.arguments[1], 'count'); + if (countVal) { + isPlural = true; + } + } + + if (isPlural) { + trs.add(tKey + "|other"); + const plurals = enPlurals[tKey]; + if (plurals) { + for (const pluralType of Object.keys(plurals)) { + trs.add(tKey + "|" + pluralType); + } + } + } else { + trs.add(tKey); + } } - } - } - }); + }, + }); + } catch (e) { + console.error(e); + process.exit(1); + } return trs; } diff --git a/yarn.lock b/yarn.lock index 47248af9cd..edbd8c1a2c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -111,6 +111,15 @@ jsesc "^2.5.1" source-map "^0.5.0" +"@babel/generator@^7.11.0": + version "7.11.0" + resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.11.0.tgz#4b90c78d8c12825024568cbe83ee6c9af193585c" + integrity sha512-fEm3Uzw7Mc9Xi//qU20cBKatTfs2aOtKqmvy/Vm7RkJEGFQ4xc9myCfbXxqK//ZS8MR/ciOHw6meGASJuKmDfQ== + dependencies: + "@babel/types" "^7.11.0" + jsesc "^2.5.1" + source-map "^0.5.0" + "@babel/helper-annotate-as-pure@^7.10.1": version "7.10.1" resolved "https://registry.yarnpkg.com/@babel/helper-annotate-as-pure/-/helper-annotate-as-pure-7.10.1.tgz#f6d08acc6f70bbd59b436262553fb2e259a1a268" @@ -400,6 +409,13 @@ dependencies: "@babel/types" "^7.10.4" +"@babel/helper-split-export-declaration@^7.11.0": + version "7.11.0" + resolved "https://registry.yarnpkg.com/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.11.0.tgz#f8a491244acf6a676158ac42072911ba83ad099f" + integrity sha512-74Vejvp6mHkGE+m+k5vHY93FX2cAtrw1zXrZXRlG4l410Nm9PxfEiVTn1PjDPV5SnmieiueY4AFg2xqhNFuuZg== + dependencies: + "@babel/types" "^7.11.0" + "@babel/helper-validator-identifier@^7.10.1", "@babel/helper-validator-identifier@^7.10.3": version "7.10.3" resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.10.3.tgz#60d9847f98c4cea1b279e005fdb7c28be5412d15" @@ -466,6 +482,11 @@ resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.10.5.tgz#e7c6bf5a7deff957cec9f04b551e2762909d826b" integrity sha512-wfryxy4bE1UivvQKSQDU4/X6dr+i8bctjUjj8Zyt3DQy7NtPizJXT8M52nqpNKL+nq2PW8lxk4ZqLj0fD4B4hQ== +"@babel/parser@^7.11.0": + version "7.11.0" + resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.11.0.tgz#a9d7e11aead25d3b422d17b2c6502c8dddef6a5d" + integrity sha512-qvRvi4oI8xii8NllyEc4MDJjuZiNaRzyb7Y7lup1NqJV8TZHF4O27CcP+72WPn/k1zkgJ6WJfnIbk4jTsVAZHw== + "@babel/plugin-proposal-async-generator-functions@^7.10.4": version "7.10.5" resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-async-generator-functions/-/plugin-proposal-async-generator-functions-7.10.5.tgz#3491cabf2f7c179ab820606cec27fed15e0e8558" @@ -1213,6 +1234,21 @@ globals "^11.1.0" lodash "^4.17.19" +"@babel/traverse@^7.11.0": + version "7.11.0" + resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.11.0.tgz#9b996ce1b98f53f7c3e4175115605d56ed07dd24" + integrity sha512-ZB2V+LskoWKNpMq6E5UUCrjtDUh5IOTAyIl0dTjIEoXum/iKWkoIEKIRDnUucO6f+2FzNkE0oD4RLKoPIufDtg== + dependencies: + "@babel/code-frame" "^7.10.4" + "@babel/generator" "^7.11.0" + "@babel/helper-function-name" "^7.10.4" + "@babel/helper-split-export-declaration" "^7.11.0" + "@babel/parser" "^7.11.0" + "@babel/types" "^7.11.0" + debug "^4.1.0" + globals "^11.1.0" + lodash "^4.17.19" + "@babel/types@^7.0.0", "@babel/types@^7.10.1", "@babel/types@^7.10.2", "@babel/types@^7.3.0", "@babel/types@^7.4.0", "@babel/types@^7.4.4", "@babel/types@^7.7.0": version "7.10.2" resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.10.2.tgz#30283be31cad0dbf6fb00bd40641ca0ea675172d" @@ -1231,6 +1267,15 @@ lodash "^4.17.19" to-fast-properties "^2.0.0" +"@babel/types@^7.11.0": + version "7.11.0" + resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.11.0.tgz#2ae6bf1ba9ae8c3c43824e5861269871b206e90d" + integrity sha512-O53yME4ZZI0jO1EVGtF1ePGl0LHirG4P1ibcD80XyzZcKhcMFeCXmh4Xb1ifGBIV233Qg12x4rBfQgA+tmOukA== + dependencies: + "@babel/helper-validator-identifier" "^7.10.4" + lodash "^4.17.19" + to-fast-properties "^2.0.0" + "@cnakazawa/watch@^1.0.3": version "1.0.4" resolved "https://registry.yarnpkg.com/@cnakazawa/watch/-/watch-1.0.4.tgz#f864ae85004d0fcab6f50be9141c4da368d1656a" @@ -4178,11 +4223,6 @@ estree-walker@^0.6.1: resolved "https://registry.yarnpkg.com/estree-walker/-/estree-walker-0.6.1.tgz#53049143f40c6eb918b23671d1fe3219f3a1b362" integrity sha512-SqmZANLWS0mnatqbSfRP5g8OXZC12Fgg1IwNtLsyHDzJizORW4khDfjPqJZsemPWBB2uqykUah5YpQ6epsqC/w== -estree-walker@^0.9.0: - version "0.9.0" - resolved "https://registry.yarnpkg.com/estree-walker/-/estree-walker-0.9.0.tgz#9116372f09c02fd88fcafb0c04343631012a0aa6" - integrity sha512-12U47o7XHUX329+x3FzNVjCx3SHEzMF0nkDv7r/HnBzX/xNTKxajBk6gyygaxrAFtLj39219oMfbtxv4KpaOiA== - esutils@^2.0.2: version "2.0.3" resolved "https://registry.yarnpkg.com/esutils/-/esutils-2.0.3.tgz#74d2eb4de0b8da1293711910d50775b9b710ef64" @@ -4525,11 +4565,6 @@ flatted@^2.0.0: resolved "https://registry.yarnpkg.com/flatted/-/flatted-2.0.2.tgz#4575b21e2bcee7434aa9be662f4b7b5f9c2b5138" integrity sha512-r5wGx7YeOwNWNlCA0wQ86zKyDLMQr+/RB8xy74M4hTphfmjlijTSSXGuH8rnvKZnfT9i+75zmd8jcKdMR4O6jA== -flow-parser@0.57.3: - version "0.57.3" - resolved "https://registry.yarnpkg.com/flow-parser/-/flow-parser-0.57.3.tgz#b8d241a1b1cbae043afa7976e39f269988d8fe34" - integrity sha1-uNJBobHLrgQ6+nl2458mmYjY/jQ= - flush-write-stream@^1.0.0: version "1.1.1" resolved "https://registry.yarnpkg.com/flush-write-stream/-/flush-write-stream-1.1.1.tgz#8dd7d873a1babc207d94ead0c2e0e44276ebf2e8"