Few fixes to ftl-to-json and localize-ftl scripts (#3707)

* Omit msg-ref-only strings from Transifex JSON
* Fix msg-ref-only strings not included in translated .ftl files
* Update ftl-tx. Simplify localize-ftl script.
* Tweak FTL -> JSON conversion to produce a single file

js-build/ftl-to-json.mjs

@@ -6,25 +6,36 @@ import { ftlFileBaseNames as sourceFileBaseNames } from './config.js';
 import { onError, onProgress, onSuccess } from './utils.js';
 
 const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..');
+const TRANSIFEX_FILE_NAME = 'zotero.json';
 
 async function getJSON() {
 	const t1 = performance.now();
 	const sourceDir = join(ROOT, 'chrome', 'locale', 'en-US', 'zotero');
+	const destFile = join(sourceDir, TRANSIFEX_FILE_NAME);
+	let messagesMap = new Map();
+	
 	for (let sourceFileBaseName of sourceFileBaseNames) {
 		const sourceFile = join(sourceDir, sourceFileBaseName + '.ftl');
-		const destFile = join(sourceDir, sourceFileBaseName + '.json');
 		const ftl = await fs.readFile(sourceFile, 'utf8');
-		const json = ftlToJSON(ftl, { transformTerms: false, storeTermsInJSON: false });
+		const json = ftlToJSON(ftl, { transformTerms: false, storeTermsInJSON: false, skipRefOnly: true });
-		await fs.outputJSON(destFile, json, { spaces: '\t' });
+		Object.entries(json).forEach(([key, value]) => {
-		onProgress(destFile, destFile, 'json');
+			if (messagesMap.has(key)) {
-	}
+				throw new Error(`Duplicate key: ${key} found in file ${sourceFileBaseName}.ftl`);
-	const t2 = performance.now();
+			}
-		return ({
+			messagesMap.set(key, value);
-			action: 'ftl->json',
-			count: sourceFileBaseNames.length,
-			totalCount: sourceFileBaseNames.length,
-			processingTime: t2 - t1
 		});
+		onProgress(`${sourceFileBaseName}.ftl`, TRANSIFEX_FILE_NAME, 'ftl->json');
+	}
+	
+	const messagesJSON = Object.fromEntries(messagesMap);
+	await fs.outputJSON(destFile, messagesJSON, { spaces: '\t' });
+	const t2 = performance.now();
+	return ({
+		action: 'ftl->json',
+		count: sourceFileBaseNames.length,
+		totalCount: sourceFileBaseNames.length,
+		processingTime: t2 - t1
+	});
 }
 
 if (process.argv[1] === fileURLToPath(import.meta.url)) {

js-build/localize-ftl.mjs

@@ -1,15 +1,13 @@
-import { extractTerms, ftlToJSON, JSONToFtl } from 'ftl-tx';
+import { ftlToJSON, JSONToFtl } from 'ftl-tx';
 import fs from 'fs-extra';
 import { dirname, join } from 'path';
 import { fileURLToPath } from 'url';
 import { ftlFileBaseNames as sourceFileBaseNames } from './config.js';
 import { onError, onProgress, onSuccess } from './utils.js';
-import { exit } from 'process';
 
 const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..');
 const localesDir = join(ROOT, 'chrome', 'locale');
-const sourceDir = join(localesDir, 'en-US', 'zotero');
+const TRANSIFEX_FILE_NAME = 'zotero.json';
-const termsSourceFTLPath = join(ROOT, 'app', 'assets', 'branding', 'locale', 'brand.ftl');
 
 function getLocaleDir(locale) {
 	return join(localesDir, locale, 'zotero');
@@ -18,13 +16,6 @@ function getLocaleDir(locale) {
 async function getFTL() {
 	const t1 = performance.now();
 	
-	if (!(await fs.pathExists(termsSourceFTLPath))) {
-		console.error(`Required file ${termsSourceFTLPath} does not exist`);
-		exit(1);
-	}
-	
-	const terms = extractTerms(await fs.readFile(termsSourceFTLPath, 'utf-8'));
-	
 	const foundLocales = (await fs.readdir(localesDir, { withFileTypes: true }))
 		.filter(dirent => dirent.isDirectory())
 		.map(dirent => dirent.name)
@@ -32,54 +23,65 @@ async function getFTL() {
 		.filter(name => /^[a-z]{2}(-[A-Z]{2})?$/.test(name));
 	
 	let count = 0;
-	for (let sourceFileBaseName of sourceFileBaseNames) {
+	for (let locale of foundLocales) {
-		const fallbackJSONPath = join(sourceDir, sourceFileBaseName + '.json');
+		// Skip source locale
-		if (!(await fs.pathExists(fallbackJSONPath))) {
+		if (locale == 'en-US') {
-			console.error(`File ${fallbackJSONPath} does not exist -- please run 'ftl-to-json' first`);
+			continue;
-			exit(1);
 		}
-		
+
-		const fallbackJSON = await fs.readJSON(fallbackJSONPath);
+		const jsonFilePath = join(getLocaleDir(locale), TRANSIFEX_FILE_NAME);
-		
+		let jsonFromTransifex = {};
-		for (let locale of foundLocales) {
+		try {
-			// Skip source locale
+			const json = await fs.readJSON(jsonFilePath);
-			if (locale == 'en-US') {
+			jsonFromTransifex = json;
-				continue;
+		}
-			}
+		catch (e) {
-			
+			// no .json file from transifex
+		}
+
+		for (let sourceFileBaseName of sourceFileBaseNames) {
 			const ftlFilePath = join(getLocaleDir(locale), sourceFileBaseName + '.ftl');
 			let jsonFromLocalFTL = {};
 			try {
 				const ftl = await fs.readFile(ftlFilePath, 'utf8');
-				jsonFromLocalFTL = ftlToJSON(ftl, { transformTerms: false, storeTermsInJSON: false });
+				jsonFromLocalFTL = ftlToJSON(ftl);
 			}
 			catch (e) {
 				// no local .ftl file
 			}
-			
+			let jsonFromEnUSFTL = {};
-			const jsonFilePath = join(getLocaleDir(locale), sourceFileBaseName + `.json`);
-			let jsonFromTransifex = {};
 			try {
-				const json = await fs.readJSON(jsonFilePath);
+				const enUSFtlPath = join(getLocaleDir('en-US'), sourceFileBaseName + '.ftl');
-				jsonFromTransifex = json;
+				const ftl = await fs.readFile(enUSFtlPath, 'utf8');
+				jsonFromEnUSFTL = ftlToJSON(ftl);
 			}
 			catch (e) {
-				// no .json file from transifex
+				console.warn(`No en-US .ftl file for ${sourceFileBaseName}.`);
 			}
-			
+			const mergedSourceJSON = { ...jsonFromEnUSFTL, ...jsonFromLocalFTL };
-			const mergedJSON = { ...fallbackJSON, ...jsonFromLocalFTL, ...jsonFromTransifex };
+			const sourceKeys = Object.keys(mergedSourceJSON);
-			const ftl = JSONToFtl(mergedJSON, { addTermsToFTL: false, storeTermsInJSON: false, transformTerms: false, terms });
+			const translated = new Map();
-			
+
+			for (let key of sourceKeys) {
+				if (key in jsonFromTransifex) {
+					translated.set(key, jsonFromTransifex[key]);
+				}
+				else {
+					translated.set(key, mergedSourceJSON[key]);
+				}
+			}
+
+			const ftl = JSONToFtl(Object.fromEntries(translated));
 			const outFtlPath = join(getLocaleDir(locale), sourceFileBaseName + '.ftl');
 			await fs.outputFile(outFtlPath, ftl);
-			onProgress(outFtlPath, outFtlPath, 'ftl');
+			onProgress(`${locale}/${sourceFileBaseName}.ftl`, null, 'localize');
 			count++;
 		}
 	}
 	
 	const t2 = performance.now();
 	return ({
-		action: 'ftl',
+		action: 'localize',
 		count,
 		totalCount: count,
 		processingTime: t2 - t1

js-build/utils.js

@@ -30,7 +30,7 @@ function onProgress(sourcefile, outfile, operation) {
 	if ('isError' in global && global.isError) {
 		return;
 	}
-	if (NODE_ENV == 'debug') {
+	if (NODE_ENV === 'debug' && outfile) {
 		console.log(`${colors.blue(`[${operation}]`)} ${sourcefile} -> ${outfile}`);
 	}
 	else {

package-lock.json

@@ -40,7 +40,7 @@
         "eslint-plugin-react": "^7.28.0",
         "eslint-plugin-react-hooks": "^4.0.4",
         "fs-extra": "^3.0.1",
-        "ftl-tx": "^0.6.0",
+        "ftl-tx": "^0.9.0",
         "globby": "^6.1.0",
         "jspath": "^0.4.0",
         "mocha": "^10.4.0",
@@ -3667,9 +3667,9 @@
       }
     },
     "node_modules/ftl-tx": {
-      "version": "0.6.0",
+      "version": "0.9.0",
-      "resolved": "https://registry.npmjs.org/ftl-tx/-/ftl-tx-0.6.0.tgz",
+      "resolved": "https://registry.npmjs.org/ftl-tx/-/ftl-tx-0.9.0.tgz",
-      "integrity": "sha512-w7s0p6RNsaUKpiuQZ5Q0KIyiX2JIDItqI4qRKt49u0aMqdGp0gDc5G5Qnubgu6l2ww5PDt4wsyL7A1iZo1VGMQ==",
+      "integrity": "sha512-cxjOfLulCEPL6K0boxTiHCYNs3UahgmfrC+u3a6oV665EWuR6LnZRWID2UKWv6JBIB4YU4k06Y5re5uA7HpEGg==",
       "dev": true,
       "dependencies": {
         "@fluent/syntax": "^0.19.0"
@@ -10588,9 +10588,9 @@
       }
     },
     "ftl-tx": {
-      "version": "0.6.0",
+      "version": "0.9.0",
-      "resolved": "https://registry.npmjs.org/ftl-tx/-/ftl-tx-0.6.0.tgz",
+      "resolved": "https://registry.npmjs.org/ftl-tx/-/ftl-tx-0.9.0.tgz",
-      "integrity": "sha512-w7s0p6RNsaUKpiuQZ5Q0KIyiX2JIDItqI4qRKt49u0aMqdGp0gDc5G5Qnubgu6l2ww5PDt4wsyL7A1iZo1VGMQ==",
+      "integrity": "sha512-cxjOfLulCEPL6K0boxTiHCYNs3UahgmfrC+u3a6oV665EWuR6LnZRWID2UKWv6JBIB4YU4k06Y5re5uA7HpEGg==",
       "dev": true,
       "requires": {
         "@fluent/syntax": "^0.19.0"

package.json

@@ -49,7 +49,7 @@
     "eslint-plugin-react": "^7.28.0",
     "eslint-plugin-react-hooks": "^4.0.4",
     "fs-extra": "^3.0.1",
-    "ftl-tx": "^0.6.0",
+    "ftl-tx": "^0.9.0",
     "globby": "^6.1.0",
     "jspath": "^0.4.0",
     "mocha": "^10.4.0",