Use LocaleMatcher to resolve system preferred locales

app/locale.ts

@@ -4,21 +4,13 @@
 import { join } from 'path';
 import { readFileSync } from 'fs';
 import { merge } from 'lodash';
+import * as LocaleMatcher from '@formatjs/intl-localematcher';
 import { setupI18n } from '../ts/util/setupI18n';
 
 import type { LoggerType } from '../ts/types/Logging';
 import type { LocaleMessagesType } from '../ts/types/I18N';
 import type { LocalizerType } from '../ts/types/Util';
 
-function removeRegion(locale: string): string {
-  const match = /^([^-]+)(-.+)$/.exec(locale);
-  if (match) {
-    return match[1];
-  }
-
-  return locale;
-}
-
 function getLocaleMessages(locale: string): LocaleMessagesType {
   const targetFile = join(__dirname, '..', '_locales', locale, 'messages.json');
 
@@ -53,47 +45,43 @@ export function load({
   logger,
 }: {
   preferredSystemLocales: Array<string>;
-  logger: Pick<LoggerType, 'error' | 'warn' | 'info'>;
+  logger: Pick<LoggerType, 'warn' | 'info'>;
 }): LocaleType {
   if (preferredSystemLocales == null) {
-    throw new TypeError('`preferredSystemLocales` is required');
+    throw new TypeError('locale: `preferredSystemLocales` is required');
   }
-
-  if (!logger || !logger.error) {
-    throw new TypeError('`logger.error` is required');
+  if (!logger.info) {
+    throw new TypeError('locale: `logger.info` is required');
   }
   if (!logger.warn) {
-    throw new TypeError('`logger.warn` is required');
+    throw new TypeError('locale: `logger.warn` is required');
   }
 
   if (preferredSystemLocales.length === 0) {
-    logger.warn('`preferredSystemLocales` was empty');
+    logger.warn('locale: `preferredSystemLocales` was empty');
   }
 
-  const english = getLocaleMessages('en');
+  const availableLocales = JSON.parse(
+    readFileSync(
+      join(__dirname, '..', 'build', 'available-locales.json'),
+      'utf-8'
+    )
+  ) as Array<string>;
 
-  for (const locale of preferredSystemLocales) {
-    try {
-      logger.info(`Loading preferred system locale: '${locale}'`);
-      return finalize(getLocaleMessages(locale), english, locale);
-    } catch (e) {
-      logger.warn(
-        `Problem loading messages for locale '${locale}', ${e.toString()}`
-      );
-    }
+  logger.info('locale: Supported locales:', availableLocales.join(', '));
+  logger.info('locale: Preferred locales: ', preferredSystemLocales.join(', '));
 
-    const languageOnly = removeRegion(locale);
-    try {
-      logger.warn(`Falling back to parent language: '${languageOnly}'`);
-      // Note: messages are from parent language, but we still keep the region
-      return finalize(getLocaleMessages(languageOnly), english, locale);
-    } catch (e) {
-      logger.error(
-        `Problem loading messages for parent locale '${languageOnly}'`
-      );
-    }
-  }
+  const matchedLocale = LocaleMatcher.match(
+    preferredSystemLocales,
+    availableLocales,
+    'en',
+    { algorithm: 'best fit' }
+  );
 
-  logger.warn("Falling back to 'en' locale");
-  return finalize(english, english, 'en');
+  logger.info(`locale: Matched locale: ${matchedLocale}`);
+
+  const matchedLocaleMessages = getLocaleMessages(matchedLocale);
+  const englishMessages = getLocaleMessages('en');
+
+  return finalize(matchedLocaleMessages, englishMessages, matchedLocale);
 }

app/main.ts

@@ -1563,11 +1563,7 @@ ipc.on('database-readonly', (_event: Electron.Event, error: string) => {
 function loadPreferredSystemLocales(): Array<string> {
   return getEnvironment() === Environment.Test
     ? ['en']
-    : [
-        // TODO(DESKTOP-4929): Temp fix to inherit Chromium's l10n_util logic
-        app.getLocale(),
-        ...app.getPreferredSystemLanguages(),
-      ];
+    : app.getPreferredSystemLanguages();
 }
 
 async function getDefaultLoginItemSettings(): Promise<LoginItemSettingsOptions> {

build/available-locales.json

@@ -0,0 +1,71 @@
+[
+  "af-ZA",
+  "ar",
+  "az-AZ",
+  "bg-BG",
+  "bn-BD",
+  "bs-BA",
+  "ca",
+  "cs",
+  "da",
+  "de",
+  "el",
+  "en",
+  "es",
+  "et-EE",
+  "eu",
+  "fa-IR",
+  "fi",
+  "fr",
+  "ga-IE",
+  "gl-ES",
+  "gu-IN",
+  "he",
+  "hi-IN",
+  "hr-HR",
+  "hu",
+  "id",
+  "it",
+  "ja",
+  "ka-GE",
+  "kk-KZ",
+  "km-KH",
+  "kn-IN",
+  "ko",
+  "ky-KG",
+  "lt-LT",
+  "lv-LV",
+  "mk-MK",
+  "ml-IN",
+  "mr-IN",
+  "ms",
+  "my",
+  "nb",
+  "nl",
+  "pa-IN",
+  "pl",
+  "pt-BR",
+  "pt-PT",
+  "ro-RO",
+  "ru",
+  "sk-SK",
+  "sl-SI",
+  "sq-AL",
+  "sr-RS",
+  "sr-YR",
+  "sv",
+  "sw",
+  "ta-IN",
+  "te-IN",
+  "th",
+  "tl-PH",
+  "tr",
+  "ug",
+  "uk-UA",
+  "ur",
+  "vi",
+  "yue",
+  "zh-CN",
+  "zh-HK",
+  "zh-TW"
+]

package.json

@@ -20,7 +20,7 @@
     "build-release": "yarn run build",
     "sign-release": "node ts/updater/generateSignature.js",
     "notarize": "echo 'No longer necessary'",
-    "get-strings": "node ts/scripts/get-strings.js && node ts/scripts/gen-nsis-script.js",
+    "get-strings": "node ts/scripts/get-strings.js && node ts/scripts/gen-nsis-script.js && node ts/scripts/gen-locales-config.js",
     "push-strings": "node ts/scripts/remove-strings.js && node ts/scripts/push-strings.js",
     "get-expire-time": "node ts/scripts/get-expire-time.js",
     "copy-components": "node ts/scripts/copy.js",
@@ -458,6 +458,7 @@
       "fonts/**",
       "sounds/*",
       "build/icons",
+      "build/available-locales.json",
       "node_modules/**",
       "!node_modules/underscore/**",
       "!node_modules/emoji-datasource/emoji_pretty.json",

ts/scripts/gen-locales-config.ts

@@ -0,0 +1,62 @@
+// Copyright 2023 Signal Messenger, LLC
+// SPDX-License-Identifier: AGPL-3.0-only
+
+import fs from 'fs/promises';
+import path from 'path';
+import fastGlob from 'fast-glob';
+import * as LocaleMatcher from '@formatjs/intl-localematcher';
+
+const ROOT_DIR = path.join(__dirname, '..', '..');
+
+function matches(input: string, expected: string) {
+  const match = LocaleMatcher.match([input], [expected], 'en', {
+    algorithm: 'best fit',
+  });
+  return match === expected;
+}
+
+async function main() {
+  const dirEntries = await fastGlob('_locales/*', {
+    cwd: ROOT_DIR,
+    onlyDirectories: true,
+  });
+
+  const localeDirNames = [];
+
+  for (const dirEntry of dirEntries) {
+    const dirName = path.basename(dirEntry);
+    const locale = new Intl.Locale(dirName);
+
+    // Smartling doesn't always use the correct language tag, so this check and
+    // reverse check are to make sure we don't accidentally add a locale that
+    // doesn't match its directory name (using LocaleMatcher).
+    //
+    // If this check ever fails, we may need to update our get-strings script to
+    // manually rename language tags before writing them to disk.
+    //
+    // Such is the case for Smartling's "zh-YU" locale, which we renamed to
+    // "yue" to match the language tag used by... everyone else.
+
+    if (!matches(dirName, locale.baseName)) {
+      throw new Error(
+        `Matched locale "${dirName}" does not match its resolved name "${locale.baseName}"`
+      );
+    }
+    if (!matches(locale.baseName, dirName)) {
+      throw new Error(
+        `Matched locale "${dirName}" does not match its dir name "${dirName}"`
+      );
+    }
+
+    localeDirNames.push(dirName);
+  }
+
+  const jsonPath = path.join(ROOT_DIR, 'build', 'available-locales.json');
+  console.log(`Writing to "${jsonPath}"...`);
+  await fs.writeFile(jsonPath, `${JSON.stringify(localeDirNames, null, 2)}\n`);
+}
+
+main().catch(error => {
+  console.error(error);
+  process.exit(1);
+});

ts/scripts/get-strings.ts

@@ -2,6 +2,8 @@
 // SPDX-License-Identifier: AGPL-3.0-only
 
 import { execSync } from 'child_process';
+import fsExtra from 'fs-extra';
+import path from 'path';
 
 const { SMARTLING_USER, SMARTLING_SECRET } = process.env;
 
@@ -29,6 +31,19 @@ execSync(
   }
 );
 
+function rename(from: string, to: string) {
+  console.log(`Renaming "${from}" to "${to}"`);
+  fsExtra.moveSync(path.join('_locales', from), path.join('_locales', to), {
+    overwrite: true,
+  });
+}
+
+// Smartling uses "zh-YU" for Cantonese (or Yue Chinese).
+// This is wrong.
+// The language tag for Yue Chinese is "yue"
+// "zh-YU" actually implies "Chinese as spoken in Yugoslavia (canonicalized to Serbia)"
+rename('zh-YU', 'yue');
+
 console.log('Formatting newly-downloaded strings!');
 console.log();
 execSync('yarn format', {

ts/test-node/app/locale_test.ts

@@ -0,0 +1,46 @@
+// Copyright 2021 Signal Messenger, LLC
+// SPDX-License-Identifier: AGPL-3.0-only
+
+import { assert } from 'chai';
+import { load } from '../../../app/locale';
+import type { LoggerType } from '../../types/Logging';
+
+const logger: Pick<LoggerType, 'info' | 'warn'> = {
+  info(..._args: Array<unknown>) {
+    // noop
+  },
+  warn(..._args: Array<unknown>) {
+    throw new Error(String(_args));
+  },
+};
+
+describe('locale', async () => {
+  describe('load', () => {
+    it('resolves expected locales correctly', async () => {
+      async function testCase(
+        preferredSystemLocales: Array<string>,
+        expectedLocale: string
+      ) {
+        const actualLocale = await load({ preferredSystemLocales, logger });
+        assert.strictEqual(actualLocale.name, expectedLocale);
+      }
+
+      // Basic tests
+      await testCase(['en'], 'en');
+      await testCase(['es'], 'es');
+      await testCase(['fr', 'hk'], 'fr');
+      await testCase(['fr-FR', 'hk'], 'fr');
+      await testCase(['fa-UK'], 'fa-IR');
+      await testCase(['an', 'fr-FR'], 'fr'); // If we ever add support for Aragonese, this test will fail.
+
+      // Specific cases we want to ensure work as expected
+      await testCase(['zh-Hant-TW'], 'zh-TW');
+      await testCase(['zh-Hant-HK'], 'zh-HK');
+      await testCase(['zh'], 'zh-CN');
+      await testCase(['yue'], 'yue');
+      await testCase(['ug'], 'ug');
+      await testCase(['nn', 'nb'], 'nb');
+      await testCase(['es-419'], 'es');
+    });
+  });
+});

ts/test-node/app/menu_test.ts

@@ -202,9 +202,6 @@ describe('createTemplate', () => {
       info(_arg: unknown) {
         // noop
       },
-      error(arg: unknown) {
-        throw new Error(String(arg));
-      },
       warn(arg: unknown) {
         throw new Error(String(arg));
       },