Fixes for locale matching

app/locale.ts

@@ -85,6 +85,15 @@ function finalize(
   };
 }
 
+export function _getAvailableLocales(): Array<string> {
+  return JSON.parse(
+    readFileSync(
+      join(__dirname, '..', 'build', 'available-locales.json'),
+      'utf-8'
+    )
+  ) as Array<string>;
+}
+
 export function load({
   preferredSystemLocales,
   logger,
@@ -100,12 +109,7 @@ export function load({
     logger.warn('locale: `preferredSystemLocales` was empty');
   }
 
-  const availableLocales = JSON.parse(
-    readFileSync(
-      join(__dirname, '..', 'build', 'available-locales.json'),
-      'utf-8'
-    )
-  ) as Array<string>;
+  const availableLocales = _getAvailableLocales();
 
   logger.info('locale: Supported locales:', availableLocales.join(', '));
   logger.info('locale: Preferred locales: ', preferredSystemLocales.join(', '));

app/spell_check.ts

@@ -10,6 +10,14 @@ import { maybeParseUrl } from '../ts/util/url';
 
 import type { MenuListType } from '../ts/types/menu';
 import type { LocalizerType } from '../ts/types/Util';
+import { strictAssert } from '../ts/util/assert';
+
+export const FAKE_DEFAULT_LOCALE = 'en-x-ignore'; // -x- is an extension space for attaching other metadata to the locale
+
+strictAssert(
+  new Intl.Locale(FAKE_DEFAULT_LOCALE).toString() === FAKE_DEFAULT_LOCALE,
+  "Ensure Intl doesn't change our fake locale ever"
+);
 
 export function getLanguages(
   preferredSystemLocales: ReadonlyArray<string>,
@@ -19,17 +27,20 @@ export function getLanguages(
   const matchedLocales = [];
 
   preferredSystemLocales.forEach(preferredSystemLocale => {
-    if (preferredSystemLocale === defaultLocale) {
-      matchedLocales.push(defaultLocale);
-      return;
-    }
     const matchedLocale = LocaleMatcher.match(
       [preferredSystemLocale],
       availableLocales as Array<string>, // bad types
-      defaultLocale,
+      // We don't want to fallback to the default locale right away in case we might
+      // match some other locales first.
+      //
+      // However, we do want to match the default locale in case the user's locales
+      // actually matches it.
+      //
+      // This fake locale allows us to reliably filter it out within the loop.
+      FAKE_DEFAULT_LOCALE,
       { algorithm: 'best fit' }
     );
-    if (matchedLocale !== defaultLocale) {
+    if (matchedLocale !== FAKE_DEFAULT_LOCALE) {
       matchedLocales.push(matchedLocale);
     }
   });

build/available-locales.json

@@ -51,8 +51,7 @@
   "sk-SK",
   "sl-SI",
   "sq-AL",
-  "sr-RS",
-  "sr-YR",
+  "sr",
   "sv",
   "sw",
   "ta-IN",

ts/scripts/get-strings.ts

@@ -4,6 +4,7 @@
 import { execSync } from 'child_process';
 import fsExtra from 'fs-extra';
 import path from 'path';
+import fastGlob from 'fast-glob';
 
 const { SMARTLING_USER, SMARTLING_SECRET } = process.env;
 
@@ -16,6 +17,16 @@ if (!SMARTLING_SECRET) {
   process.exit(1);
 }
 
+console.log('Cleaning _locales directory...');
+const dirEntries = fastGlob.sync(['_locales/*', '!_locales/en'], {
+  onlyDirectories: true,
+  absolute: true,
+});
+
+for (const dirEntry of dirEntries) {
+  fsExtra.rmdirSync(dirEntry, { recursive: true });
+}
+
 console.log('Fetching latest strings!');
 console.log();
 execSync(
@@ -51,6 +62,9 @@ rename('zh-YU', 'yue');
 // we need to rename it to "zh-Hant" explicitly to make it work.
 rename('zh-TW', 'zh-Hant');
 
+// "YR" is not a valid region subtag. Smartling made it up.
+rename('sr-YR', 'sr');
+
 console.log('Formatting newly-downloaded strings!');
 console.log();
 execSync('yarn format', {

ts/test-node/app/locale_test.ts

@@ -3,26 +3,27 @@
 
 import { assert } from 'chai';
 import { stub } from 'sinon';
-import { load } from '../../../app/locale';
+import * as LocaleMatcher from '@formatjs/intl-localematcher';
+import { load, _getAvailableLocales } from '../../../app/locale';
+import { FAKE_DEFAULT_LOCALE } from '../../../app/spell_check';
 
 describe('locale', async () => {
   describe('load', () => {
     it('resolves expected locales correctly', async () => {
+      const logger = {
+        fatal: stub().throwsArg(0),
+        error: stub().throwsArg(0),
+        warn: stub().throwsArg(0),
+        info: stub(),
+        debug: stub(),
+        trace: stub(),
+      };
+
       async function testCase(
         preferredSystemLocales: Array<string>,
         expectedLocale: string
       ) {
-        const actualLocale = await load({
-          preferredSystemLocales,
-          logger: {
-            fatal: stub().throwsArg(0),
-            error: stub().throwsArg(0),
-            warn: stub().throwsArg(0),
-            info: stub(),
-            debug: stub(),
-            trace: stub(),
-          },
-        });
+        const actualLocale = await load({ preferredSystemLocales, logger });
         assert.strictEqual(actualLocale.name, expectedLocale);
       }
 
@@ -44,6 +45,123 @@ describe('locale', async () => {
       await testCase(['ug'], 'ug');
       await testCase(['nn', 'nb'], 'nb');
       await testCase(['es-419'], 'es');
+      await testCase(['sr-RO', 'sr'], 'sr');
+      await testCase(['sr-RS', 'sr'], 'sr');
+    });
+  });
+
+  describe('Intl.LocaleMatcher', () => {
+    it('should work for single locales outside of their region', () => {
+      // Our supported locales where we only have a single region for a language
+      const SINGLE_REGION_LOCALES = [
+        'af-ZA',
+        'az-AZ',
+        'bg-BG',
+        'bn-BD',
+        'bs-BA',
+        'et-EE',
+        'fa-IR',
+        'ga-IE',
+        'gl-ES',
+        'gu-IN',
+        'hi-IN',
+        'hr-HR',
+        'ka-GE',
+        'kk-KZ',
+        'km-KH',
+        'kn-IN',
+        'ky-KG',
+        'lt-LT',
+        'lv-LV',
+        'mk-MK',
+        'ml-IN',
+        'mr-IN',
+        'pa-IN',
+        'ro-RO',
+        'sk-SK',
+        'sl-SI',
+        'sq-AL',
+        'ta-IN',
+        'te-IN',
+        'tl-PH',
+        'uk-UA',
+      ];
+
+      // Just a whole bunch of common regions
+      const TEST_REGIONS = [
+        'AE',
+        'AR',
+        'AT',
+        'AU',
+        'BE',
+        'CA',
+        'CH',
+        'CL',
+        'CN',
+        'CO',
+        'CR',
+        'CZ',
+        'DE',
+        'DK',
+        'DO',
+        'EC',
+        'EG',
+        'ES',
+        'FI',
+        'FR',
+        'GB',
+        'GR',
+        'GT',
+        'HK',
+        'IE',
+        'IL',
+        'IN',
+        'IT',
+        'JP',
+        'KR',
+        'MY',
+        'NL',
+        'NO',
+        'PA',
+        'PE',
+        'PH',
+        'PL',
+        'PT',
+        'RO',
+        'RS',
+        'RU',
+        'SA',
+        'SE',
+        'SG',
+        'SK',
+        'TH',
+        'TR',
+        'UA',
+        'US',
+        'VE',
+      ];
+
+      const availableLocales = _getAvailableLocales();
+
+      for (const locale of SINGLE_REGION_LOCALES) {
+        const { language } = new Intl.Locale(locale);
+        for (const region of TEST_REGIONS) {
+          const newLocale = new Intl.Locale(language, { region });
+
+          const matched = LocaleMatcher.match(
+            [newLocale.baseName],
+            availableLocales,
+            FAKE_DEFAULT_LOCALE,
+            { algorithm: 'best fit' }
+          );
+
+          assert.strictEqual(
+            matched,
+            locale,
+            `${locale} -> ${language} -> ${region} -> ${newLocale.baseName} -> ${matched}`
+          );
+        }
+      }
     });
   });
 });

ts/test-node/app/spell_check_test.ts

@@ -52,5 +52,12 @@ describe('SpellCheck', () => {
         'en',
       ]);
     });
+
+    it('matches en along with other languages', () => {
+      assert.deepEqual(getLanguages(['en', 'fr'], ['fr', 'en'], 'en'), [
+        'en',
+        'fr',
+      ]);
+    });
   });
 });