Move to smartling for translation services

ts/components/App.tsx

@@ -7,7 +7,6 @@ import { Globals } from '@react-spring/web';
 import classNames from 'classnames';
 
 import type { ExecuteMenuRoleType } from './TitleBarContainer';
-import type { LocaleMessagesType } from '../types/I18N';
 import type { MenuOptionsType, MenuActionType } from '../types/menu';
 import type { ToastType } from '../state/ducks/toast';
 import type { ViewStoryActionCreatorType } from '../state/ducks/stories';
@@ -24,7 +23,6 @@ import { useReducedMotion } from '../hooks/useReducedMotion';
 
 type PropsType = {
   appView: AppViewType;
-  localeMessages: LocaleMessagesType;
   openInbox: () => void;
   registerSingleDevice: (number: string, code: string) => Promise<void>;
   renderCallManager: () => JSX.Element;
@@ -71,7 +69,6 @@ export const App = ({
   isMaximized,
   isShowingStoriesView,
   hasCustomTitleBar,
-  localeMessages,
   menuOptions,
   openInbox,
   registerSingleDevice,
@@ -163,7 +160,7 @@ export const App = ({
       titleBarDoubleClick={titleBarDoubleClick}
       hasMenu
       hideMenuBar={hideMenuBar}
-      localeMessages={localeMessages}
+      i18n={i18n}
       menuOptions={menuOptions}
       executeMenuAction={executeMenuAction}
     >

ts/components/TitleBarContainer.tsx

@@ -10,13 +10,13 @@ import classNames from 'classnames';
 
 import { createTemplate } from '../../app/menu';
 import { ThemeType } from '../types/Util';
-import type { LocaleMessagesType } from '../types/I18N';
+import type { LocalizerType } from '../types/I18N';
 import type { MenuOptionsType, MenuActionType } from '../types/menu';
 import { useIsWindowActive } from '../hooks/useIsWindowActive';
 
 export type MenuPropsType = Readonly<{
   hasMenu: true;
-  localeMessages: LocaleMessagesType;
+  i18n: LocalizerType;
   menuOptions: MenuOptionsType;
   executeMenuAction: (action: MenuActionType) => void;
 }>;
@@ -64,7 +64,7 @@ ROLE_TO_ACCELERATOR.set('minimize', 'CmdOrCtrl+M');
 function convertMenu(
   menuList: ReadonlyArray<MenuItemConstructorOptions>,
   executeMenuRole: (role: MenuItemConstructorOptions['role']) => void,
-  localeMessages: LocaleMessagesType
+  i18n: LocalizerType
 ): Array<MenuItem> {
   return menuList.map(item => {
     const {
@@ -78,7 +78,7 @@ function convertMenu(
     let submenu: Array<MenuItem> | undefined;
 
     if (Array.isArray(originalSubmenu)) {
-      submenu = convertMenu(originalSubmenu, executeMenuRole, localeMessages);
+      submenu = convertMenu(originalSubmenu, executeMenuRole, i18n);
     } else if (originalSubmenu) {
       throw new Error('Non-array submenu is not supported');
     }
@@ -107,12 +107,9 @@ function convertMenu(
     // `app/main.ts`.
     accelerator = accelerator?.replace(
       /CommandOrControl|CmdOrCtrl/g,
-      localeMessages['Keyboard--Key--ctrl'].message
-    );
-    accelerator = accelerator?.replace(
-      /Shift/g,
-      localeMessages['Keyboard--Key--shift'].message
+      i18n('Keyboard--Key--ctrl')
     );
+    accelerator = accelerator?.replace(/Shift/g, i18n('Keyboard--Key--shift'));
 
     return {
       type,
@@ -221,7 +218,7 @@ export const TitleBarContainer = (props: PropsType): JSX.Element => {
 
   let maybeMenu: Array<MenuItem> | undefined;
   if (hasMenu) {
-    const { localeMessages, menuOptions, executeMenuAction } = props;
+    const { i18n, menuOptions, executeMenuAction } = props;
 
     const menuTemplate = createTemplate(
       {
@@ -243,10 +240,10 @@ export const TitleBarContainer = (props: PropsType): JSX.Element => {
         showStickerCreator: () => executeMenuAction('showStickerCreator'),
         showWindow: () => executeMenuAction('showWindow'),
       },
-      localeMessages
+      i18n
     );
 
-    maybeMenu = convertMenu(menuTemplate, executeMenuRole, localeMessages);
+    maybeMenu = convertMenu(menuTemplate, executeMenuRole, i18n);
   }
 
   return (

ts/scripts/copy-language-packs.ts

@@ -19,10 +19,10 @@ export async function afterPack({
   if (electronPlatformName === 'darwin') {
     const { productFilename } = packager.appInfo;
 
-    // en.lproj/locale.pak
-    // zh_CN.lproj/locale.pak
+    // en.lproj/*
+    // zh_CN.lproj/*
     defaultLocale = 'en.lproj';
-    ourLocales = ourLocales.map(locale => `${locale}.lproj`);
+    ourLocales = ourLocales.map(locale => `${locale.replace(/-/g, '_')}.lproj`);
 
     localesPath = path.join(
       appOutDir,
@@ -35,6 +35,8 @@ export async function afterPack({
     electronPlatformName === 'win32'
   ) {
     // Shared between windows and linux
+    // en-US.pak
+    // zh-CN.pak
     defaultLocale = 'en-US.pak';
     ourLocales = ourLocales.map(locale => {
       if (locale === 'en') {

ts/scripts/get-strings.ts

@@ -1,76 +1,36 @@
 // Copyright 2021 Signal Messenger, LLC
 // SPDX-License-Identifier: AGPL-3.0-only
 
-import { join, resolve } from 'path';
-import { existsSync, readdirSync, writeFileSync } from 'fs';
 import { execSync } from 'child_process';
 
-import { readJsonSync } from 'fs-extra';
-import type { LocaleMessagesType } from '../types/I18N';
-import * as Errors from '../types/errors';
+const { SMARTLING_USER, SMARTLING_SECRET } = process.env;
 
-console.log('Getting latest strings!');
-
-// Note: we continue after tx failures so we always restore placeholders on json files
-let failed = false;
-
-console.log();
-console.log('Getting strings, allow for new ones over 80% translated');
-try {
-  execSync('tx pull --all --use-git-timestamps --minimum-perc=80', {
-    stdio: [null, process.stdout, process.stderr],
-  });
-} catch (error: unknown) {
-  failed = true;
-  console.log(
-    'Failed first tx fetch, continuing...',
-    Errors.toLogFormat(error)
-  );
+if (!SMARTLING_USER) {
+  console.error('Need to set SMARTLING_USER environment variable!');
+  process.exit(1);
+}
+if (!SMARTLING_SECRET) {
+  console.error('Need to set SMARTLING_SECRET environment variable!');
+  process.exit(1);
 }
 
+console.log('Fetching latest strings!');
 console.log();
-console.log('Getting strings, updating everything previously missed');
-try {
-  execSync('tx pull --use-git-timestamps', {
+execSync(
+  'smartling-cli' +
+    ` --user "${SMARTLING_USER}"` +
+    ` --secret "${SMARTLING_SECRET}"` +
+    ' --config .smartling.yml' +
+    ' --verbose' +
+    ' --format "_locales/{{.Locale}}/messages.json"' +
+    ' files pull',
+  {
     stdio: [null, process.stdout, process.stderr],
-  });
-} catch (error: unknown) {
-  failed = true;
-  console.log(
-    'Failed second tx fetch, continuing...',
-    Errors.toLogFormat(error)
-  );
-}
-
-const BASE_DIR = join(__dirname, '../../_locales');
-const locales = readdirSync(join(BASE_DIR, ''));
-
-console.log();
-console.log('Deleting placeholders for all locales');
-locales.forEach((locale: string) => {
-  const target = resolve(join(BASE_DIR, locale, 'messages.json'));
-  if (!existsSync(target)) {
-    console.warn(`File not found for ${locale}: ${target}`);
-    return;
   }
+);
 
-  const messages: LocaleMessagesType = readJsonSync(target);
-  Object.keys(messages).forEach(key => {
-    delete messages[key].placeholders;
-
-    if (!messages[key].description) {
-      delete messages[key].description;
-    }
-  });
-
-  console.log(`Writing ${target}`);
-  writeFileSync(target, `${JSON.stringify(messages, null, 4)}\n`);
-});
-
+console.log('Formatting newly-downloaded strings!');
+console.log();
 execSync('yarn format', {
   stdio: [null, process.stdout, process.stderr],
 });
-
-if (failed) {
-  process.exit(1);
-}

ts/scripts/push-strings.ts

@@ -0,0 +1,29 @@
+// Copyright 2021 Signal Messenger, LLC
+// SPDX-License-Identifier: AGPL-3.0-only
+
+import { execSync } from 'child_process';
+
+const { SMARTLING_USER, SMARTLING_SECRET } = process.env;
+
+if (!SMARTLING_USER) {
+  console.error('Need to set SMARTLING_USER environment variable!');
+  process.exit(1);
+}
+if (!SMARTLING_SECRET) {
+  console.error('Need to set SMARTLING_SECRET environment variable!');
+  process.exit(1);
+}
+
+console.log('Pushing latest strings!');
+console.log();
+execSync(
+  'smartling-cli' +
+    ` --user "${SMARTLING_USER}"` +
+    ` --secret "${SMARTLING_SECRET}"` +
+    ' --config .smartling.yml' +
+    ' --verbose' +
+    ' files push _locales/en/messages.json',
+  {
+    stdio: [null, process.stdout, process.stderr],
+  }
+);

ts/test-both/util/expirationTimer_test.ts

@@ -7,9 +7,9 @@ import { setupI18n } from '../../util/setupI18n';
 import enMessages from '../../../_locales/en/messages.json';
 import esMessages from '../../../_locales/es/messages.json';
 import nbMessages from '../../../_locales/nb/messages.json';
-import nnMessages from '../../../_locales/nn/messages.json';
-import ptBrMessages from '../../../_locales/pt_BR/messages.json';
-import zhCnMessages from '../../../_locales/zh_CN/messages.json';
+import nlMessages from '../../../_locales/nl/messages.json';
+import ptBrMessages from '../../../_locales/pt-BR/messages.json';
+import zhCnMessages from '../../../_locales/zh-CN/messages.json';
 
 import * as expirationTimer from '../../util/expirationTimer';
 
@@ -67,7 +67,7 @@ describe('expiration timer utilities', () => {
       const esI18n = setupI18n('es', esMessages);
       assert.strictEqual(format(esI18n, 120), '2 minutos');
 
-      const zhCnI18n = setupI18n('zh_CN', zhCnMessages);
+      const zhCnI18n = setupI18n('zh-CN', zhCnMessages);
       assert.strictEqual(format(zhCnI18n, 60), '1 分钟');
 
       // The underlying library supports the "pt" locale, not the "pt_BR" locale. That's
@@ -80,7 +80,7 @@ describe('expiration timer utilities', () => {
 
       // The underlying library supports the Norwegian language, which is a macrolanguage
       //   for Bokmål and Nynorsk.
-      [setupI18n('nb', nbMessages), setupI18n('nn', nnMessages)].forEach(
+      [setupI18n('nb', nbMessages), setupI18n('nn', nlMessages)].forEach(
         norwegianI18n => {
           assert.strictEqual(
             format(norwegianI18n, moment.duration(6, 'hours').asSeconds()),

ts/test-both/util/getFontNameByTextScript_test.ts

@@ -92,7 +92,7 @@ describe('getFontNameByTextScript', () => {
   it('returns the correct font names (chinese simplified)', () => {
     const text = '敏捷的棕色狐狸跳过了懒狗';
 
-    const actual = getFontNameByTextScript(text, 0, setupI18n('zh_CN', {}));
+    const actual = getFontNameByTextScript(text, 0, setupI18n('zh-CN', {}));
     const expected = '"PingFang SC Regular", SimHei, sans-serif';
     assert.equal(actual, expected);
   });
@@ -100,7 +100,7 @@ describe('getFontNameByTextScript', () => {
   it('returns the correct font names (chinese traditional)', () => {
     const text = '敏捷的棕色狐狸跳過了懶狗';
 
-    const actual = getFontNameByTextScript(text, 0, setupI18n('zh_TW', {}));
+    const actual = getFontNameByTextScript(text, 0, setupI18n('zh-TW', {}));
     const expected = '"PingFang TC Regular", "JhengHei TC Regular", sans-serif';
     assert.equal(actual, expected);
   });

ts/test-node/app/SystemTrayService_test.ts

@@ -10,6 +10,11 @@ import { MINUTE } from '../../util/durations';
 
 import type { SystemTrayServiceOptionsType } from '../../../app/SystemTrayService';
 import { SystemTrayService } from '../../../app/SystemTrayService';
+import { setupI18n } from '../../util/setupI18n';
+
+import enMessages from '../../../_locales/en/messages.json';
+
+const i18n = setupI18n('en', enMessages);
 
 describe('SystemTrayService', function thisNeeded() {
   // These tests take more time on CI in some cases, so we increase the timeout.
@@ -28,12 +33,7 @@ describe('SystemTrayService', function thisNeeded() {
     options?: Partial<SystemTrayServiceOptionsType>
   ): SystemTrayService {
     const result = new SystemTrayService({
-      messages: {
-        hide: { message: 'Hide' },
-        quit: { message: 'Quit' },
-        show: { message: 'Show' },
-        signalDesktop: { message: 'Signal' },
-      },
+      i18n,
       ...options,
     });
     servicesCreated.add(result);

ts/test-node/app/menu_test.ts

@@ -196,7 +196,7 @@ const PLATFORMS = [
 ];
 
 describe('createTemplate', () => {
-  const { messages } = loadLocale({
+  const { i18n } = loadLocale({
     appLocale: 'en',
     logger: {
       error(arg: unknown) {
@@ -237,7 +237,7 @@ describe('createTemplate', () => {
           ...actions,
         };
 
-        const actual = createTemplate(options, messages);
+        const actual = createTemplate(options, i18n);
         assert.deepEqual(actual, expectedDefault);
       });
 
@@ -265,7 +265,7 @@ describe('createTemplate', () => {
           return menuItem;
         });
 
-        const actual = createTemplate(options, messages);
+        const actual = createTemplate(options, i18n);
         assert.deepEqual(actual, expected);
       });
     });

ts/types/I18N.ts

@@ -1,21 +1,39 @@
 // Copyright 2020 Signal Messenger, LLC
 // SPDX-License-Identifier: AGPL-3.0-only
 
+/* eslint-disable camelcase */
+
 import type { LocalizerType } from './Util';
 
-export type LocaleMessagesType = {
-  [key: string]: {
-    message: string;
-    description?: string;
-    placeholders?: {
-      [name: string]: {
-        content: string;
-        example: string;
-      };
+export type { LocalizerType } from './Util';
+
+type SmartlingConfigType = {
+  placeholder_format_custom: string;
+  translate_paths: Array<{
+    key: string;
+    path: string;
+    instruction: string;
+  }>;
+};
+
+type LocaleMessageType = {
+  message: string;
+  description?: string;
+  placeholders?: {
+    [name: string]: {
+      content: string;
+      example: string;
     };
   };
 };
 
+export type LocaleMessagesType = {
+  // In practice, 'smartling' is the only key which is a SmartlingConfigType, but
+  //  we get typescript error 2411 (incompatible type signatures) if we try to
+  //  special-case that key.
+  [key: string]: LocaleMessageType | SmartlingConfigType;
+};
+
 export type ReplacementValuesType<T> = {
   [key: string]: T;
 };

ts/util/expirationTimer.ts

@@ -42,7 +42,7 @@ export function format(
 
   // locale strings coming from electron use a dash as separator
   // but humanizeDuration uses an underscore
-  const locale: string = i18n.getLocale().replace('-', '_');
+  const locale: string = i18n.getLocale().replace(/-/g, '_');
 
   const localeWithoutRegion: string = locale.split('_', 1)[0];
   const fallbacks: Array<string> = [];
@@ -56,6 +56,11 @@ export function format(
     fallbacks.push('en');
   }
 
+  // humanizeDuration only supports zh_CN and zh_TW
+  if (locale === 'zh_HK') {
+    fallbacks.push('zh_TW');
+  }
+
   const allUnits: Array<Unit> = ['y', 'mo', 'w', 'd', 'h', 'm', 's'];
 
   const defaultUnits: Array<Unit> =

ts/util/getFontNameByTextScript.ts

@@ -131,9 +131,9 @@ export function getFontNameByTextScript(
   if (fontSniffer.hasCJK(text)) {
     const locale = i18n?.getLocale();
 
-    if (locale === 'zh_TW') {
+    if (locale === 'zh-TW') {
       fonts.push(FONT_MAP.zhtc[textStyleIndex]);
-    } else if (locale === 'zh_HK') {
+    } else if (locale === 'zh-HK') {
       fonts.push(FONT_MAP.zhhk[textStyleIndex]);
     } else {
       fonts.push(FONT_MAP.zhsc[textStyleIndex]);

ts/util/lint/license_comments.ts

@@ -38,6 +38,7 @@ const FILES_TO_IGNORE = new Set(
   [
     '.github/ISSUE_TEMPLATE/bug_report.md',
     '.github/PULL_REQUEST_TEMPLATE.md',
+    '.smartling-source.sh',
     'components/mp3lameencoder/lib/Mp3LameEncoder.js',
     'components/recorderjs/recorder.js',
     'components/recorderjs/recorderWorker.js',

ts/util/setupI18n.ts

@@ -18,7 +18,7 @@ export function setupI18n(
 
   const getMessage: LocalizerType = (key, substitutions) => {
     const entry = messages[key];
-    if (!entry) {
+    if (!entry || !('message' in entry)) {
       log.error(
         `i18n: Attempted to get translation for nonexistent key '${key}'`
       );